mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-16 18:08:21 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 唐小鸭
|
a609896511 |
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: rustfs-logging-governance
|
||||
description: Standardize and review RustFS logging with structured `tracing` events, lower noise on hot paths, preserve security-sensitive diagnostics, and extend guardrails to prevent legacy logging patterns from returning. Use whenever a change adds or edits any `tracing` macro call (`error!`/`warn!`/`info!`/`debug!`/`trace!`/`#[instrument]`) — including a single log line added in passing while fixing unrelated logic, which is how most new log sites enter the repo — and when reviewing RustFS logs, startup/config diagnostics, cloud metadata logs, request validation logs, or `scripts/check_logging_guardrails.sh`.
|
||||
description: Standardize and review RustFS logging with structured `tracing` events, lower noise on hot paths, preserve security-sensitive diagnostics, and extend guardrails to prevent legacy logging patterns from returning. Use when editing or reviewing RustFS logs, startup/config diagnostics, cloud metadata logs, request validation logs, or `scripts/check_logging_guardrails.sh`.
|
||||
---
|
||||
|
||||
# RustFS Logging Governance
|
||||
|
||||
@@ -60,11 +60,6 @@ body-cache-whitelist-check: ## Check the body-cache eligibility gate stays a fai
|
||||
@echo "🧱 Checking body-cache whitelist guard..."
|
||||
./scripts/check_body_cache_whitelist.sh
|
||||
|
||||
.PHONY: s3s-footprint-check
|
||||
s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
@echo "📦 Checking s3s footprint ratchet..."
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: fips-wording-check
|
||||
fips-wording-check: ## Check outward docs do not make unsupported FIPS claims
|
||||
@echo "📣 Checking FIPS wording guard..."
|
||||
|
||||
@@ -19,13 +19,13 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check fips-wording-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
@echo "✅ Fast development checks passed!"
|
||||
|
||||
+9
-45
@@ -29,8 +29,6 @@
|
||||
|
||||
[test-groups]
|
||||
ecstore-serial-flaky = { max-threads = 1 }
|
||||
embedded-test-ports = { max-threads = 1 }
|
||||
e2e-vault = { max-threads = 1 }
|
||||
|
||||
# Reliability / fault-injection e2e tests each spawn a single-node 4-disk RustFS
|
||||
# server and manipulate its disk directories at runtime (crates/e2e_test:
|
||||
@@ -56,20 +54,6 @@ test-group = 'ecstore-serial-flaky'
|
||||
filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# The production-handler relocation regression builds an isolated 8-disk,
|
||||
# 2-pool store and commits a 72 MiB multipart object. Keep that cross-disk IO
|
||||
# from overlapping the ecstore commit fixtures above.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & test(execute_get_object_resumes_from_relocated_pool_without_splicing_body)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Embedded integration-test binaries discover an ephemeral port and release
|
||||
# the probe listener before RustFS binds it. Serialize that cross-process
|
||||
# TOCTOU window; retries would only hide real startup failures.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
# Serialize the durable manual-transition checkpoint test across nextest's
|
||||
# process boundary; it mutates bucket lifecycle metadata and is not quarantined.
|
||||
[[profile.default.overrides]]
|
||||
@@ -97,12 +81,6 @@ test-group = 'e2e-reliability'
|
||||
filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
|
||||
test-group = 'e2e-inline-boundaries'
|
||||
|
||||
# Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
|
||||
# does not cross nextest process boundaries, so keep these tests in one group.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -165,16 +143,6 @@ test-group = 'e2e-reliability'
|
||||
filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & test(execute_get_object_resumes_from_relocated_pool_without_splicing_body)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Match the default-profile embedded test isolation without quarantining or
|
||||
# retrying failures in CI.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
# Serialize the durable manual-transition checkpoint test under the ci profile
|
||||
# too. No retries: failures stay visible.
|
||||
[[profile.ci.overrides]]
|
||||
@@ -218,7 +186,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
# the nightly profile derives its set as "the replication module MINUS this
|
||||
# allowlist", so any new replication test lands in nightly by default (never
|
||||
# silently unrun) until it is explicitly blessed as fast here. Keep the two
|
||||
# regexes byte-identical. Count invariant: 20 here + 36 nightly = 56 total
|
||||
# regexes byte-identical. Count invariant: 20 here + 28 nightly = 48 total
|
||||
# (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
|
||||
# HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
|
||||
# (#4724) because they set a loopback (127.0.0.1) replication target that the
|
||||
@@ -254,7 +222,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^reliant::tiering::/)
|
||||
@@ -280,10 +248,10 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
|
||||
# tests that are unfit for the per-PR e2e-smoke gate:
|
||||
#
|
||||
# * 2 remote-target TLS validation tests.
|
||||
# * 13 bucket-replication data-plane/helper tests — they PUT/delete objects
|
||||
# and poll until source and target converge; two replicate over HTTPS,
|
||||
# four pin active SSE fail-closed contracts (SSE-C, SSE-S3, SSE-KMS, and
|
||||
# the SSE-S3 resync path), and one guards event/history observers.
|
||||
# * 12 bucket-replication data-plane/helper tests — they PUT/delete objects
|
||||
# and poll until source and target converge; two replicate over HTTPS, two
|
||||
# pin active SSE failure contracts, and one guards event/history observers.
|
||||
# The SSE-S3 contract remains ignored under backlog#1291.
|
||||
# * 12 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
|
||||
# servers and drives the cross-process site-replication control plane.
|
||||
# * 1 `_real_three_node` site-replication test.
|
||||
@@ -349,9 +317,9 @@ path = "junit.xml"
|
||||
#
|
||||
# Each e2e test spawns its own single-node rustfs server on a random port with
|
||||
# an isolated temp dir (crates/e2e_test/src/common.rs), so the set is
|
||||
# parallel-safe — the same property e2e-smoke relies on. The exceptions are the
|
||||
# 4-disk reliability / degraded-read fault-injection tests and the fixed-port
|
||||
# Vault tests, both serialized below.
|
||||
# parallel-safe — the same property e2e-smoke relies on. The exception is the
|
||||
# 4-disk reliability / degraded-read fault-injection tests, serialized below
|
||||
# (identical to the ci profile) so several 4-disk servers never run at once.
|
||||
# KNOWN-FAILURE EXCLUSIONS (characterization run 29381309848, 2026-07-15:
|
||||
# 341 ran / 32 failed on the suites' first automated run ever). Deterministic
|
||||
# product failures cannot be quarantined away with retries, so each family is
|
||||
@@ -387,7 +355,3 @@ test-group = 'e2e-reliability'
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
|
||||
test-group = 'e2e-inline-boundaries'
|
||||
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
@@ -170,10 +170,6 @@ Important behavior notes:
|
||||
|
||||
- Logs and metrics usually appear during startup, so seeing those two signals
|
||||
first is expected.
|
||||
- The OpenTelemetry bridge sends `tracing` fields as log attributes. Loki stores
|
||||
those attributes as structured metadata, and the Collector also mirrors the
|
||||
common troubleshooting fields into the log line so simple line filters can
|
||||
find them.
|
||||
- Visible trace data usually requires real HTTP/S3/gRPC request traffic after
|
||||
startup, because request-path spans are created on demand.
|
||||
- `RUSTFS_OBS_LOGGER_LEVEL=info` keeps the top-level request span but filters
|
||||
@@ -199,17 +195,6 @@ curl -I http://127.0.0.1:9000/health/ready
|
||||
# Jaeger: http://localhost:16686
|
||||
```
|
||||
|
||||
For a structured RustFS log such as an inter-node RPC authentication failure,
|
||||
the Loki line now includes fields such as `event`, `component`, `subsystem`,
|
||||
`failure_reason`, `rpc_service`, `rpc_method`, and `expected_audience`. Useful
|
||||
LogQL checks:
|
||||
|
||||
```logql
|
||||
{service_name="RustFS"} |= "RPC signature verification failed"
|
||||
{service_name="RustFS"} |= "failure_reason="
|
||||
{service_name="RustFS"} | failure_reason != ""
|
||||
```
|
||||
|
||||
If logs and metrics are present but traces are sparse, the most common cause is
|
||||
"no real request traffic yet" or "`info` level filtered nested spans", not an
|
||||
OTLP routing failure.
|
||||
|
||||
@@ -169,7 +169,6 @@ RustFS 会自动在该基础 URL 后补全：
|
||||
需要注意：
|
||||
|
||||
- 启动阶段通常会先看到日志和指标，因此“先有日志/指标、后有 trace”是正常现象。
|
||||
- OpenTelemetry bridge 会把 `tracing` 字段作为日志 attributes 发送。Loki 会将这些 attributes 存为 structured metadata，同时 Collector 会把常用排障字段镜像进日志行，方便用简单的行内容过滤直接查到。
|
||||
- 可见的 trace 数据通常依赖启动后的真实 HTTP/S3/gRPC 请求流量，因为请求路径上的 span 是按需创建的。
|
||||
- `RUSTFS_OBS_LOGGER_LEVEL=info` 会保留顶层请求 span，但会过滤掉很多 `debug` 级别的嵌套 span。
|
||||
如果 Tempo 或 Jaeger 中的 trace 看起来很稀疏，建议先改成 `RUSTFS_OBS_LOGGER_LEVEL=debug`，再判断是否是 collector 或 Tempo 问题。
|
||||
@@ -193,14 +192,6 @@ curl -I http://127.0.0.1:9000/health/ready
|
||||
# Jaeger: http://localhost:16686
|
||||
```
|
||||
|
||||
对于 RustFS 结构化日志，例如节点间 RPC 鉴权失败，Loki 日志行现在会包含 `event`、`component`、`subsystem`、`failure_reason`、`rpc_service`、`rpc_method`、`expected_audience` 等字段。常用 LogQL 检查：
|
||||
|
||||
```logql
|
||||
{service_name="RustFS"} |= "RPC signature verification failed"
|
||||
{service_name="RustFS"} |= "failure_reason="
|
||||
{service_name="RustFS"} | failure_reason != ""
|
||||
```
|
||||
|
||||
如果日志和指标已经正常，但 trace 仍然稀疏，最常见的原因通常是
|
||||
“还没有真实请求流量”或“`info` 级别过滤了嵌套 span”，而不是 OTLP 路由失败。
|
||||
|
||||
|
||||
@@ -29,27 +29,11 @@ processors:
|
||||
limit_mib: 1024
|
||||
spike_limit_mib: 256
|
||||
transform/logs:
|
||||
error_mode: ignore
|
||||
log_statements:
|
||||
- context: log
|
||||
statements:
|
||||
- set(attributes["message"], body.string) where IsString(body)
|
||||
- set(attributes["log.body"], body.string) where IsString(body)
|
||||
- set(body, Concat([body, " event=", attributes["event"]], "")) where IsString(body) and attributes["event"] != nil
|
||||
- set(body, Concat([body, " component=", attributes["component"]], "")) where IsString(body) and attributes["component"] != nil
|
||||
- set(body, Concat([body, " subsystem=", attributes["subsystem"]], "")) where IsString(body) and attributes["subsystem"] != nil
|
||||
- set(body, Concat([body, " state=", attributes["state"]], "")) where IsString(body) and attributes["state"] != nil
|
||||
- set(body, Concat([body, " result=", attributes["result"]], "")) where IsString(body) and attributes["result"] != nil
|
||||
- set(body, Concat([body, " reason=", attributes["reason"]], "")) where IsString(body) and attributes["reason"] != nil
|
||||
- set(body, Concat([body, " failure_reason=", attributes["failure_reason"]], "")) where IsString(body) and attributes["failure_reason"] != nil
|
||||
- set(body, Concat([body, " rpc_path=", attributes["rpc_path"]], "")) where IsString(body) and attributes["rpc_path"] != nil
|
||||
- set(body, Concat([body, " rpc_service=", attributes["rpc_service"]], "")) where IsString(body) and attributes["rpc_service"] != nil
|
||||
- set(body, Concat([body, " rpc_method=", attributes["rpc_method"]], "")) where IsString(body) and attributes["rpc_method"] != nil
|
||||
- set(body, Concat([body, " expected_audience=", attributes["expected_audience"]], "")) where IsString(body) and attributes["expected_audience"] != nil
|
||||
- set(body, Concat([body, " peer_addr=", attributes["peer_addr"]], "")) where IsString(body) and attributes["peer_addr"] != nil
|
||||
- set(body, Concat([body, " replay_scope_bootstrap_allowed=", attributes["replay_scope_bootstrap_allowed"]], "")) where IsString(body) and attributes["replay_scope_bootstrap_allowed"] != nil
|
||||
- set(body, Concat([body, " error=", attributes["error"]], "")) where IsString(body) and attributes["error"] != nil
|
||||
- set(body, Concat([body, " exception_message=", attributes["exception.message"]], "")) where IsString(body) and attributes["exception.message"] != nil
|
||||
- set(attributes["message"], body.string)
|
||||
- set(attributes["log.body"], body.string)
|
||||
|
||||
exporters:
|
||||
otlp/tempo:
|
||||
|
||||
@@ -102,9 +102,6 @@ jobs:
|
||||
- name: Check architecture migration rules
|
||||
run: ./scripts/check_architecture_migration_rules.sh
|
||||
|
||||
- name: Check logging guardrails
|
||||
run: ./scripts/check_logging_guardrails.sh
|
||||
|
||||
- name: Check tokio io-uring feature guard
|
||||
run: ./scripts/check_no_tokio_io_uring.sh
|
||||
|
||||
@@ -114,9 +111,6 @@ jobs:
|
||||
- name: Check body-cache whitelist guard
|
||||
run: ./scripts/check_body_cache_whitelist.sh
|
||||
|
||||
- name: Check s3s footprint ratchet
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
|
||||
@@ -137,9 +137,6 @@ jobs:
|
||||
- name: Check architecture migration rules
|
||||
run: ./scripts/check_architecture_migration_rules.sh
|
||||
|
||||
- name: Check logging guardrails
|
||||
run: ./scripts/check_logging_guardrails.sh
|
||||
|
||||
- name: Check tokio io-uring feature guard
|
||||
run: ./scripts/check_no_tokio_io_uring.sh
|
||||
|
||||
@@ -149,9 +146,6 @@ jobs:
|
||||
- name: Check body-cache whitelist guard
|
||||
run: ./scripts/check_body_cache_whitelist.sh
|
||||
|
||||
- name: Check s3s footprint ratchet
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
@@ -770,32 +764,6 @@ jobs:
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install awscurl
|
||||
run: |
|
||||
python3 -m pip install --user --upgrade pip "awscurl==0.44"
|
||||
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify awscurl
|
||||
run: test -x "$AWSCURL_PATH"
|
||||
|
||||
- name: Install Vault
|
||||
run: |
|
||||
VAULT_VERSION="1.17.6"
|
||||
VAULT_ARCHIVE="vault_${VAULT_VERSION}_linux_amd64.zip"
|
||||
curl -fsSLo "$RUNNER_TEMP/$VAULT_ARCHIVE" "https://releases.hashicorp.com/vault/${VAULT_VERSION}/${VAULT_ARCHIVE}"
|
||||
echo "0cddc1fbbb88583b5ba5b845f9f8fae47c6fb39a6d48cd543c6ba6fd3ac1a669 $RUNNER_TEMP/$VAULT_ARCHIVE" | sha256sum --check --status
|
||||
unzip -q "$RUNNER_TEMP/$VAULT_ARCHIVE" -d "$RUNNER_TEMP/vault-bin"
|
||||
echo "RUSTFS_TEST_VAULT_BIN=$RUNNER_TEMP/vault-bin/vault" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify Vault
|
||||
run: |
|
||||
"$RUSTFS_TEST_VAULT_BIN" version
|
||||
|
||||
# Download after the cache restore so the freshly built binary from the
|
||||
# build job always wins over anything restored into target/debug.
|
||||
- name: Download debug binary
|
||||
|
||||
@@ -1,463 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Package Workflow - Build DEB/RPM packages
|
||||
#
|
||||
# This workflow builds DEB and RPM packages from pre-built Linux binaries
|
||||
# and uploads them to Cloudflare R2.
|
||||
#
|
||||
# Trigger:
|
||||
# - release published: automatically package when a GitHub release is published
|
||||
# - workflow_dispatch: manual trigger with optional tag/run_id
|
||||
#
|
||||
# Flow:
|
||||
# 1. Find the Build workflow run for the release tag
|
||||
# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
|
||||
# 3. Build DEB packages for amd64 and arm64
|
||||
# 4. Build RPM packages for x86_64 and aarch64
|
||||
# 5. Upload all packages to Cloudflare R2
|
||||
|
||||
name: Package DEB/RPM
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [ published ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Release tag to package (e.g. 1.0.0-beta.12). Leave empty for latest main build."
|
||||
required: false
|
||||
type: string
|
||||
build_run_id:
|
||||
description: "Build workflow run ID (overrides tag lookup)"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name || github.event.inputs.tag || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Resolve which build run to use and extract version info
|
||||
resolve:
|
||||
name: Resolve Build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
version: ${{ steps.resolve.outputs.version }}
|
||||
build_type: ${{ steps.resolve.outputs.build_type }}
|
||||
build_run_id: ${{ steps.resolve.outputs.build_run_id }}
|
||||
tag: ${{ steps.resolve.outputs.tag }}
|
||||
steps:
|
||||
- name: Resolve build run
|
||||
id: resolve
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
INPUT_TAG: ${{ github.event.inputs.tag }}
|
||||
INPUT_RUN_ID: ${{ github.event.inputs.build_run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Determine tag
|
||||
if [[ "${{ github.event_name }}" == "release" ]]; then
|
||||
TAG="${{ github.event.release.tag_name }}"
|
||||
elif [[ -n "$INPUT_TAG" ]]; then
|
||||
TAG="$INPUT_TAG"
|
||||
else
|
||||
TAG=""
|
||||
fi
|
||||
|
||||
echo "Tag: ${TAG:-<none>}"
|
||||
|
||||
# Determine build run ID
|
||||
BUILD_RUN_ID=""
|
||||
|
||||
if [[ -n "$INPUT_RUN_ID" ]]; then
|
||||
# Explicit run ID takes priority
|
||||
BUILD_RUN_ID="$INPUT_RUN_ID"
|
||||
echo "Using explicit build run ID: $BUILD_RUN_ID"
|
||||
|
||||
elif [[ -n "$TAG" ]]; then
|
||||
# Find the build run that produced this tag
|
||||
echo "Looking for build run for tag: $TAG"
|
||||
BUILD_RUN_ID=$(gh api \
|
||||
"repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=${TAG}&status=success&per_page=1" \
|
||||
--jq '.workflow_runs[0].id' 2>/dev/null || echo "")
|
||||
|
||||
if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
|
||||
# Tag might not be a branch; try event=push with head_branch matching
|
||||
BUILD_RUN_ID=$(gh api \
|
||||
"repos/${{ github.repository }}/actions/workflows/build.yml/runs?event=push&status=success&per_page=100" \
|
||||
--jq ".workflow_runs[] | select(.head_branch == \"$TAG\") | .id" 2>/dev/null | head -1 || echo "")
|
||||
fi
|
||||
|
||||
if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
|
||||
echo "❌ No successful build run found for tag: $TAG"
|
||||
exit 1
|
||||
fi
|
||||
echo "Found build run: $BUILD_RUN_ID"
|
||||
|
||||
else
|
||||
# No tag — latest successful main build
|
||||
echo "No tag specified, looking for latest main build"
|
||||
BUILD_RUN_ID=$(gh api \
|
||||
"repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=main&status=success&per_page=1" \
|
||||
--jq '.workflow_runs[0].id' 2>/dev/null || echo "")
|
||||
|
||||
if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
|
||||
echo "❌ No successful main build found"
|
||||
exit 1
|
||||
fi
|
||||
echo "Latest main build: $BUILD_RUN_ID"
|
||||
fi
|
||||
|
||||
# Determine version and build type
|
||||
if [[ -n "$TAG" ]]; then
|
||||
VERSION="$TAG"
|
||||
if [[ "$TAG" == *"-preview"* ]]; then
|
||||
BUILD_TYPE="preview"
|
||||
elif [[ "$TAG" == *"alpha"* || "$TAG" == *"beta"* || "$TAG" == *"rc"* ]]; then
|
||||
BUILD_TYPE="prerelease"
|
||||
else
|
||||
BUILD_TYPE="release"
|
||||
fi
|
||||
else
|
||||
SHORT_SHA=$(gh api "repos/${{ github.repository }}/actions/runs/${BUILD_RUN_ID}" \
|
||||
--jq '.head_sha' 2>/dev/null | head -c 7)
|
||||
VERSION="dev-${SHORT_SHA}"
|
||||
BUILD_TYPE="development"
|
||||
fi
|
||||
|
||||
{
|
||||
echo "version=$VERSION"
|
||||
echo "build_type=$BUILD_TYPE"
|
||||
echo "build_run_id=$BUILD_RUN_ID"
|
||||
echo "tag=${TAG}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "📊 Resolved:"
|
||||
echo " Version: $VERSION"
|
||||
echo " Build type: $BUILD_TYPE"
|
||||
echo " Build run ID: $BUILD_RUN_ID"
|
||||
|
||||
# Build DEB and RPM packages for each architecture
|
||||
package:
|
||||
name: Package (${{ matrix.arch }})
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- arch: x86_64
|
||||
deb_arch: amd64
|
||||
rpm_arch: x86_64
|
||||
artifact_name: "rustfs-linux-x86_64-gnu"
|
||||
- arch: aarch64
|
||||
deb_arch: arm64
|
||||
rpm_arch: aarch64
|
||||
artifact_name: "rustfs-linux-aarch64-gnu"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download binary artifact from build run
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
||||
with:
|
||||
pattern: ${{ matrix.artifact_name }}*
|
||||
path: ./binary-artifact
|
||||
run-id: ${{ needs.resolve.outputs.build_run_id }}
|
||||
github-token: ${{ github.token }}
|
||||
merge-multiple: true
|
||||
|
||||
- name: Extract binary
|
||||
id: binary
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
ZIP_FILE=$(find ./binary-artifact -name "*.zip" -type f | head -1)
|
||||
if [[ -z "$ZIP_FILE" ]]; then
|
||||
echo "❌ No binary artifact found"
|
||||
ls -la ./binary-artifact/ || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Found artifact: $ZIP_FILE"
|
||||
|
||||
mkdir -p ./bin
|
||||
unzip -o "$ZIP_FILE" -d ./bin
|
||||
|
||||
if [[ ! -f ./bin/rustfs ]]; then
|
||||
echo "❌ rustfs binary not found in archive"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
chmod +x ./bin/rustfs
|
||||
ls -lh ./bin/rustfs
|
||||
echo "✅ Binary extracted"
|
||||
|
||||
- name: Build DEB package
|
||||
id: deb
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
VERSION="${{ needs.resolve.outputs.version }}"
|
||||
DEB_ARCH="${{ matrix.deb_arch }}"
|
||||
# DEB version: replace - with ~ (1.0.0-beta.12 -> 1.0.0~beta.12)
|
||||
DEB_VERSION="${VERSION/-/~}"
|
||||
PKG_DIR="rustfs_${DEB_VERSION}_${DEB_ARCH}"
|
||||
|
||||
echo "Building DEB: ${PKG_DIR}.deb"
|
||||
|
||||
mkdir -p "${PKG_DIR}/DEBIAN"
|
||||
mkdir -p "${PKG_DIR}/usr/bin"
|
||||
mkdir -p "${PKG_DIR}/etc/default"
|
||||
mkdir -p "${PKG_DIR}/lib/systemd/system"
|
||||
mkdir -p "${PKG_DIR}/usr/share/doc/rustfs"
|
||||
|
||||
cp ./bin/rustfs "${PKG_DIR}/usr/bin/"
|
||||
chmod 755 "${PKG_DIR}/usr/bin/rustfs"
|
||||
|
||||
cp deploy/build/rustfs.service "${PKG_DIR}/lib/systemd/system/"
|
||||
|
||||
cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
cat > "${PKG_DIR}/DEBIAN/control" << EOF
|
||||
Package: rustfs
|
||||
Version: ${DEB_VERSION}
|
||||
Section: utils
|
||||
Priority: optional
|
||||
Architecture: ${DEB_ARCH}
|
||||
Depends: libc6 (>= 2.31)
|
||||
Maintainer: RustFS Team <support@rustfs.com>
|
||||
Description: High-performance distributed object storage
|
||||
RustFS is a high-performance distributed object storage software
|
||||
built using Rust. It is compatible with MinIO and S3 API.
|
||||
Homepage: https://rustfs.com
|
||||
EOF
|
||||
|
||||
cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs"
|
||||
POSTINST
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postinst"
|
||||
|
||||
cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/prerm"
|
||||
|
||||
cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postrm"
|
||||
|
||||
cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
|
||||
cp README.md "${PKG_DIR}/usr/share/doc/rustfs/"
|
||||
|
||||
fakeroot dpkg-deb --build "${PKG_DIR}"
|
||||
|
||||
DEB_FILE="${PKG_DIR}.deb"
|
||||
ls -lh "$DEB_FILE"
|
||||
echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ DEB built: $DEB_FILE"
|
||||
|
||||
- name: Build RPM package
|
||||
id: rpm
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
VERSION="${{ needs.resolve.outputs.version }}"
|
||||
RPM_ARCH="${{ matrix.rpm_arch }}"
|
||||
|
||||
echo "Building RPM for ${RPM_ARCH}"
|
||||
|
||||
sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential
|
||||
sudo gem install fpm
|
||||
|
||||
fpm -s dir -t rpm \
|
||||
--name rustfs \
|
||||
--version "$VERSION" \
|
||||
--architecture "$RPM_ARCH" \
|
||||
--depends "glibc >= 2.31" \
|
||||
--maintainer "RustFS Team <support@rustfs.com>" \
|
||||
--description "High-performance distributed object storage" \
|
||||
--url "https://rustfs.com" \
|
||||
--license "Apache-2.0" \
|
||||
--after-install <(cat <<'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTINST
|
||||
) \
|
||||
--before-remove <(cat <<'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
) \
|
||||
--after-remove <(cat <<'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
) \
|
||||
--config-files /etc/default/rustfs \
|
||||
./bin/rustfs=/usr/bin/rustfs \
|
||||
deploy/build/rustfs.service=/lib/systemd/system/rustfs.service \
|
||||
LICENSE=/usr/share/doc/rustfs/LICENSE \
|
||||
README.md=/usr/share/doc/rustfs/README.md
|
||||
|
||||
RPM_FILE=$(ls -1 rustfs-*.rpm 2>/dev/null | head -1)
|
||||
if [[ -z "$RPM_FILE" ]]; then
|
||||
echo "❌ RPM build failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ls -lh "$RPM_FILE"
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ RPM built: $RPM_FILE"
|
||||
|
||||
- name: Upload packages to artifacts
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: packages-${{ matrix.arch }}
|
||||
path: |
|
||||
*.deb
|
||||
*.rpm
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload packages to Cloudflare R2
|
||||
if: env.R2_ACCESS_KEY_ID != ''
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "⚠️ R2 credentials missing, skipping upload"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
BUILD_TYPE="${{ needs.resolve.outputs.build_type }}"
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PREFIX="artifacts/rustfs/packages/dev"
|
||||
else
|
||||
R2_PREFIX="artifacts/rustfs/packages/release"
|
||||
fi
|
||||
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
|
||||
|
||||
echo "📤 Uploading to $R2_PATH"
|
||||
|
||||
DEB_FILE="${{ steps.deb.outputs.deb_file }}"
|
||||
RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
|
||||
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading: $f"
|
||||
aws s3 cp "$f" "$R2_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
|
||||
echo "✅ Upload complete"
|
||||
|
||||
# Also upload as latest for release/prerelease
|
||||
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading latest: $(basename "$f")"
|
||||
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
needs: [ resolve, package ]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Print summary
|
||||
shell: bash
|
||||
run: |
|
||||
echo "## 📦 Package Summary" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Item | Value |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "|------|-------|" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Version | \`${{ needs.resolve.outputs.version }}\` |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Package Status | ${{ needs.package.result }} |" >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -322,28 +322,6 @@ High risk: all seven roles.
|
||||
- Use environment variables or vault tooling for sensitive configuration.
|
||||
- For localhost-sensitive tests, verify proxy settings to avoid traffic leakage.
|
||||
|
||||
## Logging
|
||||
|
||||
Applies to **every** `tracing` macro you add or edit, including a single line
|
||||
added in passing while fixing something else — not only to log-focused changes.
|
||||
|
||||
- Fields first, message second: `event`, `component`, `subsystem`,
|
||||
`result`/`state`, then key context. The message is a short label, not a
|
||||
sentence with values interpolated into it.
|
||||
- Reuse the existing `EVENT_*` / `LOG_COMPONENT_*` / `LOG_SUBSYSTEM_*`
|
||||
constants of the module you are editing; match the shape of the log sites
|
||||
already in that file rather than introducing a second style next to them.
|
||||
- Level policy: `error` for behavior/security-affecting failures, `warn` for
|
||||
degraded or fallback paths, `info` for low-frequency lifecycle, `debug` for
|
||||
targeted diagnostics, `trace` for hot paths. Per-object and per-request
|
||||
success paths are `trace`.
|
||||
- Never log secrets, tokens, credential payloads, or merged config dumps.
|
||||
- `scripts/check_logging_guardrails.sh` enforces a subset of this on the files
|
||||
it lists; passing it is a floor, not evidence the log matches the house style.
|
||||
|
||||
See `.agents/skills/rustfs-logging-governance/SKILL.md` for the full event
|
||||
model, level policy, and guardrail-update checklist.
|
||||
|
||||
## Tools
|
||||
|
||||
### xl.meta decode tool Quick Use
|
||||
|
||||
Generated
+239
-274
File diff suppressed because it is too large
Load Diff
+55
-55
@@ -69,7 +69,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-beta.12"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -86,52 +86,52 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.1" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.1" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.1" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.1" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.1" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.1" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.1" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.1" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.1" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.1" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.1" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.1" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.1" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.1" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.1" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.1" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.1" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.1" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.1", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.1" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.1" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.1" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.1" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.1" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.1" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.1" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.1" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.1" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.1" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.1" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.1" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.1" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.1" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.1" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.1" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.1" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.1" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.1" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.1" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.1" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.1" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.1" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-beta.12" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.12" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.12" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.12" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-beta.12" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.12" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.12" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.12" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.12" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.12" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.12" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.12" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.12" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.12" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.12" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.12" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.12" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.12" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.12" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.12" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.12" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.12" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.12" }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.12" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.12" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.12" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.12" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.12" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.12" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.12" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.12" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.12" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.12" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.12" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.12" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.12" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.12" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.12" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.12" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.12" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.12" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.12" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.12" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.12" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.12" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.12" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -228,15 +228,15 @@ atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.10.1" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.141.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.140.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
|
||||
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||
aws-smithy-types = { version = "1.6.1" }
|
||||
base64 = "0.23.1"
|
||||
base64 = "0.23.0"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.4"
|
||||
clap = { version = "4.6.6" }
|
||||
clap = { version = "4.6.5" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.11.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -244,7 +244,7 @@ crossbeam-queue = "0.3.13"
|
||||
crossbeam-channel = "0.5.16"
|
||||
crossbeam-deque = "0.8.7"
|
||||
crossbeam-utils = "0.8.22"
|
||||
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
|
||||
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" }
|
||||
#datafusion = { default-features = false, version = "54.1.0" }
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.14"
|
||||
@@ -341,15 +341,15 @@ unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.5" }
|
||||
russh-sftp = "2.4.0"
|
||||
russh-sftp = "2.3.0"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7", features = ["extended"] }
|
||||
hotpath = { version = "0.23.1", default-features = false }
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13", features = ["extended"] }
|
||||
hotpath = { version = "0.23.0", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
# RustFS 站点复制 / 桶复制 — MinIO 兼容性审查报告
|
||||
|
||||
> 审查日期:2026-08-05
|
||||
> 审查对象:RustFS(worktree `reatang/minio-compatibility-review-03a7fb`)vs MinIO(`/Users/tang/Documents/GitHub/minio`)
|
||||
> 审查方式:白盒代码对比(5 个维度并行审查)+ P0 问题对抗性复核
|
||||
> 审查维度:站点复制白盒对比、桶复制白盒对比、mc 工具兼容性、S3 标准协议兼容性、代码结构与分层
|
||||
|
||||
---
|
||||
|
||||
## 一、总体结论
|
||||
|
||||
| 领域 | 兼容性评价 |
|
||||
|---|---|
|
||||
| **站点复制(RustFS↔RustFS + mc 管理)** | 良好。admin 端点全覆盖、JSON 结构对齐 madmin-go、请求体 DARE 加密兼容,mc admin replicate 全家桶基本可用 |
|
||||
| **站点复制(RustFS↔MinIO 混合组网)** | **断裂**。4 个 P0:出站 join 路径 404、metainfo 大小写解析失败、STS item 类型名不一致、policy-mapping userType 数值错位 |
|
||||
| **桶复制(控制面,S3 标准 API)** | 良好。Put/Get/DeleteBucketReplication、错误码、状态机字符串、xl.meta 内部键均对齐 |
|
||||
| **桶复制(数据面,RustFS→MinIO)** | **断裂**。复制 PUT 缺 `?versionId=` 导致目标端版本漂移(P0);CopyObject 完全不复制(P0) |
|
||||
| **mc 桶复制命令** | **部分断裂**。`mc replicate add` 默认参数即失败(P0);status/resync/backlog 响应结构不匹配导致静默空输出(P1) |
|
||||
| **代码结构** | 桶复制侧迁移架构有纪律但成本高;**站点复制侧无领域层,约 9500 行业务逻辑堆在 admin handler,且存在 3 处反向依赖违反项目分层不变量(P0)** |
|
||||
|
||||
**做得好的地方**(已确认兼容,无需整改):复制状态机字符串(PENDING/COMPLETED/FAILED/REPLICA 含 legacy COMPLETE)、xl.meta 内部键双前缀(x-rustfs-internal- + x-minio-internal-)读写、ReplicateDecision 内部状态串格式、复制内部头主链路双前缀、Delete/VersionPurge 语义、Resync reset-id 判定、admin 路由 `/minio/admin/v3` 前缀别名、madmin DARE 加密流解密、站点复制 gob netperf 编码、`site-repl-<deploymentID>` 规则模板。
|
||||
|
||||
---
|
||||
|
||||
## 二、P0 问题清单(8 项)
|
||||
|
||||
| # | 问题 | 来源维度 | 断裂方向 |
|
||||
|---|---|---|---|
|
||||
| P0-1 | 出站 peer join 使用 MinIO 已移除的遗留路径 `/site-replication/join` → 404 | 站点复制 | RustFS→MinIO |
|
||||
| P0-2 | 解析 MinIO metainfo(SRInfo)字段大小写不匹配 → add preflight 失败 | 站点复制 | RustFS→MinIO |
|
||||
| P0-3 | STS 凭证复制 item 类型名 `sts-credential` vs `sts-account` | 站点复制 | 双向 |
|
||||
| P0-4 | policy-mapping `userType` 数值语义错位(RustFS: None=0/Svc=1/Sts=2/Reg=3;MinIO: reg=0/sts=1/svc=2)→ 权限静默漂移 | 站点复制 | 双向 |
|
||||
| P0-5 | 复制 PUT/CompleteMultipart 不携带 `?versionId=` query → MinIO 端版本号漂移、版本删除永久 no-op、双端静默发散(**功能视角复核:定级调整为 P1**,问题重述为"普通复制对象缺少可靠的源→目标版本身份策略";versionId query 是可行修复之一而非唯一正确方案) | 桶复制 | RustFS→MinIO |
|
||||
| P0-6 | CopyObject(含 metadata-replace 自拷贝)完全不触发复制调度,对象静默不复制(**功能视角复核:定级调整为 P1**;scanner 在 ExistingObjectReplication 启用+状态为空时可最终补齐,但同步复制语义失效,且继承 stale COMPLETED / 显式 Disabled 场景长期漏复制) | 桶复制 + S3 协议 | 所有方向 |
|
||||
| P0-7 | `mc replicate add` 默认参数(healthcheck-seconds=60)被硬拒 400;且字段单位按秒解析而 wire 为纳秒 | mc 兼容 | mc→RustFS |
|
||||
| P0-8 | 架构:站点复制约 9500 行业务逻辑堆在 admin handler 单文件;app/storage 层 3 处反向导入 admin 层,违反 ARCHITECTURE.md 分层不变量 #1(**对抗复核后降级为 P1**:反向边已被 arch 守卫棘轮基线锁死,属受控技术债) | 代码结构 | — |
|
||||
|
||||
每项 P0 的对抗性复核结论、验证方案与解决方案见 **第五节**。
|
||||
|
||||
**修复状态(2026-08-05)**:7 项确认 P0 已全部修复并创建 PR(红灯→绿灯 TDD):P0-1 [#5748](https://github.com/rustfs/rustfs/pull/5748)、P0-2 [#5749](https://github.com/rustfs/rustfs/pull/5749)、P0-3 [#5750](https://github.com/rustfs/rustfs/pull/5750)、P0-4 [#5751](https://github.com/rustfs/rustfs/pull/5751)、P0-5 [#5752](https://github.com/rustfs/rustfs/pull/5752)、P0-6+P1-10 [#5753](https://github.com/rustfs/rustfs/pull/5753)、P0-7 [#5754](https://github.com/rustfs/rustfs/pull/5754)。合并顺序:#5748+#5749 同批;#5752 先于 #5753。
|
||||
|
||||
---
|
||||
|
||||
## 三、P1 问题清单
|
||||
|
||||
### 站点复制
|
||||
|
||||
| # | 问题 | 证据 | 影响 |
|
||||
|---|---|---|---|
|
||||
| P1-1 | ILM(lc-config)复制语义:对外开关限定 `replicateILMExpiry`,但发送端把**完整** lifecycle.xml 放入 `expiry_lc_config`,接收端整体覆盖/删除本地配置(功能视角复核:**确认,维持 P1**;更新时间检查只能拒旧,不能修复整体覆盖语义) | RustFS `bucket_meta.rs:948-951`、`site_replication.rs:7590-7683` vs MinIO `site-replication.go:1784-1810,6138` | lifecycle 同时含 expiry 与本地 transition 时,非 expiry 规则被错误传播或本地 transition 被覆盖。**缺"同步 expiry 后保留本地 transition"测试** |
|
||||
| ~~P1-2~~→**P2-25** | `SRInfo.ilmExpiryRules` 从不填充,ILM 一致性状态恒为空(功能视角复核:**降级 P2**——仅影响管理面可观测性,不改变对象数据) | `site_replication.rs:4152-4266,4855-4868` | `mc admin replicate status --ilm-expiry-rules` 恒空,ILM 漂移不可见 |
|
||||
| P1-3 | 无自动跨站元数据 heal(MinIO 有周期 heal 协程) | RustFS 仅 600s 本地 wiring 修复(`site_replication_reconcile.rs:34,59-81`)+ 手动 repair 端点 vs MinIO `site-replication.go:4257-4288` | 错过的 IAM/bucket 元数据更新持续漂移,须手工 repair |
|
||||
| P1-4(拆分) | ①`sync` 同步复制指控:功能视角复核**不成立/证据不足**——RustFS 自身契约明确将 `sync_state` 定义为站点可达性/配置完整性健康状态且有测试,不能以他家同名字段判其错误(属"RustFS 独特设计保持不变"项,撤销);②`defaultbandwidth`:**确认,降级 P2**——公共 API 接受并持久化,但建 site replication bucket target 时不应用,reconcile 只保留既有 `bandwidth_limit`,配置成功但不生效 | `site_replication.rs:6303-6357,5004-5027` | ②为用户可见的"配置成功但无效"能力缺口 |
|
||||
|
||||
### 桶复制 / S3 协议
|
||||
|
||||
| # | 问题 | 证据 | 影响 |
|
||||
|---|---|---|---|
|
||||
| P1-5 | 未复制完成对象的 GET/HEAD 远端 proxy 未实现;也不识别 MinIO 的 `X-Minio-Source-Proxy-Request` 防环头 | 仅指标占位(`storage_api.rs:799-804`);`SUFFIX_SOURCE_PROXY_REQUEST` 定义后无人使用 vs MinIO `bucket-replication.go:2334,2409,2534` | active-active 复制滞后窗口内 RustFS 端 404 |
|
||||
| P1-6 | `X-Minio-Source-Replication-{Tagging,Retention,LegalHold}-Timestamp` 三个时间戳头收发均缺失 | `replication_target_boundary.rs:251-297` 填了 options 但 `PutObjectOptions::header()` 不序列化;接收端不解析 vs MinIO `object-api-options.go:377-399` | active-active 下标签/retention/legal-hold 并发修改的 LWW 冲突解析退化,可能元数据回滚 |
|
||||
| P1-7 | ARN 前缀 `arn:rustfs:` 与 `arn:minio:` 不互认(解析侧强制 `arn:rustfs:`) | `crates/ecstore/src/bucket/target/arn.rs:43,51` vs MinIO `bucket-targets.go:709` | 存量 MinIO 复制配置迁移被 StaleTarget 拒;原生 madmin SDK 解析 RustFS ARN 失败 |
|
||||
| P1-8 | PutBucketReplication 校验缺口(规则数/Priority 唯一/ID 长度/Filter 互斥/2MB 上限全缺)+ 主动拒绝 `Destination.StorageClass` 等 MinIO/AWS 合法字段 | `bucket_usecase.rs:582-616`、`config.rs:143-232` vs MinIO `internal/bucket/replication/replication.go:29-90` | 非法配置被接受、优先级冲突行为不可预测;存量 AWS/Terraform 配置(含 StorageClass)直接 400 |
|
||||
| ~~P1-9~~→**P2-26** | GetObject 响应缺 `x-amz-replication-status` 头(HEAD 有 GET 无),且 GET 专门把它从 metadata 过滤掉(功能视角复核:**降级 P2**;GET/HEAD 不一致确认,缺 GET replication-status 回归测试) | `object_usecase.rs:5696-5735`、`options.rs:702` vs MinIO `api-headers.go:236-238` | 依赖 GET 判断复制状态的客户端/监控失效;修复约一行 |
|
||||
| P1-10 | Snowball auto-extract 解包对象不触发复制(功能视角复核:**确认,维持 P1**,但"全部永不复制"不准确——scanner 在状态空+ExistingObjectReplication 启用时可补齐;显式 Disabled 等场景长期遗漏,即时复制始终失效。带 REPLICA 状态的入站成员须继续避免回环)。**已随 [#5753](https://github.com/rustfs/rustfs/pull/5753) 修复**(含入站复制 PUT 不再被误派发 extract 的次生缺陷) | `object_usecase.rs:8201` vs MinIO `object-handlers.go:2452,2510-2511` | 批量导入对象不即时复制;缺普通解包成员复制结果的测试(已在 #5753 补充 e2e) |
|
||||
|
||||
### mc 响应结构(静默空输出类)
|
||||
|
||||
| # | 问题 | 证据 | 影响 |
|
||||
|---|---|---|---|
|
||||
| P1-11 | `?replication-metrics[=2]` 响应为 Rust snake_case,minio-go MetricsV2 期望 camelCase(`currStats`/`queueStats`/…) | `stats.rs:617-770`、`admin/router.rs:1583-1592` vs MinIO `bucket-stats.go:154-188` | `mc replicate status` 不报错但全零(静默错误) |
|
||||
| P1-12 | replication-reset(resync)响应壳不匹配:`{"Targets":[{"Arn","ResetID",...}]}` vs `{"target":[{"arn","resetid","resyncStatus",...}]}` | `router.rs:126-198,1735-1803` vs MinIO `bucket-replication-utils.go:613-636` | `mc replicate resync start/status` 输出空;仅响应壳问题,修复成本低 |
|
||||
| P1-13 | `/v3/replication/mrf` 与 `/v3/replication/diff` 返回单个聚合对象而非条目流(代码自述 deliberate) | `replication.rs:695-725,879-911,998-1047` vs madmin-go `replication-api.go:104-176` | `mc replicate backlog` 输出空;`node`/`arn`/`verbose` 参数被忽略 |
|
||||
| P1-14 | set-remote-target 请求体 `deny_unknown_fields` + 字段名偏差(期望 `bandwidth_limit`,madmin 发 `bandwidthlimit`;`session_token` vs `sessionToken` 等) | `handlers/replication.rs:88-95,108-163` vs madmin-go `bucket-targets.go:76` | `mc replicate add/update --bandwidth` 整请求失败;凡 omitempty 字段一旦出现即 400 |
|
||||
|
||||
### 代码结构
|
||||
|
||||
| # | 问题 | 证据 | 影响 |
|
||||
|---|---|---|---|
|
||||
| P1-15 | 站点复制状态两套归一化实现(handler 类型化 vs service 无类型 JSON),且 reload 的 read→normalize→save 全程无共同分布式对象锁,存在 lost-update 竞争;repair state 已用 `with_config_object_write_lock` 包住完整 RMW,主 state 未采用同等保护(功能视角复核:**确认,维持 P1**;进程内 `SITE_REPLICATION_STATE_LOCK` 与单次 read/save 各自的对象锁均不能保护跨调用 RMW:A 读旧→B 另节点写入→A 用旧快照覆盖,B 丢失) | `handlers/site_replication.rs:114,347,1039-1130` vs `service/site_replication.rs:26-135` | 归一化语义可 drift;多节点/RPC 并发写状态互相覆盖。**缺多节点/双写者 lost-update 回归测试** |
|
||||
| P1-16 | 复制状态机类型双份定义:`rustfs-filemeta` 与 `rustfs-replication` 各持一份(ReplicationStatusType/VersionPurgeStatusType/ReplicationState/MrfReplicateEntry/ReplicateObjectInfo),靠 boundary 双向转换 | `crates/filemeta/src/replication.rs` vs `crates/replication/src/filemeta.rs` | 状态机语义修改须同步两处+转换层,漏一处即静默数据语义错误;建议加 enum 对账测试 |
|
||||
| P1-17 | 桶复制逻辑分裂:`crates/replication` 仅契约,执行引擎(pool 5947 行、resyncer 4090 行)仍在 ecstore,中间 20+ 个 boundary/bridge 微文件;迁移无完成判据,脚手架有固化风险 | `crates/ecstore/src/bucket/replication/README.md`、`mod.rs:15-45` | 可读性/可维护性成本;需设定迁移里程碑 |
|
||||
| P1-18 | 超长函数集中在复制热路径:`resync_bucket` 536 行、`replicate_all` 403 行、`start_mrf_processor` 305 行、`apply_iam_item` 248 行 | `replication_resyncer.rs:546`、`replication_pool.rs`、`site_replication.rs:7806` | 正确性审查与修改风险高 |
|
||||
|
||||
### 第三方复审新增与调整项(功能视角二次复核后)
|
||||
|
||||
| # | 问题 | 来源 | 影响 |
|
||||
|---|---|---|---|
|
||||
| P1-19 | 普通复制对象缺少可靠的源→目标版本身份策略:PUT 响应的目标版本 ID 未捕获/持久化,对不支持 versionId query 的目标(原生 AWS S3 等),后续版本删除复制落空;MRF 只会重试同一个错误身份,HEAD ETag fallback 不能修复删除 | P0-5 复审 | 非 MinIO 系目标的版本化复制双端发散。缺"目标自行分配版本 ID"场景测试 |
|
||||
| P1-20 | 缺少 scanner 补偿边界的 e2e:ExistingObjectReplication Enabled/Disabled × 空状态/继承状态 组合下的补齐与不补齐行为无回归覆盖(Copy 与 Snowball 两路径) | P0-6/P1-10 复审 | scanner 兜底语义变化不可见 |
|
||||
| P1-21 | delete-marker 延迟 purge 失败静默丢弃(由 P2-20① 升级):目标删除失败无日志/状态/MRF,目标端 marker/版本可能永久残留 | P2-20 复核升级 | 数据一致性;缺失败注入测试 |
|
||||
| P1-22 | 桶复制整体 SSE 支持能力缺口(替代原 P2-23):SSE-S3/SSE-KMS 所有复制模式统一 fail closed,SSE-C 失败被 e2e 钉为当前行为,无 encrypted-object resync e2e | P2-23 复核改写 | 加密对象跨站不复制;需覆盖普通复制/Heal/Resync/Multipart 四模式 |
|
||||
|
||||
### 功能视角二次复核采纳记录(backlog#1675,基于 main f0c4fbd28)
|
||||
|
||||
复核共 10 项,判定依据为 RustFS 自身功能契约与实际调用链,不以对齐 MinIO 为正确性标准。采纳结果:
|
||||
|
||||
| 原编号 | 复核结论 | 采纳动作 |
|
||||
|---|---|---|
|
||||
| P0-5 | 确认,P0→P1,问题重述为"源→目标版本身份策略缺失" | 定级调整;修复已合 [#5752](https://github.com/rustfs/rustfs/pull/5752);残留缺口 P1-19 |
|
||||
| P0-6 | 确认,P0→P1,scanner 描述纠正 | 定级调整;修复已合 [#5753](https://github.com/rustfs/rustfs/pull/5753);测试缺口 P1-20 |
|
||||
| P1-1 | 确认,维持 P1 | 补记"expiry 同步后保留本地 transition"测试缺口 |
|
||||
| P1-2 | 确认,P1→P2(仅管理面可观测性) | 改编号 P2-25 |
|
||||
| P1-4 | 拆分:`sync` 指控不成立(RustFS 自身契约定义为健康状态,有测试);`defaultbandwidth` 确认为 P2 能力缺口 | `sync` 撤销并归入"独特设计保持不变";`defaultbandwidth` 降 P2 |
|
||||
| P1-9 | 确认,P1→P2 | 改编号 P2-26;补记缺 GET 回归测试 |
|
||||
| P1-10 | 确认,维持 P1,"全部永不复制"改为"即时复制失效+部分场景长期遗漏" | 已随 [#5753](https://github.com/rustfs/rustfs/pull/5753) 修复(含回环防护) |
|
||||
| P1-15 | 确认,维持 P1(竞争机理精确化:跨调用 RMW 无共同分布式锁) | 补记缺双写者 lost-update 测试 |
|
||||
| P2-20① | 确认,P2→P1(延迟 purge 失败静默丢弃部分) | 升级为 P1-21;②③维持 P2 |
|
||||
| P2-23 | resync 专属指控不成立;暴露桶复制整体 SSE 能力缺口 | 撤销原表述,改立 P1-22 |
|
||||
|
||||
**复核指出的测试补齐清单**(均未运行跨实例集成验证,需落地):目标自行分配版本 ID、Copy/Snowball scanner 补偿边界、lifecycle expiry/transition 保留、site state 双写竞争、delayed purge 失败注入、encrypted-object resync。
|
||||
|
||||
---
|
||||
|
||||
## 四、P2 问题清单
|
||||
|
||||
### 站点复制
|
||||
- **P2-1** `showDeleted` 选项与 `bucketDeletedTimestamp` 未实现(`site_replication.rs:1364-1381`)
|
||||
- **P2-2** 错误码泛化:统一 `InvalidRequest`/`InternalError`,无 MinIO 的 9 个 `XMinioSiteReplication*` 专用码(400/503 语义丢失)
|
||||
- **P2-3** `make-with-versioning` 忽略 `versioningEnabled`/`forceCreate` 参数,恒 true(`site_replication.rs:8597-8627`)
|
||||
- **P2-4** netperf 返回"不支持"占位(gob 格式兼容不会崩);devnull 有请求体大小上限(MinIO 无限 discard)
|
||||
- **P2-5** Metrics 摘要仅含本站,无 per-peer 链路统计(downtime/latency/失败窗口)
|
||||
- **P2-6** `external-user`/`credential` IAM item 未实现——与本仓 MinIO 版本等价缺失,结构已预留;对接新版 MinIO 时会成缺口
|
||||
- **P2-7** 本地 deploymentID 缺失时回退 endpoint 哈希(16 位 hex,非 UUID 形态)
|
||||
|
||||
### 桶复制 / S3 协议
|
||||
- **P2-8** 遗留内部 client 头名错误:`X-Source-DeleteMarker`/`X-Check-Replication-Ready` 缺 `X-Minio-` 前缀(`client/api_stat.rs:191-231`,当前路径未激活,潜伏缺陷)
|
||||
- **P2-9** Remote target admin 错误码扁平化(MinIO 有 404/503 专用码,RustFS 统一 400/500)
|
||||
- **P2-10** Remote target 拒绝 `disableProxy`/`edge`/`edgeSyncBeforeExpiry` 等 madmin 字段(非默认参数,影响小)
|
||||
- **P2-11** `list-remote-targets` 序列化偏差:`bandwidth_limit`/`storage_class`/`deployment_id`/`reset_id`/`session_token` vs madmin 的 `bandwidthlimit`/`storageclass`/`deploymentID`/`resetID`/`sessionToken`;`healthCheckDuration`/`totalDowntime` 按秒序列化而 Go 按纳秒解;`type` 过滤参数被忽略
|
||||
- **P2-12** set-remote-target?update=true 忽略 madmin 的 op 标志(creds/sync/proxy/…),固定整体覆盖
|
||||
- **P2-13** XML 反序列化:Rule 内未知元素严格报 MalformedXML(顶层却跳过,行为不一致);缺 `<Role>` 报 MalformedXML(Go 容忍)——向前兼容性差,当前主流客户端不受影响
|
||||
- **P2-14** `ReplicaModifications` 默认 Disabled(与 AWS 一致、与 MinIO 的注入 Enabled 分歧);PUT 时不像 MinIO 那样注入默认元素回写
|
||||
- **P2-15** PutBucketReplication 要求预先注册 remote target(与 MinIO 同构、与纯 AWS 流程分歧),报错未指引先建 target
|
||||
- **P2-16** GetBucketReplication 响应无 xmlns(与 MinIO 一致,极少数严格 SDK 可能拒收)
|
||||
- **P2-17** 站点复制启用时不阻止普通用户直接改桶复制配置(MinIO 非 root 报 `ErrReplicationDenyEditError`)
|
||||
- **P2-18** Prometheus 指标名对齐 metrics-v3 但注册前缀为 rustfs 体系;versioning 错误文案与 MinIO 不同(code 一致)
|
||||
|
||||
### 代码结构
|
||||
- **P2-19** `apply_iam_item` / bucket-ops 用裸字符串 match 分发,无法穷尽检查;建议改 `#[serde(tag)]` 枚举
|
||||
- **P2-20(拆分)** 静默吞错:①`replication_resyncer.rs:1693` delete-marker 延迟 purge 失败被 `let _ =` 丢弃,target client 缺失时直接跳过——**功能视角复核:升级为 P1-21**(失败后无日志、无状态更新、不入 MRF,目标 delete marker/版本可能永久残留;启动前的 5 次循环只是等源 marker 消失,不是对目标删除失败的重试。缺注入目标删除失败并验证重试/状态/MRF 的测试);②`site_replication.rs:8661` purge-deleted-bucket 吞掉非 NotFound 错误、`:9227` cancel resync 失败无痕迹——维持 P2
|
||||
- **P2-21** `MrfV2` 全套机制(Error/Capabilities/Readiness/Reader/Envelope)未接线,生产只用 v1,属投机代码
|
||||
- **P2-22** `persist_site_replication_state` 双重 clone + 双重 normalize(`site_replication.rs:1143-1152` → `:1116-1122`)
|
||||
- **P2-23(撤销并改写)** 原"resync 不处理 SSE"指控不成立——`ReplicationType::Resync` 与普通复制/Heal 最终走同一 `replication_put_object_options`,`// TODO: SSE` 不构成 resync 独立行为差异。真实状态:SSE-S3/SSE-KMS 在**所有复制模式**下统一 fail closed,SSE-C 普通桶复制失败已被现有 e2e 钉为当前行为,且无 encrypted-object resync e2e → 改立能力项 **P1-22"桶复制整体 SSE 支持"**(需分别覆盖普通复制、Heal、手动 Resync、Multipart)
|
||||
- **P2-24** `crates/replication` 命名误导(名为复制引擎实为契约库),建议 lib.rs 顶部文档说明
|
||||
- 正面确认:生产代码 unwrap/expect 纪律良好(几乎全在测试模块);MinIO 概念映射(ReplicationPool/Resyncer/MRF/TargetClient)桶复制侧清晰,站点复制侧缺 `SiteReplicationSys` 聚合体
|
||||
|
||||
---
|
||||
|
||||
## 五、P0 问题对抗性分析(复核结论 + 验证方案 + 解决方案)
|
||||
|
||||
### P0-1 出站 peer join 路径 — **CONFIRMED(比原指控更严重)**
|
||||
|
||||
**复核结论**:指控全部成立,且加重三点:
|
||||
1. `/minio/admin/v3/site-replication/join` 在 MinIO 历史上**从未存在过**(`git log -S` 追到功能诞生的 2021 年首个提交,注册的就是 `peer/join`)。RustFS 实现者疑似被 MinIO `admin-handlers-site-replication.go:76` 一条过时的文档注释误导。
|
||||
2. 无任何 404 回退、版本探测或 feature flag;唯一的重试逻辑只针对 secret 不匹配(`site_replication.rs:3036-3082`),404 直接失败。
|
||||
3. 现有单测 `:13683-13696` 正在**固化错误行为**(测试名声称匹配 MinIO 路由,断言的却是不存在的路由)。RustFS↔RustFS 之所以不暴雷,是因为 RustFS 入站自己注册了该错误路径的兼容别名,掩盖了 bug。
|
||||
|
||||
**影响面**:RustFS 发起的 add(含 MinIO 站点)、服务账号轮换通知 MinIO peer 均断;MinIO→RustFS 与 RustFS↔RustFS 不受影响;其余 peer/* 端点走通用前缀改写,路径正确。
|
||||
|
||||
**修路径还不够,还有三处 join 协议分歧须同批修**:①加密判定 `site_replication_peer_payload_encrypted`(:2899-2901)只对旧路径加密,MinIO `SRPeerJoin` 强制解密,须跟随路径改;②MinIO join 成功返回**空 body**,RustFS `:8163` 强制解析 `SRPeerJoinResponse` 会失败,须容忍空 body(peer 身份回退用 preflight 已取得的数据合成);③`deferSyncStateEnable`/`bootstrapToken` 对 MinIO 无效但不阻断(行为差异,建议日志标注)。
|
||||
|
||||
**验证方案**:
|
||||
- 单测:翻转 `:13683`/`:13699` 两个测试断言为 `peer/join`(把固化 bug 的测试变成回归防护)。
|
||||
- 集成测:测试内起 axum stub 精确复刻 `admin-router.go` 路由(仅注册 `PUT .../peer/join`,其余 404),handler 内用 `decrypt_stream_io` 验证 body 是 madmin 兼容密文,返回 200 空 body;断言修复前 404、修复后全链路成功。
|
||||
- e2e:docker compose(rustfs+minio),RustFS 侧 `mc admin replicate add`,MinIO 侧 `mc admin trace -a` 断言 `PUT .../peer/join` 200。注意:**e2e 会先被 P0-2 的 preflight 挡住,两问题必须同批修复才能全链路验证**。
|
||||
|
||||
**解决方案**(均在 `handlers/site_replication.rs`):删除 :2885-2886 的 join 特判使其落入通用前缀改写;:2899-2901 加密判定改为对 `peer/join` 返回 true;:8163 响应解析容忍空 body;更新两个单测。
|
||||
**滚动升级风险**:必须保留入站的 `/v3/site-replication/join` 旧路径路由(旧版 RustFS 出站仍发它);发版前对最近 release tag 复核旧版入站已注册 `peer/join`。
|
||||
|
||||
### P0-2 SRInfo 大小写不匹配 — **CONFIRMED(范围精确化)**
|
||||
|
||||
**复核结论**:成立。madmin-go v3.0.109(minio go.mod 锁定版)`SRInfo` 除 `APIVersion` 外 12 个顶层字段**全部无 json tag**,Go 按 PascalCase 序列化;RustFS `SRInfo` serde 大小写敏感、全字段 `#[serde(default)]` → 解析 MinIO 输出**不报错而是静默全空**。精确化:**不兼容仅限 SRInfo 顶层 12 个字段**,嵌套结构(SRBucketInfo/SRStateInfo/SRIAMPolicy 等)madmin 本就带小写 tag,不受影响。`:5581` 的 `"buckets"|"Buckets"` 手写双读证明作者已知 MinIO 输出 PascalCase,只是未系统化修复。
|
||||
|
||||
**影响面**:RustFS 发起 add 时 preflight 硬失败("site did not report deploymentID")——**触发顺序先于 P0-1 的 join**;`mc admin replicate status` 对 MinIO peer 静默显示全空/全 mismatch(HTTP 200,无报错)。MinIO 读 RustFS 方向因 Go unmarshal 大小写不敏感而无恙。
|
||||
|
||||
**验证方案**:
|
||||
- 单测(crates/madmin):用 Go `json.Marshal(madmin.SRInfo{...})` 真实生成的 PascalCase JSON 作 fixture,断言反序列化后字段非空;再加序列化回归断言输出仍为 camelCase(保证 RustFS↔RustFS 不回归)。
|
||||
- 集成测:stub 在 metainfo 端点返回 PascalCase body,走 `remote_add_preflight_info`,断言不再报错。
|
||||
- e2e:与 P0-1 同批,`mc admin replicate status --json` 断言 MinIO 站点条目完整。
|
||||
|
||||
**解决方案**:`crates/madmin/src/site_replication.rs:642-670` 为 12 个顶层字段逐一加 `#[serde(alias = "...")]`(精确取 Go 字段名,注意是 `ILMExpiryRules` 不是 `IlmExpiryRules`)。alias 只影响反序列化,出站格式零变化,风险几乎为零。**只加顶层、不扩散到嵌套结构**,并留注释说明原因。回归防护关键是把 Go 真实输出固化为测试 fixture。
|
||||
|
||||
### P0-7 `mc replicate add` 默认参数被拒 + 单位错误 — **CONFIRMED**
|
||||
|
||||
**复核结论**:全部反驳方向反向坐实(本地有 mc 源码,非推断):
|
||||
- mc `replicate-add.go:93-95` 默认 `healthcheck-seconds=60`,`:301-303` 无条件调用 `SetRemoteTarget`,失败即终止,无跳过路径;
|
||||
- madmin `bucket-targets.go:79` `HealthCheckDuration time.Duration` 无自定义 Marshal → wire 上是纳秒整数 `60000000000`;
|
||||
- RustFS `handlers/replication.rs:213-225` 对非零值必拒 400;`mc replicate update` 同样失败;无老端点绕过。
|
||||
- **单位错误独立成立且双向**:请求侧按 `Duration::from_secs` 解析(60e9 ns 会被当 60e9 秒 ≈ 1900 年);响应/持久化侧 `bucket_target.rs:195-197` 按秒序列化,mc 按纳秒解(60s 显示为 60ns),同时构成与 MinIO `bucket-targets.json` 的持久化格式偏差。
|
||||
- **为何没被发现**:这是刻意的"能力契约式拒绝"策略,且有单测 `replication.rs:1353-1379` 固化拒绝行为;e2e 全部自行构造 JSON、不含该字段,测的是"RustFS 自己的请求形态"而非"mc 默认请求形态"。缓解:`--healthcheck-seconds 0` 时字段 omitempty 被省略可通过,但默认路径必失败,P0 成立。
|
||||
|
||||
**验证方案**:复现——`mc replicate add rustfs/src --remote-bucket http://ak:sk@target/dst` 预期 400;修复后——madmin 形态 payload(60e9 ns)单测断言内部 Duration==60s;set→list 往返断言响应为纳秒;e2e 增加"mc 默认 payload"用例;持久化防御性读回归(旧秒格式升级后读取不变)。
|
||||
|
||||
**解决方案(分阶段)**:
|
||||
1. **解阻塞**:从不支持清单移除 `healthCheckDuration`(能力契约版本号递增);请求按 `Duration::from_nanos` 解析(`total_downtime` 同步核查);调度上显式忽略并在契约/文档标注"接受但暂不生效";响应侧新增 DTO 按纳秒序列化(**勿直接改 `bucket_target.rs` 的 `duration_seconds`,它同时是持久化格式**);持久化读取加防御(≥10^7 视为纳秒),写入统一新格式。
|
||||
2. **落地语义**:`bucket_target_sys.rs:332-441` heartbeat 循环改为按 target 取值,对齐 MinIO(默认 5s、有下限)。
|
||||
3. **防复发**:建立容器内跑真 mc 命令的兼容 e2e 通道,覆盖 `replicate add/update/status`。
|
||||
|
||||
### P0-8 站点复制架构 — **事实 CONFIRMED,定性部分 REFUTED,降级为 P1**
|
||||
|
||||
**复核结论**:巨型文件(14614 行,非测试约 9533 行,24 个 handler)与三处反向导入全部属实;但"失察"定性被推翻:
|
||||
- `scripts/check_layer_dependencies.sh` **已建模并拦截**这些边,`layer-dependency-baseline.txt` 棘轮基线逐条列出全部 46 条存量反向边,**新增反向边 CI 必炸**;
|
||||
- `ecfs.rs` 被脚本刻意归类为 interface 层(有意的建模决策);
|
||||
- ARCHITECTURE.md 自己声明部分不变量 "currently violated... documenting them makes violations explicit and trackable";git 历史显示这是已知、受控、正在偿还的过渡态。
|
||||
- **结论:不构成正确性风险,从 P0 降为 P1(可维护性债务)**。真实成本:9.5k 行单文件的评审/合并冲突/增量编译负担,hook 直连使 app/storage 单测无法脱离 admin 层。
|
||||
|
||||
**验证方案**:每阶段跑 `make pre-pr`;每消除一条反向边即**删除基线对应行**(而非重生成),使回归必炸;行为回归靠 site replication e2e + 路由快照测试 + `git diff --color-moved` 评审纯移动。
|
||||
|
||||
**解决方案(分阶段)**:
|
||||
1. **解反向依赖(低风险,先做)**:复用 `site_replication_reconcile.rs` 已验证的 OnceLock 注册模式——bucket 三个 hook 在 app 层定义 fn-pointer 契约、admin 构建路由时注册;`node_service.rs` 的 reload 走 infra 层"运行时重载注册表"。注册缺失时显式降级(warn + no-op)。
|
||||
2. **文件拆分(纯移动)**:`site_replication.rs` → 模块目录:`transport`(peer client/DNS/TLS)、`gob`、`state`(注意 config key 路径不可变)、`iam_sync`、`heal`、`handlers`(24 个薄 handler)。
|
||||
3. **领域下沉(风险最高,最后做)**:hook 解耦后把 gob/transport/状态机移入独立 crate,注意全局状态清单(`docs/architecture/global-state-inventory.md:114`)。
|
||||
|
||||
### P0-3 STS item 类型名不一致 — **CONFIRMED(双向硬断)**
|
||||
|
||||
**复核结论**:成立,且两端都是**报错而非静默忽略**:MinIO 收到 `"sts-credential"` 走 default 分支返回 400 `errSRInvalidRequest`;RustFS 收到 `"sts-account"` 返回 NotImplemented。两端 heal/重试机制都会永久重试失败(MinIO 日志持续 "Unable to heal temporary credentials")。MinIO 当前版本 STS 复制发送面很广(AssumeRole/WebIdentity/ClientGrants/LDAPIdentity/Certificate 全系 + sftp/ftp + heal 路径)。除类型串外 `SRSTSCredential` 字段双方完全对齐——**只差这一个字符串**(推测 RustFS 实现时把 madmin 的 JSON 字段名 `stsCredential` 误当成了类型常量)。
|
||||
|
||||
**影响面**:跨厂商 STS 临时凭证双向不复制(客户端在对端站点 `InvalidAccessKeyId`),纯可用性问题,无权限漂移;RustFS↔RustFS 自洽。
|
||||
|
||||
**验证方案**:单测——出站产物断言 `type == "sts-account"`(改 `federated_identity.rs:497` 现有快照测试);入站构造 `"sts-account"` item 断言不落 NotImplemented。e2e——compose(RustFS+MinIO,root 凭证必须一致,否则 token 验签失败会误判修复无效):对 MinIO assume-role 拿临时凭证访问 RustFS,修复前 InvalidAccessKeyId、修复后成功;反向同测。
|
||||
|
||||
**解决方案**:出站(`sts.rs:248`、`federated_identity.rs:241`)改发 `"sts-account"`(提常量集中定义);入站(`site_replication.rs:7857`)match 臂改 `"sts-account" | "sts-credential"`(**永久保留旧别名**兼容旧 RustFS peer)。滚动升级窗口内新→旧 RustFS 会降级(warn+重试,peer 升级后收敛);STS 凭证短生命周期,不建议为此拆两阶段发布。
|
||||
|
||||
### P0-4 policy-mapping userType 数值错位 — **CONFIRMED(比指控更严重)**
|
||||
|
||||
**复核结论**:数值表属实(RustFS: None=0/Svc=1/Sts=2/Reg=3;MinIO: unknown=-1/reg=0/sts=1/svc=2),wire 上确为数值、无翻译层。对抗复核修正与加重:
|
||||
- **RustFS→MinIO 方向今天"侥幸能用"**:RustFS 当前只出站 Reg=3 与组的 0,MinIO 对超范围值静默落 default 分支,恰好落对位置;
|
||||
- **MinIO→RustFS 方向三类断裂**:①**组映射硬失败(新发现)**——MinIO 组映射发 `UserType: -1`,RustFS `user_type: u64` 反序列化直接报错,整个 item 被拒,组→策略映射完全无法同步;②STS 用户映射(MinIO 发 1)被 RustFS 解释为 Svc,落错前缀/缓存,联邦用户在 RustFS 站点**静默丢权限**;③svc=2 被解释为 Sts,同类错位;
|
||||
- **低概率提权路径**:LDAP DN/OIDC 主体的映射被误存入常规用户缓存后,若本地恰有同名静态用户则继承本不属于它的策略——名字碰撞概率低但非零,这是保 P0 的理由。
|
||||
|
||||
**验证方案**:单测——wire 编解码全矩阵(-1/0/1/2/3/非法值);e2e——MinIO 侧 `mc admin policy attach --group` 修复前 RustFS 查不到组实体、修复后可见;`mc idp ldap policy attach` 修复前落 `policydb/service-accounts/` 且访问被拒、修复后落 `sts-users/` 且放行;反向回归守住"侥幸兼容";混版本(旧+新 RustFS)双向 attach 互通。
|
||||
|
||||
**解决方案(核心原则:不改 `UserType::to_u64/from_u64`)**——该编码被集群内部节点 RPC 使用(`node_service.rs:1513`),改动会破坏同集群滚动重启。只在站点复制 wire 边界加 MinIO 语义编解码:
|
||||
1. `SRPolicyMapping.user_type` 由 `u64` 改 `i64`(必须,才能收下 -1);
|
||||
2. 出站 `sr_wire_user_type`:Reg→0/Sts→1/Svc→2,组一律发 0(对 MinIO 与旧 RustFS 同时兼容);入站 `user_type_from_sr_wire`:-1→None/0→Reg/1→Sts/2→Svc/**3→Reg(旧 RustFS 别名,永久保留)**;
|
||||
3. 兼容矩阵已逐格验证:新↔旧 RustFS、MinIO↔新 RustFS 全通;唯一残余窗口(未来出站 Sts/Svc 映射对旧 RustFS 错读)当前不可达,在 doc comment 写明约束;
|
||||
4. 回归防护:编解码矩阵单测 + "wire 常量契约"字面值断言测试(防止将来被"顺手统一"回内部编码)+ e2e 进 P0 套件;顺带把 `SRCredInfo.iam_user_type` 一并改 `i64` 复用同一编解码,消除同族隐患。
|
||||
|
||||
### P0-5 复制 PUT 缺 `?versionId=` query — **CONFIRMED**
|
||||
|
||||
**复核结论**:所有反驳方向均失败,指控成立:
|
||||
- minio-go 官方复制端(v7.0.91)`api-put-object-streaming.go:767-776` 等三处全部是 `urlValues.Set("versionId", ...)`——**query,不是 header**;`x-minio-source-version-id` 这个 header 在 MinIO 全仓不存在,被静默忽略;
|
||||
- multipart 的版本在 **initiate 时**决定(`erasure-multipart.go:458-460`,为空即生成新 UUID),complete 不读 versionId;
|
||||
- aws-sdk-s3 `PutObjectInput` 无 versionId 成员属实,但 DELETE 路径已用 `.set_version_id()` 正确落 query,证明是遗漏而非不可行;
|
||||
- RustFS↔RustFS 不受影响的原因:RustFS 接收端有私有 header fallback(`options.rs:296-301`),恰好掩盖了 bug。
|
||||
|
||||
**影响加重**:除版本漂移与按版本删除永久 no-op 外,目标校验/heal 用源 versionId `head_object` 永远 miss → **反复重传,目标端版本无限膨胀**。另有边缘缺陷:RustFS 内部 null 版本是 nil-UUID,直接发 query 会被 MinIO 当真实版本;minio-go 约定发字面 `"null"`。
|
||||
|
||||
**验证方案**:L1 e2e(本仓可落地,红→绿)——复用 `crates/e2e_test/src/fake_s3_target/`(已解析 versionId query 并写 journal),断言 PutObject/CreateMultipartUpload 请求的 query == 源版本;L2 互操作(docker + 真 MinIO)`mc ls --versions` 断言目标 versionId == 源、删源版本目标同步消失;L3 单测 nil-UUID→`"null"` 映射。
|
||||
|
||||
**解决方案**(`bucket_target_sys.rs`):`put_object`/`create_multipart_upload` 在 `map_request` 闭包内改写 URI 追加 `versionId` query(nil-UUID 映射 `"null"`);保留双 header 兼容旧版 RustFS 接收端;顺带核对 delete 路径的 nil-UUID 映射。**签名安全性已验证**:`map_request` 挂在 `modify_before_signing`,query 会进 canonical request,不会 SignatureDoesNotMatch。非版本化目标桶沿用"空则不发",`"null"` 值 MinIO 免检。
|
||||
|
||||
### P0-6 CopyObject 不触发复制 — **CONFIRMED(附带加重发现)**
|
||||
|
||||
**复核结论**:三个反驳方向全部不成立:
|
||||
- copy 直接调 `store.copy_object`,不经 put 路径;ecstore 层 copy 实现无任何调度;
|
||||
- **scanner 兜底不存在(关键)**:heal 入队条件是状态为 Pending/Failed 或手动 resync;而 copy 路径不 stamp PENDING(对照 put 路径 `object_usecase.rs:5255-5266`),状态为空 → heal 判定 Skip。
|
||||
- **加重发现**:copy 路径没有 MinIO `filterReplicationStatusMetadata` 的等价清理——COPY 指令下源对象的旧复制状态可能原样带到目的对象,**伪造 COMPLETED 假状态**。
|
||||
- 附带 P1(snowball `execute_put_object_extract`)同样确认:无 stamp 无 schedule。
|
||||
|
||||
**影响面**:配复制规则的桶上,CopyObject 写入的对象(跨桶复制、rename 工作流、REPLACE 元数据更新)永不复制、scanner 不捞、仅手动 resync 可补;还可能带 stale 假状态。
|
||||
|
||||
**验证方案**:e2e(参照 `replication_extension_test.rs` 双实例)——copy 后断言目的对象在目标桶超时内出现、源 COMPLETED、目标 REPLICA、无 stale 状态;snowball 参照 `snowball_auto_extract_test.rs` 加成员对象复制断言;usecase 单测用 `storage_api.rs:641` 现有 test-only 调用计数断言 copy/extract 触发决策与调度。
|
||||
|
||||
**解决方案**(`object_usecase.rs`):
|
||||
1. `execute_copy_object` 在 `store.copy_object` 之前算一次 `dsc = must_replicate_object(...)`,`replicate_any` 时向 `dst_opts.user_defined` stamp pending + timestamp(严格镜像 put 路径,单一 dsc 决策贯穿两阶段);
|
||||
2. 同处清理源带来的复制状态 reserved 元数据;
|
||||
3. copy 成功、锁释放后 `schedule_object_replication`;
|
||||
4. `execute_put_object_extract` 对每个解出对象同样处理。
|
||||
风险已排除:replica 判定内置于 `must_replicate_object` 不会回环;self-copy 调度与 MinIO 一致。
|
||||
**落地顺序约束:先修 P0-5 再修 P0-6**——否则 copy 的失败重试经 heal 兜底后,只会在 MinIO 端制造更多漂移版本。
|
||||
|
||||
### 第三方复审修正(2026-08-05,修复分支均已完成 review)
|
||||
|
||||
**P0-5 修正**:问题的准确表述应为"**普通复制对象缺少可靠的源→目标版本身份策略**"——复制 PUT 只返回成功/失败,未捕获目标实际分配的版本 ID(已核实 `bucket_target_sys.rs` put 路径无 `res.version_id()` 捕获,delete 路径 :2030 有);multipart 只保留 upload ID。`fix/p0-5` 的 versionId query 方案对 MinIO/RustFS 目标成立(目标端沿用源版本 ID,身份问题消解),但对**忽略该私有 query 的目标(如原生 AWS S3)**身份问题仍在:目标自行生成版本 ID → 后续按源版本 ID 的删除复制落空。第三方建议定级 P1(修复已完成,残留缺口另行跟进):可选方案包括捕获 PUT 响应的 `x-amz-version-id` 并持久化源→目标映射。→ 记为 **P1-19(新增)**。
|
||||
|
||||
**P0-6 修正**:scanner"兜底不存在"的表述过度。已核实 `crates/replication/src/operation.rs` `resync_target_for_object`:无 reset 记录且复制状态为 Empty 时返回 `replicate=true`,即 ExistingObjectReplication 启用时 scanner **可能最终补齐**空状态对象,无需手动 resync。准确结论:即时/同步复制语义失效(P0 定级依据),且以下场景**长期**漏复制——①源对象 COMPLETED 等复制元数据被 Copy 继承致误判(`fix/p0-6` 已修,清理先于决策);②显式 ExistingObjectReplication=Disabled;③其他无法进入 existing-object 补偿的场景。`fix/p0-6` 分支已含 copy 调度 e2e 与 stale 元数据白盒断言;**scanner 补偿边界的 e2e 仍缺** → 记为 **P1-20(新增)**。
|
||||
|
||||
### 对抗性复核总览
|
||||
|
||||
| 问题 | 复核结论 | 关键修正/加重 |
|
||||
|---|---|---|
|
||||
| P0-1 join 路径 | CONFIRMED,加重 | 路径在 MinIO 从未存在;现有单测固化错误;修复需同批改加密判定与空响应容忍 |
|
||||
| P0-2 SRInfo 大小写 | CONFIRMED,精确化 | 仅顶层 12 个无 tag 字段;preflight 失败先于 P0-1 触发 |
|
||||
| P0-3 STS 类型名 | CONFIRMED | 双向硬断、两端 heal 永久重试;只差一个字符串 |
|
||||
| P0-4 userType 错位 | CONFIRMED,加重 | MinIO 组映射发 -1 → RustFS u64 解析硬失败;存在低概率名字碰撞提权路径;修复不得触碰内部 RPC 编码 |
|
||||
| P0-5 versionId query | CONFIRMED,加重 | heal 反复重传致目标版本膨胀;nil-UUID 需映射 "null" |
|
||||
| P0-6 CopyObject | CONFIRMED,加重 | scanner 兜底不存在;stale COMPLETED 假状态;须在 P0-5 之后落地 |
|
||||
| P0-7 healthCheckDuration | CONFIRMED | 单位错误双向独立成立;有单测固化拒绝行为 |
|
||||
| P0-8 架构 | 事实 CONFIRMED,定性 REFUTED | 反向边被棘轮基线锁死,降级 P1(受控技术债) |
|
||||
|
||||
---
|
||||
|
||||
## 六、修复路线图(2026-08-05 更新)
|
||||
|
||||
**✅ 第一批已完成**:全部 7 项 P0 已修复并创建 PR(见第二节修复状态;P1-10 snowball 随 #5753 一并修复)。待合并,注意顺序约束:#5748+#5749 同批、#5752 先于 #5753。
|
||||
|
||||
**第二批(数据一致性优先,采纳功能视角复核定级)**
|
||||
1. **P1-21** delete-marker 延迟 purge 失败静默丢弃(复核升级,数据一致性,建议单独小 PR + 失败注入测试)
|
||||
2. **P1-19** 源→目标版本身份策略(捕获 PUT 响应 `x-amz-version-id` / 持久化映射,覆盖非 MinIO 系目标)
|
||||
3. **P1-1** ILM expiry 同步语义(只传播 expiry、保留接收端本地 transition + 对应测试)
|
||||
4. **P1-15** site state RMW 分布式锁统一(对齐 repair state 的 `with_config_object_write_lock` 模式)+ 双写者回归测试
|
||||
5. **P1-22** 桶复制 SSE 能力(普通复制/Heal/Resync/Multipart 四模式,先补 encrypted-object e2e 钉现状)
|
||||
|
||||
**第三批(mc 可观测性与互操作补齐)**
|
||||
6. P1-11/12/14 mc 响应结构 serde rename(改动小、消除静默空输出)
|
||||
7. P1-7 ARN 解析侧兼容 `arn:minio:` 前缀
|
||||
8. P1-5 GET/HEAD proxy、P1-6 时间戳头、P1-3 自动跨站 heal
|
||||
9. P1-20 scanner 补偿边界 e2e;P0-7 阶段 2(per-target 心跳 + healthcheck update op)
|
||||
10. P2-26 GET 补 `x-amz-replication-status`(约一行)+ 回归测试;P2 清单其余项
|
||||
|
||||
**第四批(架构与长期)**
|
||||
11. P0-8(降级 P1)架构:先解 3 处反向依赖(复用 reconcile 注册模式),再拆分/下沉站点复制领域模块
|
||||
12. P1-16 类型对账测试、P1-17 迁移完成判据、P1-8 配置校验补齐
|
||||
@@ -0,0 +1,195 @@
|
||||
# P1 逐条复审订正与方案计划
|
||||
|
||||
> 复审基线:main @ `77f2b948c`(7 个 P0 修复 #5748~#5754 已全部合入)
|
||||
> 复审方式:5 组对抗性复审 agent 并行,先怀疑后确认;以 RustFS 自身功能契约为正确性标准,不以"未对齐 MinIO"为根因;RustFS 更优/独特设计标注"保持不变"
|
||||
> 参照:MinIO 源码、mc@cf909e1063a9、madmin-go v3.0.109、minio-go v7.0.91
|
||||
> 日期:2026-08-06
|
||||
|
||||
---
|
||||
|
||||
## 〇、复审总裁定表
|
||||
|
||||
| 项 | 主题 | 复审结论 | 关键订正 | 工作量 |
|
||||
|---|---|---|---|---|
|
||||
| P1-1 | ILM expiry 复制语义 | CONFIRMED(范围扩大) | 发送点共 4 处非 1 处;接收端无门禁;修复重心移到接收端 merge | M |
|
||||
| P1-3 | 自动跨站元数据 heal | CONFIRMED(范围收窄) | 真实缺口="retry queue 有账本无消费者";不移植 MinIO 全量 heal | M |
|
||||
| P1-5 | GET/HEAD 远端 proxy | CONFIRMED | 同步复制模式是已实现的部分缓解(保持不变);proxy 指标语义被出站 HEAD 污染 | L(P0 段 M) |
|
||||
| P1-6 | 三类时间戳头收发 | CONFIRMED(缺口扩大) | 实为三段缺失:tagging 无本地写入方 + 不发头 + 接收端无 LWW 合并点 | M |
|
||||
| P1-7 | ARN 前缀不互认 | CONFIRMED+(加重) | 新发现 FromStr id/region 互换 bug;madmin ParseARN 硬校验实锤 → 生成侧必须改 | M |
|
||||
| P1-8 | 配置校验缺口 + StorageClass | 部分 CONFIRMED | 2MB 子项 REFUTED(MinIO 亦无);StorageClass 属刻意设计成立(MinIO 也不消费 rule 级,target 级 RustFS 已生效)| S |
|
||||
| P1-11 | replication-metrics snake_case | CONFIRMED | BucketStats 复用内部 RPC 线格式实锤 → 必须独立响应 DTO | M |
|
||||
| P1-12 | replication-reset 响应壳 | CONFIRMED(面缩小) | 致命键仅 5 个(壳 `Targets`≠`target` + 4 个字段名);其余靠 Go 大小写不敏感能对上 | S |
|
||||
| P1-13 | mrf/diff 聚合响应 | CONFIRMED(症状加重) | 实际输出**伪数据行**而非空;diff/mrf 数据源均可支撑逐条流 | diff S / mrf M |
|
||||
| P1-14 | set-remote-target 请求体 | 原缺口已缓解;**新 CONFIRMED 阻断** | #5754 后 26 字段已全覆盖;但**零值 `expiration` 恒被拒 → mc replicate add 仍 100% 失败**;latency 单位 round-trip 污染 | S(**建议立即修**) |
|
||||
| P1-15 | site state RMW 竞争 | CONFIRMED(加重) | hook 路径 enqueue/dequeue 同进程内绕过既有 Mutex → 单节点即可触发 | M-L |
|
||||
| P1-16 | 状态机类型双份定义 | CONFIRMED(加重+收窄) | drift 已发生(MrfOpKind 两侧不一致);但 filemeta 侧 worker DTO 是死代码,活跃双份仅 3 个 wire 类型;"抽公共 crate"否决 | S+M |
|
||||
| P1-17 | 桶复制逻辑分裂 | CONFIRMED;微文件合并子项 REFUTED | boundary 微文件是棘轮机制的机械接缝(守护脚本按文件名锚定),合并负收益;缺的是完成判据 | M0=S,整体 L |
|
||||
| P1-18 | 超长函数 | 行数 CONFIRMED;apply_iam_item 降级 | apply_iam_item 长而不复杂(6 臂 dispatch),不拆降 P2;其余 4 个给纯移动拆分草案 | M |
|
||||
| P1-19 | 源→目标版本身份策略 | CONFIRMED(范围收窄) | delete-marker 的"捕获+持久化映射"模式已落地(保持不变);推荐能力探测+显式拒绝而非全量映射 | M |
|
||||
| P1-20 | scanner 补偿边界 e2e | CONFIRMED(缺口收窄) | 决策函数单测与 Failed-heal e2e 已存在;缺 existing-object 矩阵与 Replica 防环 e2e;附完整入队真值表 | M |
|
||||
| P1-21 | delayed purge 静默丢弃 | CONFIRMED | 映射损坏防护已加固(保持不变);`let _ =` 与无 MRF 通道仍在;附带发现 MRF outcome 恒 false 滞留问题 | M |
|
||||
| P1-22 | 桶复制 SSE 能力 | CONFIRMED(前提订正) | SSE-S3 自 #5633 已 fail closed,被 ignore 的 e2e 理由过期(先摘 ignore);SSE-C 缺的是目标侧头摄取 | L(4 阶段) |
|
||||
|
||||
**"保持不变"清单(复审确认的 RustFS 更优/刻意设计,不纳入修复)**:per-PUT 即时元数据传播 hook(优于 MinIO 纯周期 heal)、单向推送+stale 守卫收敛模型、delete 走 merge-with-empty(优于 MinIO 整删)、delete-marker 版本映射持久化+损坏拒猜、同步复制模式(partition_by_sync)、能力契约式显式拒绝+`deny_unknown_fields`(字段清单已与 madmin v3.0.109 同步)、StorageClass 显式拒绝非 STANDARD(target 级已真正生效)、replication-check 真实探针写删、响应中的 RustFS 增强字段(ResetBeforeDate/Error/可观测性键,Go 忽略未知键可共存)。
|
||||
|
||||
---
|
||||
|
||||
## 一、紧急项(建议立即处理)
|
||||
|
||||
### ⚡ P1-14 新阻断:零值 `expiration` 拒绝 → mc replicate add 仍 100% 失败
|
||||
|
||||
- **证据**:Go `omitempty` 不省略零值 `time.Time`(已用 Go 程序按 madmin 逐字 tag 实测),mc/madmin marshal 恒输出 `"credentials":{"expiration":"0001-01-01T00:00:00Z"}` 与 `"resetBeforeDate":"0001-01-01T00:00:00Z"`;RustFS `handlers/replication.rs:286-291` 对 `expiration.is_some()` 一律 400。#5754 的测试全部用手写 payload(`expiration: None`),未被现网形状打中。
|
||||
- **修复(S)**:①`expiration` 改"非 Go 零值时间才拒"(与 `sessionToken` trim-empty 判断对称);②`latency` 请求字段直接忽略(消除 #5754 后纳秒响应 ↔ 毫秒请求的 round-trip 1e6 倍污染);③把"Go 真实 marshal 形状 payload"固化为测试夹具惯例。
|
||||
- **红灯测试**:用实测 Go marshal 全形状 body(含零值 expiration/resetBeforeDate/latency{0,0,0}/edge:false/healthCheckDuration:60000000000)打 set-remote-target,期望 200;非零 expiration 仍 400(能力契约保持)。
|
||||
|
||||
### ⚡ P1-7 附带 bug:ARN FromStr 字段互换
|
||||
|
||||
`arn.rs` Display 输出 `{type}:{region}:{id}:{bucket}`,FromStr 却读 `id=parts[3], region=parts[4]`——id 与 region 互换。当前仅因消费方只用 arn_type 而潜伏。随 P1-7 一并修。
|
||||
|
||||
---
|
||||
|
||||
## 二、逐项方案计划
|
||||
|
||||
### P1-1 ILM expiry 复制语义(M)
|
||||
|
||||
**订正后事实**:发送完整 lifecycle XML 的路径 4 处——PUT hook(`bucket_usecase.rs:2177-2180`)、DELETE hook(`:1512-1514`,触发接收端**整删**)、import(`bucket_meta.rs:948-951`)、build_sr_info/bootstrap(`site_replication.rs:4190,2241-2249`);接收端 `apply_bucket_meta_item`(`:7669-7683`)整体覆盖/删除,且**无 `replicate_ilm_expiry` 门禁**。P0 后已有缓解(发送开关、bootstrap 跳过、stale 判定)只解决"发不发/新旧",不解决"发什么/怎么合"。
|
||||
|
||||
**方案**:接收端 merge 为主(信任边界),发送端 expiry-only 提取为辅:
|
||||
1. 新增纯函数 `extract_expiry_only(cfg)` 与 `merge_expiry_rules(local, incoming)`——语义对齐 MinIO `mergeWithCurrentLCConfig`,两处 RustFS 改进:incoming 一律先剥 transition(防旧端);`None` 走 merge-with-empty 而非整删(**MinIO 整删连本地 transition 一起删是缺陷,不照抄**);
|
||||
2. 接收端 lc-config 分支改 读→merge→条件写/删,保留 stale 判定与 incarnation 守卫;补 `replicate_ilm_expiry` 门禁;
|
||||
3. 4 个发送点接 `extract_expiry_only`;expiry 判定用 RustFS 口径(含 `del_marker_expiration`)。
|
||||
|
||||
**红灯测试**:L1 单测 5 例(提取剥离/合并保留 T/防御剥离/merge-with-empty/import 无 transition);L3 e2e——B 配本地 transition,A PUT expiry → B 两者共存;A DELETE lifecycle → B transition 仍在。
|
||||
**兼容**:旧端发完整 XML → 新接收端剥后 merge 正确;新端 expiry-only → 旧接收端仍整覆盖(不劣于现状)。规则按 ID 对齐,`rule-{idx}` 撞名同 MinIO 语义,文档注明。
|
||||
|
||||
### P1-3 自动跨站 heal → 改为"retry queue 自动 drain"(M)
|
||||
|
||||
**订正后事实**:retry queue 是现成增量账本(失败即入队 `:3243-3262`,持久化于 state,`retry_count` 字段存在)但**全库无消费者**;手动 repair 是本地快照单向推送,收敛方向依赖运维判断。即时 hook + 显式 repair 模型保持不变。
|
||||
|
||||
**方案**:
|
||||
- 阶段 1(核心):周期任务挂进现有 reconcile ticker,per-event 重发(body 从本地当前元数据重建,复用 `SiteReplicationRepairTask::send`,天然发"当前值"+对端 stale 守卫幂等);指数退避(`retry_count`+上限转 failed);drain 全程包分布式锁去抖(先用 `with_config_object_write_lock` 专用对象,P1-15 落地后并入统一 state store);结构化 tracing 汇总一条。
|
||||
- 阶段 2(可选,默认关闭):每 N tick 比对 repair plan token,不同才自动 dry-run→execute。**不移植** MinIO 跨站取最新 pull 语义(各站各自 drain 即双向收敛)。
|
||||
|
||||
**红灯测试**:L2——state 带 retry event,调 `drain_site_replication_retry_queue()`(现不存在),fake peer 成功后断言队列清空;退避断言。L3——停 B→A PUT policy 失败入队→起 B→drain 后 B 收到且 SRRetryStats 归零。
|
||||
|
||||
### P1-5 GET/HEAD 远端 proxy(L;P0 段 M)
|
||||
|
||||
**订正后事实**:`SUFFIX_SOURCE_PROXY_REQUEST` 零消费者;`ProxyMetric` 字段与 admin 汇总通路已就位,但 resyncer 把**出站** HEAD 计入 `head_total` 污染语义;`disable_proxy` 管道存在无人消费;同步复制模式(`partition_by_sync`,`replication_pool.rs:2667-2689`)是部分缓解但不等价(手动 per-target、失败仍 404、不覆盖兜底窗口)。防环头当前仅潜在问题,但 proxy 实现与防环识别**必须同 PR**(否则 RustFS↔RustFS 成环)。
|
||||
|
||||
**方案**(P0 段):新增 `replication_proxy_boundary.rs`——`proxy_targets`(version_suspended/入站 proxy 头/disable_proxy 三重 gate)+ `proxy_get/head_to_replication_target`(走现有 TargetClient,range/条件头透传);触发点在 usecase 层 NotFound/VersionNotFound 分支;接收侧 options.rs 解析防环头,出站双前缀发送;`tokio::timeout`(~3s env 可调)、仅 2xx 采纳其余回落本地 404、复用离线标记短路;指标接 `record_replication_proxy` 并纠正 resyncer 计数语义。P1 段:tagging 三操作 proxy(依赖 P1-6)。
|
||||
**红灯测试**:e2e 双站断复制链路后从对端 GET/HEAD 应 200(现 404);防环负例(带头请求不转发、计数不增);降级负例(target 全离线时限时 404);disable_proxy 负例。
|
||||
|
||||
### P1-6 时间戳头收发(M;三段修复)
|
||||
|
||||
**订正后事实**:①`SUFFIX_TAGGING_TIMESTAMP` 全仓无写入方(retention/legalhold 已有双前缀写入);②`PutObjectOptions::header()` 只序列化 4 个内部头,三类时间戳被丢弃,multipart 同;③接收端不解析,且 replica PUT 是 verbatim 覆盖——解析后必须在写盘前与本地版本做 per-类别 LWW 合并才有效;④`AdvancedPutOptions` 默认 `now_utc()` 无法当"未设置"哨兵,需 Option 化。
|
||||
|
||||
**方案**:阶段 0——`put/delete_object_tagging` 落 `SUFFIX_TAGGING_TIMESTAMP`(双前缀);阶段 1——新增三个 suffix 常量(对齐 MinIO headers.go:239-243),三字段 Option 化,`header()` 与 multipart 条件序列化;阶段 2——接收端解析(仅授权复制请求)+ PUT 路径 LWW 合并并持久化赢家时间戳(合并仅限三类元数据,不触碰数据与其余元数据,与 verbatim-replica 不变式共存)。
|
||||
**红灯测试**:单测 header 双前缀序列化断言/未设置缺席断言;接收端解析单测;e2e active-active tagging 并发收敛(晚者胜,现 main 旧值覆盖新值为红)。
|
||||
|
||||
### P1-7 ARN 前缀(M)
|
||||
|
||||
**订正后事实**:madmin `ParseARN` 硬校验 `arn:minio:` 前缀 + ID/bucket 非空(v3.0.109 remote-target-commands.go:50-63);mc 爆炸点仅 `replicate update`(fatalIf)与 `replicate ls`(软降级);`replicate add` 把 ARN 当不透明串不受影响——解释了"add 通 update 挂"。RustFS ARN 结构(`type::id:bucket`)与 madmin 兼容,仅 vendor token 障碍;另有 FromStr id/region 互换 bug(见紧急项)。
|
||||
|
||||
**方案(推荐路线 A)**:生成侧默认改 `arn:minio:`(留常量可品牌化);解析侧接受双前缀(存量 `arn:rustfs:` 靠双前缀解析 + 现有字符串等值匹配继续工作);修字段序;改 `generate_arn`、`site_replication.rs:6329` 与相关测试断言。混合版本集群前缀不一致靠双前缀解析吸收;不做存量数据前缀归一化改写。
|
||||
**红灯测试**:单测 `from_str("arn:minio:replication:us-east-1:depl:bucket")` 成功且 id/region 正确(现双重红灯);round-trip 属性测试;e2e set-remote-target 返回 ARN 可被 madmin 语义解析、预置 `arn:minio:` 目标可 remove。
|
||||
|
||||
### P1-8 配置校验(S)
|
||||
|
||||
**订正后事实**:2MB 上限 REFUTED(MinIO 亦无显式检查,剔除);StorageClass 已缓解且刻意设计成立——MinIO 自己也不消费 rule 级 `Destination.StorageClass`(复制 PUT 用 target 级 `tgt.StorageClass`),RustFS target 级 storage_class 已真正生效(`bucket_target_sys.rs:1633-1634`),容忍显式 STANDARD 已实现。仍缺:规则数≤1000、≥1 条、Priority 唯一非负、ID≤255、Filter 互斥、Tag×DeleteMarkerReplication 互斥、sameTarget 拒绝。
|
||||
|
||||
**方案**:`config.rs` 新增 `validate_replication_config_structure` 纯函数,`bucket_usecase.rs:2418` 接入;StorageClass 保持现状+契约文档化("rule 级请改用 remote target 的 storageclass 字段")。
|
||||
**红灯测试**:单测逐格(1001 规则/重复 Priority/256 字符 ID/Filter 并存/Tag+DMR)期望特定错误;e2e aws-sdk 形状 XML 断言 InvalidRequest。
|
||||
|
||||
### P1-11 replication-metrics DTO(M)
|
||||
|
||||
**订正后事实**:`BucketStats` 走内部 peer RPC 线格式(`rmp_serde::to_vec_named` 字段名入线,node_service.rs:1401 / peer_rest_client.rs:88-104)——**改原结构 serde 名会破坏混合版本集群 RPC,禁止**;必须走 #5754 的响应 DTO 模式(同文件先例 `remote_target_admin_json`)。
|
||||
|
||||
**方案**:新增仅 Serialize 的 `MetricsV2Dto{uptime,currStats,queueStats,downtimeInfo}`/`MetricsDto`/`TargetMetricsDto`,显式映射(`q_stat`→`queued`、`bandwidth_limit_bytes_per_sec`→`limitInBits`、failed→TimedErrStats total-only);`queueStats.nodes` 先填本机一条;RustFS 可观测性扩展键保留(Go 忽略未知键,双栖零成本)。
|
||||
**红灯测试**:e2e 用镜像 minio-go MetricsV2 tag 的结构反序列化断言 `currStats.completedReplicationSize > 0`(现全零);DTO 键名 snapshot 单测。
|
||||
|
||||
### P1-12 replication-reset 响应壳(S)
|
||||
|
||||
**订正后事实**:致命键仅 5 个——壳 `Targets`≠`target`、`Status`≠`resyncStatus`、`ReplicatedSize`≠`completedReplicationSize`、`ReplicatedCount`≠`replicationCount`、`FailedSize/FailedCount`≠`failedReplicationSize/failedReplicationCount`;其余(Arn/ResetID/StartTime/...)靠 Go 大小写不敏感能对上;`ResetBeforeDate`/`Error` 是增强字段可保留。响应结构是 router.rs 独立 DTO 无内部复用,改名零风险。
|
||||
|
||||
**方案**:纯 serde rename(建议全字段精确对齐 madmin 小写形态),保留增强键+文档标注。
|
||||
**红灯测试**:e2e 断言响应含 `target` 数组且 `target[0].resetid` 非空、status 侧 `resyncStatus`/`completedReplicationSize` 键存在。
|
||||
|
||||
### P1-13 mrf/diff 流式响应(diff S / mrf M)
|
||||
|
||||
**订正后事实**:症状比"输出空"更糟——聚合对象会被 madmin `json.Decoder` 成功解码一次,`mc replicate backlog` 输出一条 object 为空的**伪行**(静默伪数据);路线 A(保持聚合+文档化)无法消除伪行且与 madmin 同 path 无内容协商,**不可行**。数据源评估:diff 已逐条扫描只需去壳;mrf 的 durable backlog(`MrfReplicateEntry` 字段恰好覆盖 `ReplicationMRF` 所需)已可枚举。
|
||||
|
||||
**方案(路线 B)**:diff 去壳输出 NDJSON `DiffInfo` 形状(仅 `IsDeleteMarker`/`ReplicationStatus` 需 rename;truncation 信息入日志不入流);mrf 遍历 durable entries 逐条输出 `ReplicationMRF` 形状(nodeName 填本机);聚合响应保留在 `?aggregate=true`(RustFS 扩展,deliberate 注释随迁)。条目量有 `REPLICATION_DIFF_MAX_SCAN` 封顶,内存拼 NDJSON 即可不必真流式。
|
||||
**红灯测试**:e2e 制造失败复制后逐行反序列化断言至少一条 `object` 非空(现为伪空行);diff 断言无 `Entries` 壳。
|
||||
|
||||
### P1-14 set-remote-target(S,含紧急项)
|
||||
|
||||
见"一、紧急项"。另:`deny_unknown_fields` **保留**(推荐)——字段清单已与 madmin v3.0.109 全同步,严格模式+显式清单兼得契约哲学与防静默;代价写进维护清单:"madmin 版本升级时同步字段清单"(加对照 madmin tag 列表的常量测试防漂移)。
|
||||
|
||||
### P1-15 site state 统一 store(M-L,两 PR)
|
||||
|
||||
**订正后事实**:主 state 有进程内 Mutex(`:347`)但两处不完备——①无分布式锁(多节点 RMW 丢更新);②**retry event enqueue/dequeue 不持锁**(挂在所有 hook 广播路径上,同进程即可丢更新);reload 路径完全无锁(稳态不写盘收窄窗口,迁移期可覆盖并发写)。repair state 的 `with_config_object_write_lock` + no-lock IO 是正确样板(`:1097-1114`);两套归一化的语义差异(JSON-level 容忍畸形 peer)是**有意的**,统一时必须保留。锁序注释 `:346` 可挂靠。
|
||||
|
||||
**方案**:PR1——新建 `admin/site_replication_state.rs`:两阶段归一化合一(JSON 宽容清洗→类型化)、`read_state()/update_state(F)`(分布式锁包完整 RMW,锁内禁网络调用与嵌套配置锁)、常量收敛;service reload 接入;迁移 service 侧 5 个归一化测试保语义。PR2——迁移全部 ~30 个 RMW 调用点(含 enqueue/dequeue),**移除**进程内 Mutex(避免双锁新顺序约束);dequeue 热路径保留"先无锁读、命中才进 update_state"两段式;更新锁序注释。每个调用点做重入审查(现有 drop-reacquire 模式保持)。
|
||||
**红灯测试**:L2 单进程并发——持锁 RMW(mark_pending_rotation_peer_acked)×绕锁写者(enqueue_retry_event)注入交错,断言最终 state 两者共存(现必丢其一,确定性红灯);L1 归一化等价性测试迁移;L3 双节点并发(nice-to-have)。
|
||||
**风险**:盘上格式不变;锁超时从"静默丢更新"变"显式报错",hook 路径保持 warn 不阻断 S3 主路径。
|
||||
|
||||
### P1-16 类型对账护栏(S)+ 死代码清理(M)
|
||||
|
||||
**订正后事实**:drift 已发生(filemeta 侧 `MrfOpKind` 缺 Metadata/Heal/ExistingObject 三 variant、`MrfReplicateEntry` 缺 force_delete/target_arns)——但 filemeta 侧 8 个 worker DTO 全是**死代码**(零消费者);活跃双份仅 `ReplicationStatusType/VersionPurgeStatusType/ReplicationState` 三个 wire 类型(filemeta 绑 xl.meta 磁盘格式,replication 绑 MRF/resync 持久化格式);boundary 枚举转换 `as_str()` 兜底 `_ => Empty` 会静默降级。"抽公共 leaf crate"否决(两 wire 格式演进节奏不同,迁移规则 #12 本意是所有权独立)。
|
||||
|
||||
**方案**:Step 1(S,即刻)——boundary 加对账测试:两侧枚举穷尽 match(新增 variant 即编译失败)+ as_str 双向 round-trip + ReplicationState 全字段往返;Step 2(M)——清理 filemeta 侧 ~600 行死代码 DTO,注意 crates.io semver(先 `#[deprecated]` 一版再删);Step 3(S)——replication 侧注释指向对账测试。
|
||||
|
||||
### P1-17 迁移完成判据(M0=S;整体 L)
|
||||
|
||||
**订正后事实**:"合并 boundary 微文件"REFUTED——守护脚本按具体文件名锚定每个 boundary,合并要同步改脚本+mod+导入点而功能收益为零;微文件是棘轮机制的机械接缝。唯一可退役:`datatypes.rs`(消费者迁完即删)。README 建议的第一步(event sink/runtime boundary)实际已部分落地,文档滞后。
|
||||
|
||||
**方案**:M0(S)文档 PR——完成判据 = Required Contracts 表 "Current dependency to remove" 列清空;终态 = pool/resyncer/state 移入 crates/replication,boundary 随 crate 移动自然消解;更新 split-plan "Proposal only" 状态。M2(M)resyncer 纯决策逻辑下沉;M3(L)trait 稳定后移 worker 运行时(全计划唯一高危段,最后做);M4(S)统一退役 boundary 与守护条目。**不做**批量合并微文件。
|
||||
|
||||
### P1-18 超长函数拆分(M;4 个 PR)
|
||||
|
||||
**订正后事实**:行数确认(resync_bucket 537 / start_mrf_processor 306 / replicate_all 409 / delete 路径 replicate_object 299 / apply_iam_item 255);`apply_iam_item` **降级 P2 不拆**(6 臂 dispatch,每臂线性短小,拆分违反 "Prefer direct, local code");`replicate_object` 有两个同名体,原清单指 delete 路径 trait impl。
|
||||
|
||||
**方案**(每函数独立 PR,纯移动,`git diff --color-moved=dimmed-zebra` 验证):
|
||||
1. `resync_bucket`(最优先,三处历史并发 bug 注释所在):acquire_resync_leadership / load_resync_replication_config / spawn workers+collector 三段抽出,并发 bug 注释随代码移动,每个 return 前的 mark_status 逐一保持;
|
||||
2. `start_mrf_processor`:抽 `reconstruct_mrf_delete/object` 纯函数(主循环 -150 行,重建逻辑可单测);
|
||||
3. `replicate_all` + delete 路径 `replicate_object`:各拆 3-4 个阶段 helper;**明确不合并两函数**(delete-marker 404/405 校验语义是刻意差异)。
|
||||
**排序依赖**:先 P1-18 拆分、后 P1-17 M2/M3 迁移(小函数降低搬运风险)。
|
||||
|
||||
### P1-19 版本身份策略(M,推荐方案 B)
|
||||
|
||||
**订正后事实**:#5752 已合入(PUT/multipart initiate 带 query,RustFS 目标侧也支持);PUT 响应 `x-amz-version-id` 仍被丢弃(`:1891 Ok(_)`);**delete-marker 子案已系统性缓解**——`remove_object` 捕获目标版本号→`target_delete_marker_version_ids` 持久化进 xl.meta(含上限与损坏标记)→延迟 purge 优先用映射、损坏拒猜(**保持不变**);RustFS 无"仅支持 MinIO 目标"契约声明;replication-check 探针已捕获响应版本号但不比对。MinIO 同样丢弃响应版本号(平价),RustFS 已有两点增强。
|
||||
|
||||
**方案对比**:A 全量映射持久化(完整但 xl.meta 膨胀、全链路改造,L);**B(推荐)**:契约=仅支持"沿用源版本 ID"的目标,在 replication-check 增加 VersionFidelity phase(探针 PUT 带 versionId query,比对响应版本号)+ `validate_target` 复用同一探测,不镜像则新错误 `BucketRemoteTargetVersionMismatch` 显式拒绝/告警(M);C 混合(无需求支撑)。探针是主动写,进 validate_target 会扩 set-target 副作用面——可先只做 check phase + 运行期首次 PUT 抽查告警。
|
||||
**红灯测试**:FakeS3Target 加 `assign_own_version_ids` 开关模拟原生 S3,断言版本删除复制落空(现红)与探测后显式拒绝(修后绿)。
|
||||
|
||||
### P1-20 scanner 补偿边界 e2e(M,纯测试)
|
||||
|
||||
**订正后事实**:决策函数单测(queue.rs 7 例等)与 scanner 驱动的 Failed-heal e2e(target 断电恢复/源重启重放,FAST_SCANNER_ENV)已存在;真实缺口=无任何"先写对象→后配复制"的 existing-object 用例。完整入队真值表已梳理(见复审记录):Enabled×Empty 补齐、Pending/Failed 恒补(不受 existing 开关影响)、Disabled×Empty 永不补、Replica 恒不补(防环)、null-version 永不入队、reset_id 重置补齐。
|
||||
|
||||
**方案**:e2e 矩阵 1-2 个用例(先 PUT 四种来源对象含 Copy/Snowball 产物→后配 Enabled/Disabled 规则→正例 wait_for_replicated_object / 负例 assert_failed_replication_stays_absent_for ≥3 周期,**"永不补齐"是契约必须显式断言**)+ Replica 防环变体 + queue.rs 补 2 格单测;null-version 跳过行为先写"记录现状"断言并注明出处。不改产品代码。
|
||||
|
||||
### P1-21 delayed purge 失败处理(M)
|
||||
|
||||
**订正后事实**:静默点两处——target client 缺失 `continue` 无日志(`:1673-1675`)、`let _ = remove_object`(`:1693-1700`);5 次循环是等源 marker 消失非重试;purge 调用后无条件 break;MRF 入队接口(`queue_replica_delete_task`,队满自动落盘)同 crate 可用无分层障碍;映射优先/损坏拒猜是已加固项保持不变。**附带发现**(建议单独跟进):`requires_delayed_purge` 恒真使 delete-marker 类 MRF 条目 outcome 恒 false → 重放永远 Missed 保留,可能永久滞留。
|
||||
|
||||
**方案**:①purge 函数返回 per-target 成败,失败 warn(带 event 常量)+ metrics,client 缺失同样 warn(S);②循环内失败重试、轮次耗尽入 MRF、入队失败 warn+metric 兜底(S/M);③两层失败注入测试(mock 503 断言重试/状态/MRF;FakeS3Target inject 断言故障清除后最终收敛)(M)。风险:MRF 重放重发 DELETE marker 创建——mtime 幂等,风险低。
|
||||
|
||||
### P1-22 SSE 能力(L,4 阶段)
|
||||
|
||||
**订正后事实**:fail-closed 由 #5633 引入(`replication_target_boundary.rs:101-174`),普通/Heal/Resync/Multipart 全走同一函数;SSE-C 发送半边已建(内部头→`X-Rustfs-Replication-*` 映射+CRC),**目标侧摄取代码完全缺失**(链路必断,e2e 已钉 FAILED);SSE-S3 契约 e2e 的 `#[ignore]` 理由(backlog#1291 silently drops)已被 #5633 过期;直传托管 SSE 不可行(封存密钥绑本站 KMS),MinIO 是源解密+目标重加密;ecstore 已有 `ObjectEncryptionResolver` trait seam,解密不破分层。
|
||||
|
||||
**方案**:阶段 0(S)摘 ignore + 补 encrypted resync/heal e2e 钉全矩阵 fail-closed 现状;阶段 1(M)SSE-C 目标侧头摄取+加密尺寸/CRC(MinIO :1670-1740 参照);阶段 2(M/L)SSE-S3 经 resolver 解密+目标 AES256 重加密(resolver 未注册必须继续 fail closed;multipart 按明文尺寸分片);阶段 3(L)SSE-KMS + key id 随行开关(目标站无同名 key 显式失败,禁止回退 SSE-S3)。过渡期全矩阵维持 fail closed,禁止明文降级。
|
||||
|
||||
---
|
||||
|
||||
## 三、执行批次建议
|
||||
|
||||
| 批次 | 内容 | 性质 |
|
||||
|---|---|---|
|
||||
| **B0 立即** | P1-14 零值 expiration + latency 忽略(S);P1-7 FromStr 字段互换(并入 P1-7 或先行) | mc 阻断修复 |
|
||||
| **B1 小改动高收益** | P1-12 响应壳 rename(S)、P1-13 diff 去壳(S)、P1-8 结构校验(S)、P1-16 Step1 对账测试(S)、P1-17 M0 文档判据(S)、P1-22 阶段 0 摘 ignore(S) | serde/校验/测试护栏 |
|
||||
| **B2 数据一致性** | P1-21 purge 失败处理(M)→ P1-20 scanner 矩阵 e2e(M,纯测试)→ P1-19 方案 B 能力探测(M)→ P1-15 state store PR1+PR2(M-L) | 一致性核心 |
|
||||
| **B3 互操作补齐** | P1-7 ARN 路线 A(M)、P1-11 MetricsV2 DTO(M)、P1-13 mrf 流(M)、P1-6 时间戳三段(M)、P1-1 ILM merge(M)、P1-3 retry drain(M) | mc/跨站语义 |
|
||||
| **B4 大功能与架构** | P1-5 proxy P0 段(M→L)、P1-22 阶段 1-3(L)、P1-18 四函数拆分(M)→ P1-17 M2-M4(L)、P1-16 Step2 死代码(M) | 长期 |
|
||||
|
||||
**批内依赖**:P1-6 先于 P1-5 的 tagging proxy;P1-18 先于 P1-17 M2/M3;P1-15 PR1 的锁对象可先供 P1-3 drain 使用。
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.12
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.12
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录：
|
||||
|
||||
@@ -39,15 +39,11 @@ tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
tonic = { workspace = true, features = ["gzip", "deflate"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
rmp-serde = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
use crate::heal_channel::HealScanMode;
|
||||
use crate::last_minute::{AccElem, LastMinuteLatency};
|
||||
use chrono::{DateTime, Utc};
|
||||
use jiff::Timestamp;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::{BTreeSet, HashMap},
|
||||
@@ -670,7 +669,7 @@ impl LockedLastMinuteLatency {
|
||||
#[derive(Clone, Debug)]
|
||||
struct CurrentPathState {
|
||||
path: String,
|
||||
updated_at: Timestamp,
|
||||
updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
struct CurrentPathTracker {
|
||||
@@ -679,10 +678,10 @@ struct CurrentPathTracker {
|
||||
|
||||
impl CurrentPathTracker {
|
||||
fn new(initial_path: String) -> Self {
|
||||
Self::new_at(initial_path, Timestamp::now())
|
||||
Self::new_at(initial_path, Utc::now())
|
||||
}
|
||||
|
||||
fn new_at(initial_path: String, updated_at: Timestamp) -> Self {
|
||||
fn new_at(initial_path: String, updated_at: DateTime<Utc>) -> Self {
|
||||
Self {
|
||||
state: Arc::new(RwLock::new(CurrentPathState {
|
||||
path: initial_path,
|
||||
@@ -694,7 +693,7 @@ impl CurrentPathTracker {
|
||||
async fn update_path(&self, path: String) {
|
||||
let mut state = self.state.write().await;
|
||||
state.path = path;
|
||||
state.updated_at = Timestamp::now();
|
||||
state.updated_at = Utc::now();
|
||||
}
|
||||
|
||||
async fn get_state(&self) -> CurrentPathState {
|
||||
@@ -702,36 +701,6 @@ impl CurrentPathTracker {
|
||||
}
|
||||
}
|
||||
|
||||
fn chrono_to_jiff_timestamp(dt: DateTime<Utc>) -> Timestamp {
|
||||
let seconds = dt.timestamp();
|
||||
let nanoseconds = match i32::try_from(dt.timestamp_subsec_nanos()) {
|
||||
Ok(nanoseconds) => nanoseconds,
|
||||
Err(_) => {
|
||||
return if seconds < 0 { Timestamp::MIN } else { Timestamp::MAX };
|
||||
}
|
||||
};
|
||||
|
||||
match Timestamp::new(seconds, nanoseconds) {
|
||||
Ok(timestamp) => timestamp,
|
||||
Err(_) => {
|
||||
if seconds < 0 {
|
||||
Timestamp::MIN
|
||||
} else {
|
||||
Timestamp::MAX
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
|
||||
let duration = now.duration_since(earlier);
|
||||
if duration.is_negative() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
u64::try_from(duration.as_secs()).map_or(u64::MAX, |seconds| seconds)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
struct ScannerDiskBucketScanState {
|
||||
concurrency_limit: u64,
|
||||
@@ -1197,12 +1166,12 @@ pub struct ScannerLastMinute {
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct ScannerMetricsReport {
|
||||
pub collected_at: Timestamp,
|
||||
pub collected_at: DateTime<Utc>,
|
||||
pub current_cycle: u64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_active: bool,
|
||||
pub current_started: Timestamp,
|
||||
pub cycles_completed_at: Vec<Timestamp>,
|
||||
pub current_started: DateTime<Utc>,
|
||||
pub cycles_completed_at: Vec<DateTime<Utc>>,
|
||||
pub ongoing_buckets: usize,
|
||||
#[serde(default)]
|
||||
pub active_scan_paths: usize,
|
||||
@@ -3019,8 +2988,8 @@ impl Metrics {
|
||||
let cycle = self.cycle_info.read().await;
|
||||
let has_cycle = if let Some(cycle) = cycle.as_ref() {
|
||||
m.current_cycle = cycle.current;
|
||||
m.cycles_completed_at = cycle.cycle_completed.iter().copied().map(chrono_to_jiff_timestamp).collect();
|
||||
m.current_started = chrono_to_jiff_timestamp(cycle.started);
|
||||
m.cycles_completed_at = cycle.cycle_completed.clone();
|
||||
m.current_started = cycle.started;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
@@ -3055,15 +3024,15 @@ impl Metrics {
|
||||
};
|
||||
|
||||
if !has_cycle && let Some(init_time) = crate::get_global_init_time().await {
|
||||
m.current_started = chrono_to_jiff_timestamp(init_time);
|
||||
m.current_started = init_time;
|
||||
}
|
||||
|
||||
m.collected_at = Timestamp::now();
|
||||
m.collected_at = Utc::now();
|
||||
let current_path_snapshots = self.current_path_snapshots().await;
|
||||
m.active_scan_paths = current_path_snapshots.len();
|
||||
m.oldest_active_path_age_seconds = current_path_snapshots
|
||||
.iter()
|
||||
.map(|(_, state)| timestamp_elapsed_seconds_since(m.collected_at, state.updated_at))
|
||||
.map(|(_, state)| m.collected_at.signed_duration_since(state.updated_at).num_seconds().max(0) as u64)
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
m.active_paths = current_path_snapshots
|
||||
@@ -3339,22 +3308,6 @@ impl Drop for CloseDiskGuard {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_report_timestamps_serialize_as_rfc3339_utc() {
|
||||
let report = ScannerMetricsReport {
|
||||
collected_at: Timestamp::constant(1_700_000_000, 123_456_000),
|
||||
current_started: Timestamp::constant(1_699_999_940, 0),
|
||||
cycles_completed_at: vec![Timestamp::constant(1_700_000_060, 987_654_000)],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let value = serde_json::to_value(&report).expect("scanner metrics report should serialize");
|
||||
|
||||
assert_eq!(value["collected_at"].as_str(), Some("2023-11-14T22:13:20.123456Z"));
|
||||
assert_eq!(value["current_started"].as_str(), Some("2023-11-14T22:12:20Z"));
|
||||
assert_eq!(value["cycles_completed_at"][0].as_str(), Some("2023-11-14T22:14:20.987654Z"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn close_disk_guard_runs_cleanup_when_an_early_return_drops_it() {
|
||||
let (closed_tx, closed_rx) = tokio::sync::oneshot::channel();
|
||||
@@ -3413,7 +3366,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn report_counts_active_scan_paths() {
|
||||
let metrics = Metrics::new();
|
||||
let updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(12);
|
||||
let updated_at = Utc::now() - chrono::Duration::seconds(12);
|
||||
metrics.current_paths.write().await.insert(
|
||||
"disk-a".to_string(),
|
||||
Arc::new(CurrentPathTracker::new_at("bucket-a".to_string(), updated_at)),
|
||||
@@ -3435,7 +3388,7 @@ mod tests {
|
||||
let metrics = Metrics::new();
|
||||
let tracker = Arc::new(CurrentPathTracker::new_at(
|
||||
"bucket-a".to_string(),
|
||||
Timestamp::now() - jiff::SignedDuration::from_secs(60 * 60),
|
||||
Utc::now() - chrono::Duration::hours(1),
|
||||
));
|
||||
metrics
|
||||
.current_paths
|
||||
@@ -4208,7 +4161,7 @@ mod tests {
|
||||
let report = metrics.report().await;
|
||||
*crate::globals::GLOBAL_INIT_TIME.write().await = previous_init_time;
|
||||
|
||||
assert_eq!(report.current_started, chrono_to_jiff_timestamp(cycle_started));
|
||||
assert_eq!(report.current_started, cycle_started);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -4631,7 +4584,7 @@ mod tests {
|
||||
let active = metrics.report().await;
|
||||
assert!(active.current_cycle_active);
|
||||
assert_eq!(active.current_cycle, 12);
|
||||
assert_eq!(active.current_started, chrono_to_jiff_timestamp(cycle_started));
|
||||
assert_eq!(active.current_started, cycle_started);
|
||||
|
||||
let idle_cycle = CurrentCycle {
|
||||
current: 0,
|
||||
|
||||
@@ -177,9 +177,10 @@ const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
|
||||
///
|
||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
|
||||
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
|
||||
/// state holds roughly `authenticated RPC RPS x 601s` entries. This default is the minimum floor:
|
||||
/// explicit operator values and resource-aware auto sizing both clamp upward to at least this
|
||||
/// value. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
|
||||
/// state holds roughly `authenticated RPC RPS x 601s` entries. The default sustains about 1,700
|
||||
/// authenticated RPCs per second (about 120 MiB worst case, allocated only under sustained load);
|
||||
/// operators must size it for the node's aggregate peak RPC rate before enabling strict replay
|
||||
/// scope. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
|
||||
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
|
||||
/// the shared secret) — and increments
|
||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
|
||||
|
||||
@@ -37,10 +37,6 @@ pub const USAGE_LAST_UPDATE_FUTURE_TOLERANCE: Duration = Duration::from_secs(5 *
|
||||
/// Keeping the existing object name preserves rolling-upgrade and rollback
|
||||
/// compatibility without allowing an ambiguous snapshot to become authoritative.
|
||||
pub const DATA_USAGE_OBJECT_NAME: &str = ".usage.v2.json";
|
||||
/// Latest structurally complete scanner observation. Unlike
|
||||
/// [`DATA_USAGE_OBJECT_NAME`], this object is never authoritative for quota
|
||||
/// admission because namespace activity may have raced the scan.
|
||||
pub const DATA_USAGE_OBSERVED_OBJECT_NAME: &str = ".usage.observed.json";
|
||||
|
||||
/// Usage snapshot written by scanner implementations predating distributed
|
||||
/// leadership fencing. It is read only when neither authoritative snapshot
|
||||
@@ -222,20 +218,6 @@ pub struct DataUsageInfo {
|
||||
/// explicit entry for every bucket, including confirmed-empty buckets.
|
||||
#[serde(default)]
|
||||
pub usage_snapshot_complete: bool,
|
||||
/// Whether no namespace activity or dirty-usage generation changed while
|
||||
/// the coordinated snapshot was being produced.
|
||||
///
|
||||
/// `false` still describes a structurally complete, useful point-in-time
|
||||
/// usage view, but follow-up scanner work remains pending. `None` is kept
|
||||
/// for snapshots written before this status became observable.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_snapshot_converged: Option<bool>,
|
||||
/// Identity of the authoritative snapshot from which a nonconverged
|
||||
/// observation started. Admin readers require an exact match before using
|
||||
/// the observation, so bucket namespace mutations fence old observations
|
||||
/// without relying on synchronized clocks.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_snapshot_authoritative_baseline: Option<DataUsageSnapshotIdentity>,
|
||||
/// Deprecated kept here for backward compatibility reasons
|
||||
pub bucket_sizes: HashMap<String, u64>,
|
||||
/// Per-disk snapshot information when available
|
||||
@@ -243,59 +225,6 @@ pub struct DataUsageInfo {
|
||||
pub disk_usage_status: Vec<DiskUsageStatus>,
|
||||
}
|
||||
|
||||
/// Stable identity fields changed by both coordinated scanner publication and
|
||||
/// backward-compatible bucket namespace cleanup.
|
||||
#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct DataUsageSnapshotIdentity {
|
||||
pub last_update: Option<SystemTime>,
|
||||
pub scanner_cycle: Option<u64>,
|
||||
pub scanner_epoch: Option<u64>,
|
||||
}
|
||||
|
||||
impl DataUsageInfo {
|
||||
pub fn snapshot_identity(&self) -> DataUsageSnapshotIdentity {
|
||||
DataUsageSnapshotIdentity {
|
||||
last_update: self.last_update,
|
||||
scanner_cycle: self.scanner_cycle,
|
||||
scanner_epoch: self.scanner_epoch,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether `candidate` was produced after `baseline`.
|
||||
///
|
||||
/// New coordinated snapshots are ordered by leadership epoch and scanner
|
||||
/// cycle. The timestamp fallback preserves ordering for legacy snapshots that
|
||||
/// predate those fields.
|
||||
pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUsageInfo) -> bool {
|
||||
match (
|
||||
candidate.scanner_epoch.zip(candidate.scanner_cycle),
|
||||
baseline.scanner_epoch.zip(baseline.scanner_cycle),
|
||||
) {
|
||||
(Some(candidate), Some(baseline)) => candidate > baseline,
|
||||
(Some(_), None) => true,
|
||||
(None, Some(_)) => false,
|
||||
(None, None) => match (candidate.last_update, baseline.last_update) {
|
||||
(Some(candidate), Some(baseline)) => candidate > baseline,
|
||||
(Some(_), None) => true,
|
||||
(None, Some(_) | None) => false,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether a nonconverged observation may safely supersede the admin
|
||||
/// view of `authoritative`.
|
||||
///
|
||||
/// The exact baseline identity is independent of clock ordering. Older binaries
|
||||
/// already advance the authoritative timestamp when deleting a bucket, so a
|
||||
/// rollback delete/recreate fences the previous bucket incarnation too.
|
||||
pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &DataUsageInfo) -> bool {
|
||||
observed.usage_snapshot_converged == Some(false)
|
||||
&& observed.is_complete_bucket_usage_snapshot()
|
||||
&& observed.usage_snapshot_authoritative_baseline.as_ref() == Some(&authoritative.snapshot_identity())
|
||||
&& data_usage_snapshot_is_newer(observed, authoritative)
|
||||
}
|
||||
|
||||
/// Metadata describing the status of a disk-level data usage snapshot.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct DiskUsageStatus {
|
||||
@@ -1854,8 +1783,6 @@ mod tests {
|
||||
let current = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(DataUsageSnapshotIdentity::default()),
|
||||
..Default::default()
|
||||
};
|
||||
let encoded = rmp_serde::to_vec_named(¤t).expect("encode current data usage snapshot");
|
||||
@@ -1863,76 +1790,6 @@ mod tests {
|
||||
|
||||
assert_eq!(legacy.buckets_count, 0);
|
||||
assert!(current.is_complete_bucket_usage_snapshot());
|
||||
assert_eq!(current.usage_snapshot_converged, Some(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn convergence_marker_defaults_to_unknown_for_older_snapshots() {
|
||||
let encoded = rmp_serde::to_vec_named(&DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("encode pre-convergence data usage snapshot");
|
||||
let decoded: DataUsageInfo = rmp_serde::from_slice(&encoded).expect("decode older data usage snapshot");
|
||||
|
||||
assert!(decoded.is_complete_bucket_usage_snapshot());
|
||||
assert_eq!(decoded.usage_snapshot_converged, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observation_selection_is_clock_independent_and_baseline_fenced() {
|
||||
let mut authoritative = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(600)),
|
||||
scanner_epoch: Some(7),
|
||||
scanner_cycle: Some(10),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let observed = DataUsageInfo {
|
||||
// A newer leader may have a slower wall clock.
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
|
||||
scanner_epoch: Some(8),
|
||||
scanner_cycle: Some(1),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(observed_data_usage_is_newer(&observed, &authoritative));
|
||||
|
||||
authoritative.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(601));
|
||||
assert!(
|
||||
!observed_data_usage_is_newer(&observed, &authoritative),
|
||||
"an old-binary namespace mutation must fence the prior bucket incarnation regardless of clock skew"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observation_selection_requires_nonconverged_complete_newer_data() {
|
||||
let authoritative = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(10),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let baseline = Some(authoritative.snapshot_identity());
|
||||
let candidate = |epoch, cycle, converged, complete| DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
|
||||
scanner_epoch: Some(epoch),
|
||||
scanner_cycle: Some(cycle),
|
||||
usage_snapshot_complete: complete,
|
||||
usage_snapshot_converged: converged,
|
||||
usage_snapshot_authoritative_baseline: baseline,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(observed_data_usage_is_newer(&candidate(2, 11, Some(false), true), &authoritative));
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 9, Some(false), true), &authoritative));
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(true), true), &authoritative));
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(false), false), &authoritative));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -16,17 +16,91 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, signed_s3_request};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{
|
||||
AccelerateConfiguration, BucketAccelerateStatus, BucketLoggingStatus, IndexDocument, LoggingEnabled, Payer,
|
||||
RequestPaymentConfiguration, WebsiteConfiguration,
|
||||
};
|
||||
use http::Method;
|
||||
use http::header::CONTENT_TYPE;
|
||||
use serial_test::serial;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::info;
|
||||
|
||||
fn awscurl_binary_path() -> PathBuf {
|
||||
std::env::var_os("AWSCURL_PATH")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("awscurl"))
|
||||
}
|
||||
|
||||
fn awscurl_available() -> bool {
|
||||
Command::new(awscurl_binary_path()).arg("--version").output().is_ok()
|
||||
}
|
||||
|
||||
fn execute_s3_awscurl(
|
||||
method: &str,
|
||||
url: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let output = Command::new(awscurl_binary_path())
|
||||
.args([
|
||||
"--service",
|
||||
"s3",
|
||||
"--region",
|
||||
"us-east-1",
|
||||
"--access_key",
|
||||
access_key,
|
||||
"--secret_key",
|
||||
secret_key,
|
||||
"-i",
|
||||
"-X",
|
||||
method,
|
||||
url,
|
||||
])
|
||||
.output()?;
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into());
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&output.stdout).to_string())
|
||||
}
|
||||
|
||||
fn parse_status(raw: &str) -> Option<u16> {
|
||||
raw.lines()
|
||||
.filter_map(|line| {
|
||||
if line.starts_with("HTTP/") {
|
||||
line.split_whitespace().nth(1)?.parse::<u16>().ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.next_back()
|
||||
}
|
||||
|
||||
fn parse_body(raw: &str) -> String {
|
||||
if let Some(pos) = raw.rfind("\r\n\r\n") {
|
||||
return raw[pos + 4..].to_string();
|
||||
}
|
||||
if let Some(pos) = raw.rfind("\n\n") {
|
||||
return raw[pos + 2..].to_string();
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn parse_headers(raw: &str) -> String {
|
||||
let start = raw.rfind("HTTP/").unwrap_or(0);
|
||||
let tail = &raw[start..];
|
||||
if let Some(pos) = tail.find("\r\n\r\n") {
|
||||
return tail[..pos].to_string();
|
||||
}
|
||||
if let Some(pos) = tail.find("\n\n") {
|
||||
return tail[..pos].to_string();
|
||||
}
|
||||
tail.to_string()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_dummy_bucket_compatibility_endpoints() {
|
||||
@@ -396,6 +470,10 @@ mod tests {
|
||||
async fn test_dummy_bucket_endpoints_http_contracts() {
|
||||
init_logging();
|
||||
info!("Starting test: dummy-compat bucket API HTTP contracts");
|
||||
if !awscurl_available() {
|
||||
info!("Skipping test_dummy_bucket_endpoints_http_contracts: awscurl binary not found");
|
||||
return;
|
||||
}
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
@@ -410,112 +488,56 @@ mod tests {
|
||||
.await
|
||||
.expect("Failed to create bucket");
|
||||
|
||||
let logging_response = signed_s3_request(
|
||||
Method::GET,
|
||||
&format!("{}/{bucket}?logging=", env.url),
|
||||
None,
|
||||
None,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await
|
||||
.expect("GetBucketLogging HTTP request failed");
|
||||
assert_eq!(logging_response.status(), 200, "GetBucketLogging should return 200");
|
||||
let logging_body = logging_response
|
||||
.text()
|
||||
.await
|
||||
.expect("Failed to read GetBucketLogging response body");
|
||||
let logging_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?logging=", env.url), &env.access_key, &env.secret_key)
|
||||
.expect("GetBucketLogging HTTP request failed");
|
||||
assert_eq!(parse_status(&logging_raw), Some(200), "GetBucketLogging should return 200");
|
||||
let logging_body = parse_body(&logging_raw);
|
||||
assert!(
|
||||
logging_body.contains("<BucketLoggingStatus"),
|
||||
"GetBucketLogging response should contain BucketLoggingStatus XML, got: {logging_body}"
|
||||
);
|
||||
|
||||
let accel_response = signed_s3_request(
|
||||
Method::GET,
|
||||
&format!("{}/{bucket}?accelerate=", env.url),
|
||||
None,
|
||||
None,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await
|
||||
.expect("GetBucketAccelerateConfiguration HTTP request failed");
|
||||
assert_eq!(accel_response.status(), 200, "GetBucketAccelerateConfiguration should return 200");
|
||||
let accel_body = accel_response
|
||||
.text()
|
||||
.await
|
||||
.expect("Failed to read GetBucketAccelerateConfiguration response body");
|
||||
let accel_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?accelerate=", env.url), &env.access_key, &env.secret_key)
|
||||
.expect("GetBucketAccelerateConfiguration HTTP request failed");
|
||||
assert_eq!(parse_status(&accel_raw), Some(200), "GetBucketAccelerateConfiguration should return 200");
|
||||
let accel_body = parse_body(&accel_raw);
|
||||
assert!(
|
||||
accel_body.contains("<AccelerateConfiguration"),
|
||||
"GetBucketAccelerateConfiguration response should contain AccelerateConfiguration XML, got: {accel_body}"
|
||||
);
|
||||
|
||||
let payment_response = signed_s3_request(
|
||||
Method::GET,
|
||||
&format!("{}/{bucket}?requestPayment=", env.url),
|
||||
None,
|
||||
None,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await
|
||||
.expect("GetBucketRequestPayment HTTP request failed");
|
||||
assert_eq!(payment_response.status(), 200, "GetBucketRequestPayment should return 200");
|
||||
let payment_body = payment_response
|
||||
.text()
|
||||
.await
|
||||
.expect("Failed to read GetBucketRequestPayment response body");
|
||||
let payment_raw =
|
||||
execute_s3_awscurl("GET", &format!("{}/{bucket}?requestPayment=", env.url), &env.access_key, &env.secret_key)
|
||||
.expect("GetBucketRequestPayment HTTP request failed");
|
||||
assert_eq!(parse_status(&payment_raw), Some(200), "GetBucketRequestPayment should return 200");
|
||||
let payment_body = parse_body(&payment_raw);
|
||||
assert!(
|
||||
payment_body.contains("<Payer>BucketOwner</Payer>"),
|
||||
"GetBucketRequestPayment should return BucketOwner payer, got: {payment_body}"
|
||||
);
|
||||
|
||||
let website_response = signed_s3_request(
|
||||
Method::GET,
|
||||
&format!("{}/{bucket}?website=", env.url),
|
||||
None,
|
||||
None,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await
|
||||
.expect("GetBucketWebsite HTTP request failed");
|
||||
let website_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
|
||||
.expect("GetBucketWebsite HTTP request failed");
|
||||
assert_eq!(
|
||||
website_response.status(),
|
||||
404,
|
||||
parse_status(&website_raw),
|
||||
Some(404),
|
||||
"GetBucketWebsite should return 404 when website config is absent"
|
||||
);
|
||||
let website_content_type = website_response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.expect("GetBucketWebsite response should include Content-Type")
|
||||
.to_str()
|
||||
.expect("GetBucketWebsite Content-Type should be valid ASCII")
|
||||
.to_ascii_lowercase();
|
||||
let website_content_type = parse_headers(&website_raw).to_ascii_lowercase();
|
||||
assert!(
|
||||
website_content_type.contains("xml"),
|
||||
website_content_type.contains("content-type:") && website_content_type.contains("xml"),
|
||||
"GetBucketWebsite error response should be XML, got content-type: {website_content_type}"
|
||||
);
|
||||
let website_body = website_response
|
||||
.text()
|
||||
.await
|
||||
.expect("Failed to read GetBucketWebsite response body");
|
||||
let website_body = parse_body(&website_raw);
|
||||
assert!(
|
||||
website_body.contains("<Code>NoSuchWebsiteConfiguration</Code>"),
|
||||
"GetBucketWebsite should return NoSuchWebsiteConfiguration code, got: {website_body}"
|
||||
);
|
||||
|
||||
let delete_response = signed_s3_request(
|
||||
Method::DELETE,
|
||||
&format!("{}/{bucket}?website=", env.url),
|
||||
None,
|
||||
None,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await
|
||||
.expect("DeleteBucketWebsite HTTP request failed");
|
||||
assert_eq!(delete_response.status(), 204, "DeleteBucketWebsite should return 204");
|
||||
let delete_raw =
|
||||
execute_s3_awscurl("DELETE", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
|
||||
.expect("DeleteBucketWebsite HTTP request failed");
|
||||
assert_eq!(parse_status(&delete_raw), Some(204), "DeleteBucketWebsite should return 204");
|
||||
|
||||
env.stop_server();
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, awscurl_get, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, awscurl_get, init_logging};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use rustfs_data_usage::DataUsageInfo;
|
||||
@@ -65,7 +65,7 @@ mod tests {
|
||||
info!("RT-09: bucket object count updates after PUT");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV)
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
@@ -88,21 +88,12 @@ mod tests {
|
||||
|
||||
// Wait for scanner to process (up to 90 seconds)
|
||||
let mut found_nonzero = false;
|
||||
let mut last_query_error = None;
|
||||
for attempt in 0..18 {
|
||||
sleep(Duration::from_secs(5)).await;
|
||||
|
||||
let usage = match get_data_usage(&env).await {
|
||||
Ok(usage) => {
|
||||
last_query_error = None;
|
||||
usage
|
||||
}
|
||||
Err(err) => {
|
||||
last_query_error = Some(err.to_string());
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(bucket_usage) = usage.buckets_usage.get(bucket) {
|
||||
if let Ok(usage) = get_data_usage(&env).await
|
||||
&& let Some(bucket_usage) = usage.buckets_usage.get(bucket)
|
||||
{
|
||||
info!(" attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
|
||||
if bucket_usage.objects_count >= 10 {
|
||||
found_nonzero = true;
|
||||
@@ -113,8 +104,7 @@ mod tests {
|
||||
|
||||
assert!(
|
||||
found_nonzero,
|
||||
"RT-09 FAIL: bucket object count did not update after PUT 10 objects (regression: stats stuck at 0); last query error: {}",
|
||||
last_query_error.as_deref().unwrap_or("none")
|
||||
"RT-09 FAIL: bucket object count did not update after PUT 10 objects (regression: stats stuck at 0)"
|
||||
);
|
||||
|
||||
info!("RT-09 PASS: bucket object count updates after PUT");
|
||||
@@ -132,7 +122,7 @@ mod tests {
|
||||
info!("RT-09b: bucket object count updates after DELETE");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV)
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
@@ -153,22 +143,6 @@ mod tests {
|
||||
.expect("put object");
|
||||
}
|
||||
|
||||
let mut found_nonzero = false;
|
||||
for attempt in 0..18 {
|
||||
sleep(Duration::from_secs(5)).await;
|
||||
|
||||
if let Ok(usage) = get_data_usage(&env).await
|
||||
&& let Some(bucket_usage) = usage.buckets_usage.get(bucket)
|
||||
{
|
||||
info!(" baseline attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
|
||||
if bucket_usage.objects_count >= 5 {
|
||||
found_nonzero = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(found_nonzero, "RT-09b setup failed: scanner did not observe the 5 uploaded objects");
|
||||
|
||||
// Delete all objects
|
||||
for i in 0..5 {
|
||||
client
|
||||
@@ -182,21 +156,12 @@ mod tests {
|
||||
|
||||
// Wait for scanner to update stats (up to 90 seconds)
|
||||
let mut found_zero = false;
|
||||
let mut last_query_error = None;
|
||||
for attempt in 0..18 {
|
||||
sleep(Duration::from_secs(5)).await;
|
||||
|
||||
let usage = match get_data_usage(&env).await {
|
||||
Ok(usage) => {
|
||||
last_query_error = None;
|
||||
usage
|
||||
}
|
||||
Err(err) => {
|
||||
last_query_error = Some(err.to_string());
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(bucket_usage) = usage.buckets_usage.get(bucket) {
|
||||
if let Ok(usage) = get_data_usage(&env).await
|
||||
&& let Some(bucket_usage) = usage.buckets_usage.get(bucket)
|
||||
{
|
||||
info!(" attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
|
||||
if bucket_usage.objects_count == 0 {
|
||||
found_zero = true;
|
||||
@@ -207,8 +172,7 @@ mod tests {
|
||||
|
||||
assert!(
|
||||
found_zero,
|
||||
"RT-09b FAIL: bucket object count did not update to 0 after deleting all objects (regression rustfs#5615); last query error: {}",
|
||||
last_query_error.as_deref().unwrap_or("none")
|
||||
"RT-09b FAIL: bucket object count did not update to 0 after deleting all objects (regression rustfs#5615)"
|
||||
);
|
||||
|
||||
info!("RT-09b PASS: bucket object count updates to 0 after DELETE");
|
||||
|
||||
@@ -47,8 +47,6 @@ use walkdir::WalkDir;
|
||||
pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
|
||||
pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
|
||||
pub const ENV_RUSTFS_BUILD_FEATURES: &str = "RUSTFS_BUILD_FEATURES";
|
||||
pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
|
||||
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
|
||||
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
||||
const RUSTFS_FULL_FEATURE: &str = "full";
|
||||
|
||||
@@ -67,14 +65,8 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
pub(crate) fn build_test_s3_config(
|
||||
endpoint_url: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
session_token: Option<&str>,
|
||||
provider_name: &'static str,
|
||||
) -> Config {
|
||||
let credentials = Credentials::new(access_key, secret_key, session_token.map(str::to_owned), None, provider_name);
|
||||
fn build_test_s3_config(endpoint_url: &str, access_key: &str, secret_key: &str, provider_name: &'static str) -> Config {
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, provider_name);
|
||||
let mut config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
@@ -89,33 +81,6 @@ pub(crate) fn build_test_s3_config(
|
||||
config.build()
|
||||
}
|
||||
|
||||
pub(crate) fn build_test_sts_client(
|
||||
endpoint_url: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
session_token: Option<&str>,
|
||||
provider_name: &'static str,
|
||||
) -> aws_sdk_sts::Client {
|
||||
let mut config = aws_sdk_sts::Config::builder()
|
||||
.credentials_provider(aws_sdk_sts::config::Credentials::new(
|
||||
access_key,
|
||||
secret_key,
|
||||
session_token.map(str::to_owned),
|
||||
None,
|
||||
provider_name,
|
||||
))
|
||||
.region(aws_sdk_sts::config::Region::new("us-east-1"))
|
||||
.endpoint_url(endpoint_url)
|
||||
.retry_config(aws_sdk_sts::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.behavior_version_latest();
|
||||
|
||||
if endpoint_url.starts_with("http://") {
|
||||
config = config.http_client(SmithyHttpClientBuilder::new().build_http());
|
||||
}
|
||||
|
||||
aws_sdk_sts::Client::from_conf(config.build())
|
||||
}
|
||||
|
||||
pub fn workspace_root() -> PathBuf {
|
||||
let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||
path.pop(); // e2e_test
|
||||
@@ -130,38 +95,6 @@ pub fn local_http_client() -> HttpClient {
|
||||
.expect("failed to build local reqwest client")
|
||||
}
|
||||
|
||||
pub(crate) async fn signed_s3_request(
|
||||
method: http::Method,
|
||||
url: &str,
|
||||
body: Option<String>,
|
||||
content_type: Option<&str>,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string();
|
||||
let mut request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
if let Some(content_type) = content_type {
|
||||
request = request.header(CONTENT_TYPE, content_type);
|
||||
}
|
||||
|
||||
let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?;
|
||||
let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let mut request = local_http_client().request(method, url);
|
||||
for (name, value) in signed.headers() {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
if let Some(body) = body {
|
||||
request = request.body(body);
|
||||
}
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
/// Signs and sends an admin HTTP request with the given credentials.
|
||||
pub(crate) async fn admin_request(
|
||||
base_url: &str,
|
||||
@@ -172,8 +105,28 @@ pub(crate) async fn admin_request(
|
||||
secret_key: &str,
|
||||
) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{base_url}{path_and_query}");
|
||||
let content_type = body.as_ref().map(|_| "application/json");
|
||||
let response = signed_s3_request(method, &url, body, content_type, access_key, secret_key).await?;
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
|
||||
let mut request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
if body.is_some() {
|
||||
request = request.header(CONTENT_TYPE, "application/json");
|
||||
}
|
||||
|
||||
let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "admin request body is too large")?;
|
||||
let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let mut request = local_http_client().request(method, &url);
|
||||
for (name, value) in signed.headers() {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
if let Some(body) = body {
|
||||
request = request.body(body);
|
||||
}
|
||||
let response = request.send().await?;
|
||||
let status = response.status();
|
||||
let body = response.text().await?;
|
||||
Ok((status, body))
|
||||
@@ -611,12 +564,7 @@ impl RustFSTestEnvironment {
|
||||
|
||||
/// Create an AWS S3 client configured for this RustFS instance
|
||||
pub fn create_s3_client(&self) -> Client {
|
||||
self.create_s3_client_with_credentials(&self.access_key, &self.secret_key)
|
||||
}
|
||||
|
||||
/// Create an AWS S3 client with explicit credentials for this RustFS instance.
|
||||
pub fn create_s3_client_with_credentials(&self, access_key: &str, secret_key: &str) -> Client {
|
||||
Client::from_conf(build_test_s3_config(&self.url, access_key, secret_key, None, "e2e-test"))
|
||||
Client::from_conf(build_test_s3_config(&self.url, &self.access_key, &self.secret_key, "e2e-test"))
|
||||
}
|
||||
|
||||
/// Create test bucket
|
||||
@@ -1348,7 +1296,6 @@ impl RustFSTestClusterEnvironment {
|
||||
&self.nodes[node_idx].url,
|
||||
&self.access_key,
|
||||
&self.secret_key,
|
||||
None,
|
||||
"cluster-test",
|
||||
)))
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ use rustfs_data_usage::DataUsageInfo;
|
||||
use serial_test::serial;
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging};
|
||||
|
||||
async fn get_data_usage_info(env: &RustFSTestEnvironment) -> Result<DataUsageInfo, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/datausageinfo", env.url);
|
||||
@@ -35,26 +35,16 @@ where
|
||||
F: FnMut(&DataUsageInfo) -> bool,
|
||||
{
|
||||
let mut last_usage = DataUsageInfo::default();
|
||||
let mut last_query_error = None;
|
||||
for _ in 0..45 {
|
||||
match get_data_usage_info(env).await {
|
||||
Ok(usage) => {
|
||||
last_query_error = None;
|
||||
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
|
||||
return Ok(usage);
|
||||
}
|
||||
last_usage = usage;
|
||||
}
|
||||
Err(err) => last_query_error = Some(err.to_string()),
|
||||
let usage = get_data_usage_info(env).await?;
|
||||
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
|
||||
return Ok(usage);
|
||||
}
|
||||
last_usage = usage;
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
|
||||
Err(format!(
|
||||
"bucket usage did not converge for {bucket}; last usage: {last_usage:?}; last query error: {}",
|
||||
last_query_error.as_deref().unwrap_or("none")
|
||||
)
|
||||
.into())
|
||||
Err(format!("bucket usage did not converge for {bucket}; last usage: {last_usage:?}").into())
|
||||
}
|
||||
|
||||
/// Regression test for data usage accuracy (issue #1012).
|
||||
@@ -66,7 +56,7 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV).await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
|
||||
@@ -84,14 +74,8 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
|
||||
.await?;
|
||||
}
|
||||
|
||||
let usage = wait_for_bucket_usage(&env, TEST_BUCKET, |usage| {
|
||||
usage
|
||||
.buckets_usage
|
||||
.get(TEST_BUCKET)
|
||||
.map(|bucket_usage| usage.objects_total_count >= 1000 && bucket_usage.objects_count >= 1000)
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.await?;
|
||||
// Query admin data usage API
|
||||
let usage = get_data_usage_info(&env).await?;
|
||||
|
||||
// Assert total object count and per-bucket count are not truncated
|
||||
let bucket_usage = usage
|
||||
@@ -124,7 +108,7 @@ async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(),
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV).await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "data-usage-versioned";
|
||||
@@ -200,8 +184,8 @@ async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(),
|
||||
assert_eq!(usage.versions_total_count, 3, "total version count should match bucket usage");
|
||||
assert_eq!(usage.delete_markers_total_count, 1, "total delete marker count should match bucket usage");
|
||||
|
||||
env.restart_server_preserving_data(vec![], FAST_DATA_USAGE_SCANNER_ENV)
|
||||
.await?;
|
||||
env.stop_server();
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let restarted_usage = wait_for_bucket_usage(&env, bucket, |usage| {
|
||||
usage
|
||||
|
||||
@@ -574,8 +574,7 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
|
||||
(&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
|
||||
(&Method::POST, true) if upload_id.is_some() => Operation::CompleteMultipartUpload,
|
||||
(&Method::DELETE, true) if upload_id.is_some() => Operation::AbortMultipartUpload,
|
||||
// A replication PUT addresses the source version via `?versionId=`.
|
||||
(&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject,
|
||||
(&Method::PUT, true) if only_query_keys(&[]) => Operation::PutObject,
|
||||
(&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject,
|
||||
(&Method::HEAD, true) if only_query_keys(&["versionId"]) => Operation::HeadObject,
|
||||
(&Method::DELETE, true) if only_query_keys(&["versionId"]) => Operation::DeleteObject,
|
||||
|
||||
@@ -2211,6 +2211,11 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
||||
"queue_snapshot.{field} must be readable in terminal status: {terminal}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
cold_tier_object_count(&cold_client).await? < 64,
|
||||
"queue pressure should leave at least one object untransitioned"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -21,12 +21,9 @@
|
||||
|
||||
use super::common::LocalKMSTestEnvironment;
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ServerSideEncryption,
|
||||
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
};
|
||||
use rustfs_rio::{Checksum, ChecksumType};
|
||||
use serial_test::serial;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
@@ -276,7 +273,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err
|
||||
/// Test 3: When bucket is configured with default encryption, create_multipart_upload should inherit the configuration
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
async fn test_bucket_default_encryption_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Testing bucket default encryption impact on create_multipart_upload");
|
||||
|
||||
@@ -312,16 +309,15 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
.await
|
||||
.expect("Failed to set bucket encryption");
|
||||
|
||||
// Step 2: Declare CRC32 without specifying encryption parameters. The AWS SDK
|
||||
// calculates each UploadPart checksum and sends it as a flexible checksum.
|
||||
info!("Creating CRC32 multipart upload that should use bucket default encryption");
|
||||
let test_key = "test-multipart-bucket-default-crc32.bin";
|
||||
// Step 2: Create multipart upload (without specifying encryption parameters)
|
||||
info!("Creating multipart upload (without specifying encryption parameters, should use bucket default configuration)");
|
||||
let test_key = "test-multipart-bucket-default.txt";
|
||||
|
||||
let create_multipart_response = s3_client
|
||||
.create_multipart_upload()
|
||||
.bucket(TEST_BUCKET)
|
||||
.key(test_key)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
// Note: No encryption parameters specified here, should use bucket default configuration
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to create multipart upload");
|
||||
@@ -347,61 +343,28 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
"create_multipart_upload response should contain correct KMS key ID"
|
||||
);
|
||||
|
||||
// Step 3: Upload two parts. The first is exactly the S3 minimum size so this
|
||||
// follows the same managed SSE-KMS multipart path as issue #5756.
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
let part1: Vec<u8> = (0..PART_SIZE).map(|i| (i % 251) as u8).collect();
|
||||
let part2: Vec<u8> = (0..1024 * 1024).map(|i| ((i + 17) % 251) as u8).collect();
|
||||
let expected_body: Vec<u8> = part1.iter().chain(&part2).copied().collect();
|
||||
// Step 3: Upload a part and complete multipart upload
|
||||
info!("Uploading part and completing multipart upload");
|
||||
let test_data = b"test-multipart-bucket-default-encryption-data";
|
||||
|
||||
let upload_part = |part_number: i32, body: Vec<u8>| {
|
||||
s3_client
|
||||
.upload_part()
|
||||
.bucket(TEST_BUCKET)
|
||||
.key(test_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(part_number)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
.body(ByteStream::from(body))
|
||||
.send()
|
||||
};
|
||||
// Upload part 1
|
||||
let upload_part_response = s3_client
|
||||
.upload_part()
|
||||
.bucket(TEST_BUCKET)
|
||||
.key(test_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(test_data.to_vec().into())
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to upload part");
|
||||
|
||||
let expected_part1_crc32 = Checksum::new_from_data(ChecksumType::CRC32, &part1)
|
||||
.expect("calculate part 1 CRC32")
|
||||
.encoded;
|
||||
let upload1 = upload_part(1, part1).await.expect("Failed to upload part 1 with CRC32");
|
||||
assert_eq!(
|
||||
upload1.checksum_crc32(),
|
||||
Some(expected_part1_crc32.as_str()),
|
||||
"UploadPart must return the CRC32 calculated over plaintext"
|
||||
);
|
||||
|
||||
let expected_part2_crc32 = Checksum::new_from_data(ChecksumType::CRC32, &part2)
|
||||
.expect("calculate part 2 CRC32")
|
||||
.encoded;
|
||||
let upload2 = upload_part(2, part2).await.expect("Failed to upload part 2 with CRC32");
|
||||
assert_eq!(
|
||||
upload2.checksum_crc32(),
|
||||
Some(expected_part2_crc32.as_str()),
|
||||
"UploadPart must return the CRC32 calculated over plaintext"
|
||||
);
|
||||
let etag = upload_part_response.e_tag().unwrap().to_string();
|
||||
|
||||
// Complete multipart upload
|
||||
let completed_upload = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(upload1.e_tag().expect("No ETag for part 1"))
|
||||
.checksum_crc32(upload1.checksum_crc32().expect("No CRC32 for part 1"))
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(upload2.e_tag().expect("No ETag for part 2"))
|
||||
.checksum_crc32(upload2.checksum_crc32().expect("No CRC32 for part 2"))
|
||||
.build(),
|
||||
)
|
||||
let completed_part = aws_sdk_s3::types::CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(&etag)
|
||||
.build();
|
||||
|
||||
let complete_multipart_response = s3_client
|
||||
@@ -409,7 +372,11 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
.bucket(TEST_BUCKET)
|
||||
.key(test_key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(completed_upload)
|
||||
.multipart_upload(
|
||||
aws_sdk_s3::types::CompletedMultipartUpload::builder()
|
||||
.parts(completed_part)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to complete multipart upload");
|
||||
@@ -433,7 +400,6 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
.get_object()
|
||||
.bucket(TEST_BUCKET)
|
||||
.key(test_key)
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to get object");
|
||||
@@ -444,13 +410,6 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
Some(&ServerSideEncryption::AwsKms),
|
||||
"Final object should contain SSE-KMS encryption information"
|
||||
);
|
||||
if let Some(completed_crc32) = complete_multipart_response.checksum_crc32() {
|
||||
assert_eq!(
|
||||
get_response.checksum_crc32(),
|
||||
Some(completed_crc32),
|
||||
"GetObject should return the persisted composite CRC32 when completion reports it"
|
||||
);
|
||||
}
|
||||
|
||||
// Verify data integrity
|
||||
let downloaded_data = get_response
|
||||
@@ -459,11 +418,7 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
.await
|
||||
.expect("Failed to collect body")
|
||||
.into_bytes();
|
||||
assert_eq!(
|
||||
downloaded_data.as_ref(),
|
||||
expected_body.as_slice(),
|
||||
"Downloaded data should match the uploaded multipart body"
|
||||
);
|
||||
assert_eq!(&downloaded_data[..], test_data, "Downloaded data should match original data");
|
||||
|
||||
// Cleanup is handled automatically when the test environment is dropped
|
||||
info!("Test passed: bucket default encryption correctly applied to multipart upload");
|
||||
|
||||
@@ -290,14 +290,6 @@ mod overwrite_cleanup_regression_test;
|
||||
#[cfg(test)]
|
||||
mod list_buckets_double_slash_test;
|
||||
|
||||
// Regression coverage for bucket-scoped ListBuckets authorization fallback.
|
||||
#[cfg(test)]
|
||||
mod list_buckets_auth_test;
|
||||
|
||||
// ListBuckets visibility follows IAM authorization, not bucket policy.
|
||||
#[cfg(test)]
|
||||
mod list_buckets_iam_filter_test;
|
||||
|
||||
// Regression test for backlog#629(b): region-aware CreateBucket SigV4.
|
||||
#[cfg(test)]
|
||||
mod create_bucket_region_test;
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression coverage for the MinIO-compatible filtered ListBuckets fallback.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
|
||||
use std::error::Error;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_scoped_policy_returns_only_authorized_bucket() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let root_client = env.create_s3_client();
|
||||
let allowed_bucket = "list-buckets-authorized";
|
||||
let hidden_bucket = "list-buckets-hidden";
|
||||
let user = "listbucketsuser";
|
||||
let secret = "listbucketssecret";
|
||||
let policy = "list-buckets-scoped";
|
||||
|
||||
root_client.create_bucket().bucket(allowed_bucket).send().await?;
|
||||
root_client.create_bucket().bucket(hidden_bucket).send().await?;
|
||||
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={policy}"),
|
||||
Some(
|
||||
serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{allowed_bucket}"),
|
||||
format!("arn:aws:s3:::{allowed_bucket}/*")
|
||||
]
|
||||
}]
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={user}"),
|
||||
Some(serde_json::json!({ "secretKey": secret, "status": "enabled" }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::POST,
|
||||
"/rustfs/admin/v3/idp/builtin/policy/attach",
|
||||
Some(serde_json::json!({ "policies": [policy], "user": user }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let client = env.create_s3_client_with_credentials(user, secret);
|
||||
// Capture ListBuckets first so the direct-access control cannot warm bucket metadata and mask the regression.
|
||||
let listed = client.list_buckets().send().await;
|
||||
client.list_objects_v2().bucket(allowed_bucket).send().await?;
|
||||
|
||||
let listed = listed?;
|
||||
let names = listed
|
||||
.buckets()
|
||||
.iter()
|
||||
.filter_map(|bucket| bucket.name().map(ToOwned::to_owned))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(names, vec![allowed_bucket]);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,459 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, build_test_s3_config, build_test_sts_client, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use serial_test::serial;
|
||||
use tokio::time::{Duration, Instant};
|
||||
|
||||
fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
|
||||
Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
access_key,
|
||||
secret_key,
|
||||
session_token,
|
||||
"list-buckets-iam-filter",
|
||||
))
|
||||
}
|
||||
|
||||
fn bucket_names(buckets: &[aws_sdk_s3::types::Bucket]) -> Vec<String> {
|
||||
let mut names = buckets
|
||||
.iter()
|
||||
.filter_map(|bucket| bucket.name().map(str::to_owned))
|
||||
.collect::<Vec<_>>();
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
async fn create_user(
|
||||
env: &RustFSTestEnvironment,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let body = serde_json::json!({ "secretKey": secret_key, "status": "enabled" }).to_string();
|
||||
admin_ok(
|
||||
env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={access_key}"),
|
||||
Some(body),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn create_service_account(
|
||||
env: &RustFSTestEnvironment,
|
||||
target_user: &str,
|
||||
policy: Option<&serde_json::Value>,
|
||||
) -> Result<(String, String), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let request = match policy {
|
||||
Some(policy) => serde_json::json!({ "targetUser": target_user, "policy": policy }),
|
||||
None => serde_json::json!({ "targetUser": target_user }),
|
||||
};
|
||||
let response = admin_ok(env, http::Method::PUT, "/rustfs/admin/v3/add-service-accounts", Some(request.to_string())).await?;
|
||||
let response: serde_json::Value = serde_json::from_str(&response)?;
|
||||
let access_key = response["credentials"]["accessKey"]
|
||||
.as_str()
|
||||
.ok_or("service account response should contain credentials.accessKey")?
|
||||
.to_owned();
|
||||
let secret_key = response["credentials"]["secretKey"]
|
||||
.as_str()
|
||||
.ok_or("service account response should contain credentials.secretKey")?
|
||||
.to_owned();
|
||||
Ok((access_key, secret_key))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn list_buckets_filters_with_iam_bucket_resources() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.capture_log_path = Some(format!("{}/server.log", env.temp_dir));
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUST_LOG", "rustfs=debug,rustfs_notify=debug")])
|
||||
.await?;
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
for bucket in [
|
||||
"benchmark-artifacts",
|
||||
"benchmark-denied",
|
||||
"benchmark-location-only",
|
||||
"benchmark-test1",
|
||||
"testuser1-artifacts",
|
||||
] {
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
}
|
||||
assert_eq!(
|
||||
bucket_names(admin_client.list_buckets().send().await?.buckets()),
|
||||
vec![
|
||||
"benchmark-artifacts",
|
||||
"benchmark-denied",
|
||||
"benchmark-location-only",
|
||||
"benchmark-test1",
|
||||
"testuser1-artifacts"
|
||||
]
|
||||
);
|
||||
|
||||
let access_key = "benchmark";
|
||||
let secret_key = "benchmark-secret-1234567890";
|
||||
create_user(&env, access_key, secret_key).await?;
|
||||
|
||||
let policy_name = "benchmark-bucket-prefix";
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-*", "arn:aws:s3:::benchmark-*/*"],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:prefix": [""],
|
||||
"s3:delimiter": ["/"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-denied"]
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-location-only"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["sts:AssumeRole"],
|
||||
"Resource": ["arn:aws:s3:::*"]
|
||||
}
|
||||
]
|
||||
})
|
||||
.to_string();
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
|
||||
Some(policy),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={access_key}&isGroup=false"),
|
||||
Some(String::new()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let bucket_policy_allow = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [access_key] },
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::testuser1-artifacts"]
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_client
|
||||
.put_bucket_policy()
|
||||
.bucket("testuser1-artifacts")
|
||||
.policy(bucket_policy_allow)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let bucket_policy_deny = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Deny",
|
||||
"Principal": { "AWS": [access_key] },
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-artifacts"]
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_client
|
||||
.put_bucket_policy()
|
||||
.bucket("benchmark-artifacts")
|
||||
.policy(bucket_policy_deny)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let benchmark_client = user_client(&env, access_key, secret_key, None);
|
||||
benchmark_client
|
||||
.list_objects_v2()
|
||||
.bucket("testuser1-artifacts")
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
bucket_names(benchmark_client.list_buckets().send().await?.buckets()),
|
||||
vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
|
||||
);
|
||||
let log_path = env.capture_log_path.as_deref().expect("server log path should be configured");
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
let audit_log = loop {
|
||||
let audit_log = tokio::fs::read_to_string(log_path).await?;
|
||||
if [
|
||||
"iam_implicit_deny",
|
||||
"s3_authorization_denied",
|
||||
"ListAllMyBucketsAction",
|
||||
"benchmark",
|
||||
"DEBUG",
|
||||
]
|
||||
.iter()
|
||||
.all(|field| audit_log.contains(field))
|
||||
|| Instant::now() >= deadline
|
||||
{
|
||||
break audit_log;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
};
|
||||
assert_eq!(audit_log.matches("iam_implicit_deny").count(), 1, "{audit_log}");
|
||||
for field in ["s3_authorization_denied", "ListAllMyBucketsAction", "benchmark", "DEBUG"] {
|
||||
assert!(audit_log.contains(field), "missing {field} in {audit_log}");
|
||||
}
|
||||
|
||||
let denied_access_key = "no-bucket-access";
|
||||
let denied_secret_key = "no-bucket-access-secret-1234567890";
|
||||
create_user(&env, denied_access_key, denied_secret_key).await?;
|
||||
let denied = user_client(&env, denied_access_key, denied_secret_key, None)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a user without IAM bucket permissions must be denied");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
|
||||
let put_only_policy_name = "put-only-no-bucket-discovery";
|
||||
let put_only_policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-*/*"]
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={put_only_policy_name}"),
|
||||
Some(put_only_policy),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!(
|
||||
"/rustfs/admin/v3/set-user-or-group-policy?policyName={put_only_policy_name}&userOrGroup={denied_access_key}&isGroup=false"
|
||||
),
|
||||
Some(String::new()),
|
||||
)
|
||||
.await?;
|
||||
let denied = user_client(&env, denied_access_key, denied_secret_key, None)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an unrelated IAM action must not reveal bucket names");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
|
||||
let list_all_policy_name = "list-all-buckets";
|
||||
let list_all_policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListAllMyBuckets"],
|
||||
"Resource": ["arn:aws:s3:::*"]
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={list_all_policy_name}"),
|
||||
Some(list_all_policy),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!(
|
||||
"/rustfs/admin/v3/set-user-or-group-policy?policyName={list_all_policy_name}&userOrGroup={denied_access_key}&isGroup=false"
|
||||
),
|
||||
Some(String::new()),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
bucket_names(
|
||||
user_client(&env, denied_access_key, denied_secret_key, None)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
),
|
||||
vec![
|
||||
"benchmark-artifacts",
|
||||
"benchmark-denied",
|
||||
"benchmark-location-only",
|
||||
"benchmark-test1",
|
||||
"testuser1-artifacts"
|
||||
]
|
||||
);
|
||||
|
||||
let group_user = "benchmark-group-user";
|
||||
let group_secret = "benchmark-group-secret-1234567890";
|
||||
let group_name = "benchmark-group";
|
||||
create_user(&env, group_user, group_secret).await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/update-group-members",
|
||||
Some(
|
||||
serde_json::json!({
|
||||
"group": group_name,
|
||||
"members": [group_user],
|
||||
"isRemove": false,
|
||||
"groupStatus": "enabled"
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
|
||||
Some(String::new()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
bucket_names(
|
||||
user_client(&env, group_user, group_secret, None)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
),
|
||||
vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
|
||||
);
|
||||
|
||||
let (service_access_key, service_secret_key) = create_service_account(&env, group_user, None).await?;
|
||||
assert_eq!(
|
||||
bucket_names(
|
||||
user_client(&env, &service_access_key, &service_secret_key, None)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
),
|
||||
vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
|
||||
);
|
||||
|
||||
let service_account_policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-test1"],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:prefix": [""],
|
||||
"s3:delimiter": ["/"]
|
||||
}
|
||||
}
|
||||
}]
|
||||
});
|
||||
let (restricted_service_access_key, restricted_service_secret_key) =
|
||||
create_service_account(&env, group_user, Some(&service_account_policy)).await?;
|
||||
assert_eq!(
|
||||
bucket_names(
|
||||
user_client(&env, &restricted_service_access_key, &restricted_service_secret_key, None,)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
),
|
||||
vec!["benchmark-test1"]
|
||||
);
|
||||
|
||||
let sts_client = build_test_sts_client(&env.url, group_user, group_secret, None, "list-buckets-iam-filter-sts");
|
||||
let inherited = sts_client
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/list-buckets")
|
||||
.role_session_name("list-buckets-iam-filter-inherited")
|
||||
.send()
|
||||
.await?;
|
||||
let inherited = inherited
|
||||
.credentials()
|
||||
.ok_or("AssumeRole response should contain inherited temporary credentials")?;
|
||||
assert_eq!(
|
||||
bucket_names(
|
||||
user_client(
|
||||
&env,
|
||||
inherited.access_key_id(),
|
||||
inherited.secret_access_key(),
|
||||
Some(inherited.session_token()),
|
||||
)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
),
|
||||
vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
|
||||
);
|
||||
|
||||
let session_policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-test1"],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:prefix": [""],
|
||||
"s3:delimiter": ["/"]
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
let assumed = sts_client
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/list-buckets")
|
||||
.role_session_name("list-buckets-iam-filter")
|
||||
.policy(session_policy)
|
||||
.send()
|
||||
.await?;
|
||||
let temporary = assumed
|
||||
.credentials()
|
||||
.ok_or("AssumeRole response should contain temporary credentials")?;
|
||||
assert_eq!(
|
||||
bucket_names(
|
||||
user_client(
|
||||
&env,
|
||||
temporary.access_key_id(),
|
||||
temporary.secret_access_key(),
|
||||
Some(temporary.session_token()),
|
||||
)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
),
|
||||
vec!["benchmark-test1"]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -6028,33 +6028,6 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
}
|
||||
|
||||
let version_condition_client = restricted_user_client(&env, version_condition_user, version_condition_secret);
|
||||
let mismatching_version_pax = HashMap::from([("minio.versionId", Uuid::new_v4().to_string())]);
|
||||
let archive = make_tar_with_pax_entry("version-mismatch-entry.txt", b"must-not-write", None, &mismatching_version_pax).await;
|
||||
let err = version_condition_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("version-mismatch.tar")
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a mismatching PAX version ID must fail the replication condition");
|
||||
assert_eq!(err.as_service_error().and_then(|error| error.meta().code()), Some("AccessDenied"));
|
||||
let err = admin_client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key("version-mismatch-entry.txt")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a denied PAX entry must not be written");
|
||||
assert!(matches!(
|
||||
err.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("NoSuchKey" | "NotFound")
|
||||
));
|
||||
|
||||
let matching_version_pax = HashMap::from([("minio.versionId", conditional_version_id)]);
|
||||
let archive = make_tar_with_pax_entry("condition-entry.txt", b"condition-body", None, &matching_version_pax).await;
|
||||
version_condition_client
|
||||
@@ -6068,13 +6041,6 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
let stored = admin_client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("condition-entry.txt")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), b"condition-body");
|
||||
|
||||
let pax_context_client = restricted_user_client(&env, pax_context_user, pax_context_secret);
|
||||
let tag_pax = HashMap::from([("minio.metadata.x-amz-tagging", "classification=public".to_string())]);
|
||||
|
||||
@@ -12,11 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_post, awscurl_put, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_get, awscurl_post, awscurl_put, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use http::{Method, StatusCode};
|
||||
use serial_test::serial;
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
use tracing::{debug, info};
|
||||
|
||||
fn skip_without_awscurl() -> bool {
|
||||
@@ -39,8 +37,7 @@ impl QuotaTestEnv {
|
||||
pub async fn new() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let bucket_name = format!("quota-test-{}", uuid::Uuid::new_v4());
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")])
|
||||
.await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let client = env.create_s3_client();
|
||||
|
||||
Ok(Self {
|
||||
@@ -70,7 +67,18 @@ impl QuotaTestEnv {
|
||||
}
|
||||
|
||||
pub async fn set_bucket_quota(&self, quota_bytes: u64) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.set_bucket_quota_for(&self.bucket_name, quota_bytes).await
|
||||
let url = format!("{}/rustfs/admin/v3/quota/{}", self.env.url, self.bucket_name);
|
||||
let quota_config = serde_json::json!({
|
||||
"quota": quota_bytes,
|
||||
"quota_type": "HARD"
|
||||
});
|
||||
|
||||
let response = awscurl_put(&url, "a_config.to_string(), &self.env.access_key, &self.env.secret_key).await?;
|
||||
if response.contains("error") {
|
||||
Err(format!("Failed to set quota: {}", response).into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_bucket_quota(&self) -> Result<Option<u64>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
@@ -170,29 +178,6 @@ impl QuotaTestEnv {
|
||||
bucket: &str,
|
||||
quota_bytes: u64,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let stats_path = format!("/rustfs/admin/v3/quota-stats/{bucket}");
|
||||
let readiness = async {
|
||||
loop {
|
||||
let (status, response) =
|
||||
admin_request(&self.env.url, Method::GET, &stats_path, None, &self.env.access_key, &self.env.secret_key)
|
||||
.await?;
|
||||
if status.is_success() {
|
||||
return Ok::<(), Box<dyn std::error::Error + Send + Sync>>(());
|
||||
}
|
||||
if status != StatusCode::SERVICE_UNAVAILABLE {
|
||||
return Err(format!("quota usage readiness failed for {bucket}: {status} {response}").into());
|
||||
}
|
||||
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
};
|
||||
match timeout(Duration::from_secs(30), readiness).await {
|
||||
Ok(result) => result?,
|
||||
Err(_) => {
|
||||
return Err(format!("quota usage did not become authoritative for {bucket} within 30 seconds").into());
|
||||
}
|
||||
}
|
||||
|
||||
let url = format!("{}/rustfs/admin/v3/quota/{}", self.env.url, bucket);
|
||||
let quota_config = serde_json::json!({
|
||||
"quota": quota_bytes,
|
||||
|
||||
@@ -16,7 +16,6 @@ use crate::common::{
|
||||
RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
|
||||
replication_fast_env, rustfs_binary_path,
|
||||
};
|
||||
use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, Operation as FakeTargetOperation};
|
||||
use crate::kms::common::{create_key_with_specific_id, sse_customer_key_md5_base64};
|
||||
use crate::storage_api::replication_extension::BucketTargetSys;
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
@@ -363,21 +362,19 @@ impl Drop for SlowReplicationTargetGuard {
|
||||
}
|
||||
}
|
||||
|
||||
// Mirrors madmin-go `ResyncTargetsInfo`/`ResyncTarget` json tags — the same
|
||||
// shape `mc replicate resync status` decodes.
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
struct ReplicationResetStatusResponse {
|
||||
#[serde(rename = "target", default)]
|
||||
#[serde(rename = "Targets", default)]
|
||||
targets: Vec<ReplicationResetStatusTarget>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
struct ReplicationResetStatusTarget {
|
||||
#[serde(rename = "arn", default)]
|
||||
#[serde(rename = "Arn", default)]
|
||||
arn: String,
|
||||
#[serde(rename = "resetid", default)]
|
||||
#[serde(rename = "ResetID", default)]
|
||||
reset_id: String,
|
||||
#[serde(rename = "resyncStatus", default)]
|
||||
#[serde(rename = "Status", default)]
|
||||
status: String,
|
||||
}
|
||||
|
||||
@@ -1657,30 +1654,19 @@ async fn wait_for_source_delete_marker_replication_failed(
|
||||
if response.status() != StatusCode::OK {
|
||||
return Err(format!("replication diff failed with status {}", response.status()).into());
|
||||
}
|
||||
// The default diff response is a madmin-style stream of bare DiffInfo
|
||||
// JSON documents (one per line) with no envelope; assert the envelope
|
||||
// is gone so an aggregate-shaped regression fails loudly here.
|
||||
let body = response.text().await?;
|
||||
let entries = body
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.map(serde_json::from_str::<serde_json::Value>)
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
for entry in &entries {
|
||||
if entry.get("Entries").is_some() {
|
||||
return Err(format!("replication diff must stream bare DiffInfo documents, got envelope: {entry}").into());
|
||||
}
|
||||
}
|
||||
let failed = entries.iter().any(|entry| {
|
||||
entry["object"].as_str() == Some(key)
|
||||
&& entry["deletemarker"].as_bool() == Some(true)
|
||||
&& entry["rStatus"].as_str() == Some("FAILED")
|
||||
let diff: serde_json::Value = response.json().await?;
|
||||
let failed = diff["Entries"].as_array().is_some_and(|entries| {
|
||||
entries.iter().any(|entry| {
|
||||
entry["Object"].as_str() == Some(key)
|
||||
&& entry["IsDeleteMarker"].as_bool() == Some(true)
|
||||
&& entry["ReplicationStatus"].as_str() == Some("FAILED")
|
||||
})
|
||||
});
|
||||
if failed {
|
||||
return Ok(());
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("source delete marker {key} never reported FAILED; last diff={body}").into());
|
||||
return Err(format!("source delete marker {key} never reported FAILED; last diff={diff}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
@@ -2289,30 +2275,6 @@ async fn site_replication_state_edit(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Start a bucket-level replication resync (`PUT ?replication-reset`) and
|
||||
/// return the target `(arn, reset_id)`, asserting the response carries the
|
||||
/// madmin `ResyncTargetsInfo` shape (`target[0].arn` / `target[0].resetid`)
|
||||
/// that `mc replicate resync start` decodes.
|
||||
async fn start_bucket_replication_reset(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
) -> Result<(String, String), Box<dyn Error + Send + Sync>> {
|
||||
let url = format!("{}/{bucket}?replication-reset", env.url);
|
||||
let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, None, None).await?;
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("replication reset start failed: {status} {body}").into());
|
||||
}
|
||||
let payload: serde_json::Value = response.json().await?;
|
||||
let arn = payload["target"][0]["arn"].as_str().unwrap_or_default().to_string();
|
||||
let reset_id = payload["target"][0]["resetid"].as_str().unwrap_or_default().to_string();
|
||||
if arn.is_empty() || reset_id.is_empty() {
|
||||
return Err(format!("replication reset response missing madmin target[0].arn/resetid: {payload}").into());
|
||||
}
|
||||
Ok((arn, reset_id))
|
||||
}
|
||||
|
||||
async fn get_replication_reset_status(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
@@ -2473,107 +2435,6 @@ async fn build_replication_pair(
|
||||
Ok((source_env, target_env, source_bucket.to_string()))
|
||||
}
|
||||
|
||||
/// P0-6: CopyObject creates a new object on the destination key, so it must be
|
||||
/// scheduled for bucket replication exactly like PutObject (MinIO
|
||||
/// CopyObjectHandler parity). Before the fix the copy path never consulted the
|
||||
/// replication config: the destination object stayed local forever (its status
|
||||
/// metadata was inherited wholesale from the source, so the scanner heal pass
|
||||
/// skipped it too — no PENDING/FAILED marker meant nothing to re-drive).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_object_replicates_to_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (source_env, target_env, source_bucket) = build_replication_pair(true).await?;
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
let target_bucket = "replication-check-dst";
|
||||
|
||||
let src_key = "copy-repl-source.txt";
|
||||
let dst_key = "copy-repl-destination.txt";
|
||||
let payload = b"copy object replication payload".to_vec();
|
||||
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(src_key)
|
||||
.body(ByteStream::from(payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(wait_for_object_on_target(&target_client, target_bucket, src_key).await?, payload);
|
||||
// Wait for the source object's terminal COMPLETED status so the copy below
|
||||
// starts from metadata that carries a stale terminal replication state; the
|
||||
// copy must not inherit it (MinIO filterReplicationStatusMetadata parity)
|
||||
// and must drive its own PENDING -> COMPLETED cycle.
|
||||
wait_for_source_replication_status(&source_client, &source_bucket, src_key, "COMPLETED", false).await?;
|
||||
|
||||
source_client
|
||||
.copy_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(dst_key)
|
||||
.copy_source(format!("{source_bucket}/{src_key}"))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
wait_for_object_on_target(&target_client, target_bucket, dst_key).await?,
|
||||
payload,
|
||||
"CopyObject destination must replicate to the remote target"
|
||||
);
|
||||
wait_for_source_replication_status(&source_client, &source_bucket, dst_key, "COMPLETED", false).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// P0-6 companion: snowball auto-extract writes each archive member as an
|
||||
/// independent object; every member must replicate to the remote target like a
|
||||
/// regular PUT (MinIO PutObjectExtract parity).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_snowball_extract_replicates_members_to_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (source_env, target_env, source_bucket) = build_replication_pair(true).await?;
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
let target_bucket = "replication-check-dst";
|
||||
|
||||
let members: [(&str, &[u8]); 2] = [
|
||||
("snowball/member-one.txt", b"first member payload"),
|
||||
("snowball/member-two.txt", b"second member payload"),
|
||||
];
|
||||
|
||||
let mut builder = tokio_tar::Builder::new(std::io::Cursor::new(Vec::new()));
|
||||
for (path, data) in members {
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_size(data.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder.append_data(&mut header, path, std::io::Cursor::new(data)).await?;
|
||||
}
|
||||
let archive = builder.into_inner().await?.into_inner();
|
||||
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key("members.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(archive))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
for (key, data) in members {
|
||||
assert_eq!(
|
||||
wait_for_object_on_target(&target_client, target_bucket, key).await?,
|
||||
data,
|
||||
"snowball-extracted member {key} must replicate to the remote target"
|
||||
);
|
||||
wait_for_source_replication_status(&source_client, &source_bucket, key, "COMPLETED", false).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
@@ -2893,7 +2754,7 @@ async fn test_set_remote_target_update_requires_arn() -> Result<(), Box<dyn Erro
|
||||
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||
assert!(body.contains("InvalidRequest"), "unexpected response: {body}");
|
||||
assert!(body.to_ascii_lowercase().contains("arn is required"), "unexpected response: {body}");
|
||||
assert!(body.to_ascii_lowercase().contains("arn is empty"), "unexpected response: {body}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -2951,128 +2812,6 @@ async fn test_set_remote_target_update_rejects_missing_target() -> Result<(), Bo
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn send_set_replication_target_update_request(
|
||||
source_env: &RustFSTestEnvironment,
|
||||
source_bucket: &str,
|
||||
ops: &[&str],
|
||||
body: serde_json::Value,
|
||||
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
|
||||
let mut url = format!(
|
||||
"{}/rustfs/admin/v3/set-remote-target?bucket={}&update=true",
|
||||
source_env.url,
|
||||
urlencoding::encode(source_bucket)
|
||||
);
|
||||
for op in ops {
|
||||
url.push_str(&format!("&{op}=true"));
|
||||
}
|
||||
signed_request(
|
||||
http::Method::PUT,
|
||||
&url,
|
||||
&source_env.access_key,
|
||||
&source_env.secret_key,
|
||||
Some(body.to_string().into_bytes()),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn fetch_single_target(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
|
||||
let response = list_replication_targets_request(env, Some(bucket)).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let mut targets: Vec<serde_json::Value> = response.json().await?;
|
||||
assert_eq!(targets.len(), 1, "expected exactly one remote target");
|
||||
Ok(targets.remove(0))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_partial_update_preserves_credentials() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
source_env
|
||||
.start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
|
||||
.await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
let source_bucket = "replication-partial-update-src";
|
||||
let target_bucket = "replication-partial-update-dst";
|
||||
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
|
||||
let arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
|
||||
// A sync-only update whose body omits credentials entirely must succeed and
|
||||
// leave the stored connection settings untouched.
|
||||
let response = send_set_replication_target_update_request(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
&["sync"],
|
||||
serde_json::json!({
|
||||
"arn": arn,
|
||||
"type": "replication",
|
||||
"replicationSync": true
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::OK, "sync-only update failed: {}", response.text().await?);
|
||||
|
||||
let target = fetch_single_target(&source_env, source_bucket).await?;
|
||||
assert_eq!(target["replicationSync"], serde_json::json!(true));
|
||||
assert_eq!(target["endpoint"], serde_json::json!(target_env.address));
|
||||
assert_eq!(target["credentials"]["accessKey"], serde_json::json!(target_env.access_key));
|
||||
|
||||
// An update naming no field groups is a no-op: a body carrying a different
|
||||
// endpoint and credentials must not leak into the stored target.
|
||||
let response = send_set_replication_target_update_request(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
&[],
|
||||
serde_json::json!({
|
||||
"arn": arn,
|
||||
"type": "replication",
|
||||
"endpoint": "203.0.113.1:9000",
|
||||
"credentials": { "accessKey": "other-access", "secretKey": "other-secret" },
|
||||
"targetbucket": "elsewhere",
|
||||
"secure": false,
|
||||
"replicationSync": false
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(response.status(), StatusCode::OK, "no-op update failed: {}", response.text().await?);
|
||||
|
||||
let target = fetch_single_target(&source_env, source_bucket).await?;
|
||||
assert_eq!(
|
||||
target["replicationSync"],
|
||||
serde_json::json!(true),
|
||||
"no-op update must not change sync mode"
|
||||
);
|
||||
assert_eq!(
|
||||
target["endpoint"],
|
||||
serde_json::json!(target_env.address),
|
||||
"no-op update must not change endpoint"
|
||||
);
|
||||
assert_eq!(
|
||||
target["credentials"]["accessKey"],
|
||||
serde_json::json!(target_env.access_key),
|
||||
"no-op update must not change credentials"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_set_remote_target_rejects_invalid_target_url() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
@@ -3975,7 +3714,7 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
|
||||
<Role></Role>
|
||||
<Rule>
|
||||
<ID>matrix-prefix</ID>
|
||||
<Priority>110</Priority>
|
||||
<Priority>100</Priority>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix>prefix/</Prefix></Filter>
|
||||
<DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
|
||||
@@ -3986,7 +3725,7 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>matrix-both-prefix</ID>
|
||||
<Priority>120</Priority>
|
||||
<Priority>100</Priority>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix>both/</Prefix></Filter>
|
||||
<DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
|
||||
@@ -3996,7 +3735,7 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>matrix-tag</ID>
|
||||
<Priority>130</Priority>
|
||||
<Priority>100</Priority>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Tag><Key>route</Key><Value>tagged</Value></Tag></Filter>
|
||||
<DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication>
|
||||
@@ -4006,7 +3745,7 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>matrix-disabled</ID>
|
||||
<Priority>140</Priority>
|
||||
<Priority>100</Priority>
|
||||
<Status>Disabled</Status>
|
||||
<Filter><Prefix>disabled/</Prefix></Filter>
|
||||
<DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
|
||||
@@ -4488,76 +4227,16 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult {
|
||||
}
|
||||
|
||||
/// backlog#1147 repl-17 / backlog#1291: SSE-S3 must fail closed until managed
|
||||
/// encryption is supported on the target. The silent plaintext replication
|
||||
/// that originally kept this test ignored was fixed by the fail-closed gate in
|
||||
/// `crates/ecstore/src/bucket/replication/replication_target_boundary.rs`
|
||||
/// (all replication modes route through it), so this now pins the current
|
||||
/// fail-closed contract: FAILED status, failure event, readable source, and a
|
||||
/// stable absence of all target versions.
|
||||
/// encryption is supported on the target. The current plaintext replication is
|
||||
/// a known security bug, so this pins the required contract without blessing it.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
#[ignore = "backlog#1291: SSE-S3 replication silently drops encryption"]
|
||||
async fn test_bucket_replication_sse_s3_contract() -> TestResult {
|
||||
init_logging();
|
||||
assert_managed_sse_replication_fails_explicitly("sse-s3", false).await
|
||||
}
|
||||
|
||||
/// P1-22 stage 0: the existing-object resync path must fail closed for
|
||||
/// managed-SSE objects exactly like inline replication (which
|
||||
/// `test_bucket_replication_sse_s3_contract` pins, including the scanner heal
|
||||
/// re-drive). Resync re-drives every object version through the same
|
||||
/// fail-closed target boundary, so a resync over an encrypted bucket must
|
||||
/// terminate without ever materializing a plaintext (or unreadable) replica;
|
||||
/// the post-resync stays-absent window also spans further fast-scanner heal
|
||||
/// cycles.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_sse_s3_resync_stays_fail_closed() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (source_env, target_env, source_bucket, target_bucket) = build_sse_replication_pair("sse-resync", true).await?;
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
let key = "sse-resync-contract.txt";
|
||||
let body = b"repl-22 sse resync payload".to_vec();
|
||||
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_source_replication_status(&source_client, &source_bucket, key, "FAILED", false).await?;
|
||||
|
||||
// Resync: drive the existing-object resync path over the failed object.
|
||||
let (target_arn, reset_id) = start_bucket_replication_reset(&source_env, &source_bucket).await?;
|
||||
let terminal = wait_for_replication_reset_target(&source_env, &source_bucket, &target_arn, |target| {
|
||||
target.reset_id == reset_id && matches!(target.status.as_str(), "Completed" | "Failed")
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(terminal.reset_id, reset_id);
|
||||
|
||||
// The resync pass must have failed closed: still no target version (the
|
||||
// window also spans further scanner heal cycles), and the source object
|
||||
// stays readable and encrypted.
|
||||
assert_failed_replication_stays_absent_for(
|
||||
&source_client,
|
||||
&source_bucket,
|
||||
&target_client,
|
||||
&target_bucket,
|
||||
key,
|
||||
false,
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await?;
|
||||
let source = source_client.get_object().bucket(&source_bucket).key(key).send().await?;
|
||||
assert_eq!(source.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
|
||||
assert_eq!(source.body.collect().await?.into_bytes().as_ref(), body.as_slice());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// backlog#1147 repl-17: SSE-KMS currently fails closed rather than creating an
|
||||
/// unreadable replica; the shared helper verifies FAILED, the failure event,
|
||||
/// source readability, and a stable absence of all target versions.
|
||||
@@ -6838,140 +6517,3 @@ async fn test_site_replication_replicates_service_accounts_created_from_sts_sess
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Poll the fake target journal until `operation` arrives for `key`, then
|
||||
/// return the `versionId` query value the request carried.
|
||||
async fn wait_for_target_request_version_id(
|
||||
target: &FakeS3Target,
|
||||
operation: FakeTargetOperation,
|
||||
key: &str,
|
||||
) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
if let Some(record) = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == operation && record.key.as_deref() == Some(key))
|
||||
{
|
||||
return Ok(record.version_id);
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("fake target never received {operation:?} for {key}; journal: {:?}", target.requests()).into());
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// P0-5: MinIO derives the replicated version exclusively from the `versionId`
|
||||
/// query parameter (`putOptsFromReq`); the internal x-*-source-version-id
|
||||
/// headers do not exist there. Without the query, a MinIO target mints fresh
|
||||
/// version ids and RustFS -> MinIO replication drifts. PutObject and
|
||||
/// CreateMultipartUpload (the version is decided at initiate time) must both
|
||||
/// carry the source version as `?versionId=`.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_put_and_create_multipart_carry_source_version_id_query() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "versionid-query-dst";
|
||||
target.create_bucket(target_bucket);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_process_env = replication_fast_env();
|
||||
source_process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_process_env.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?;
|
||||
|
||||
let source_bucket = "versionid-query-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
|
||||
let target_arn = set_replication_target_with_options(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
// Small object -> replicated through a single PutObject.
|
||||
let put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("small.txt")
|
||||
.body(ByteStream::from_static(b"versionid query payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let put_source_version = put
|
||||
.version_id()
|
||||
.ok_or("versioned source PUT must return a version id")?
|
||||
.to_string();
|
||||
let recorded = wait_for_target_request_version_id(&target, FakeTargetOperation::PutObject, "small.txt").await?;
|
||||
assert_eq!(
|
||||
recorded.as_deref(),
|
||||
Some(put_source_version.as_str()),
|
||||
"replication PutObject must carry the source version in the versionId query"
|
||||
);
|
||||
|
||||
// Multipart source object -> replicated through CreateMultipartUpload;
|
||||
// the target version is fixed at initiate time.
|
||||
let create = source_client
|
||||
.create_multipart_upload()
|
||||
.bucket(source_bucket)
|
||||
.key("large.bin")
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = create
|
||||
.upload_id()
|
||||
.ok_or("multipart initiate must return an upload id")?
|
||||
.to_string();
|
||||
let mut completed_parts = Vec::new();
|
||||
for (part_number, body) in [(1, vec![b'a'; 5 * 1024 * 1024]), (2, vec![b'b'; 1024])] {
|
||||
let uploaded = source_client
|
||||
.upload_part()
|
||||
.bucket(source_bucket)
|
||||
.key("large.bin")
|
||||
.upload_id(&upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(body))
|
||||
.send()
|
||||
.await?;
|
||||
completed_parts.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.e_tag(uploaded.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
let complete = source_client
|
||||
.complete_multipart_upload()
|
||||
.bucket(source_bucket)
|
||||
.key("large.bin")
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build())
|
||||
.send()
|
||||
.await?;
|
||||
let multipart_source_version = complete
|
||||
.version_id()
|
||||
.ok_or("versioned multipart completion must return a version id")?
|
||||
.to_string();
|
||||
let recorded = wait_for_target_request_version_id(&target, FakeTargetOperation::CreateMultipartUpload, "large.bin").await?;
|
||||
assert_eq!(
|
||||
recorded.as_deref(),
|
||||
Some(multipart_source_version.as_str()),
|
||||
"replication CreateMultipartUpload must carry the source version in the versionId query"
|
||||
);
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -12,10 +12,13 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, build_test_s3_config, build_test_sts_client, init_logging};
|
||||
use aws_sdk_sts::Client;
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
|
||||
use aws_sdk_sts::config::retry::RetryConfig;
|
||||
use aws_sdk_sts::config::{Credentials, Region};
|
||||
use aws_sdk_sts::error::ProvideErrorMetadata;
|
||||
use aws_sdk_sts::operation::RequestId;
|
||||
use aws_sdk_sts::{Client, Config};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use bytes::Bytes;
|
||||
use http::header::{AUTHORIZATION, CONTENT_TYPE};
|
||||
use http::{Request, Response};
|
||||
@@ -29,8 +32,9 @@ use serial_test::serial;
|
||||
use std::collections::BTreeSet;
|
||||
use std::convert::Infallible;
|
||||
use std::error::Error;
|
||||
use std::sync::Arc;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::sync::{Notify, mpsc};
|
||||
use tokio::task::{JoinHandle, JoinSet};
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
@@ -39,7 +43,22 @@ type TestResult = Result<(), BoxError>;
|
||||
const OPA_AUTH_TOKEN: &str = "sts-opa-token";
|
||||
|
||||
fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
|
||||
build_test_sts_client(url, access_key, secret_key, session_token, "e2e-sts-query-compat")
|
||||
let mut config = Config::builder()
|
||||
.credentials_provider(Credentials::new(
|
||||
access_key,
|
||||
secret_key,
|
||||
session_token.map(str::to_owned),
|
||||
None,
|
||||
"e2e-sts-query-compat",
|
||||
))
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(url)
|
||||
.retry_config(RetryConfig::standard().with_max_attempts(1))
|
||||
.behavior_version_latest();
|
||||
if url.starts_with("http://") {
|
||||
config = config.http_client(SmithyHttpClientBuilder::new().build_http());
|
||||
}
|
||||
Client::from_conf(config.build())
|
||||
}
|
||||
|
||||
async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(String, String), BoxError> {
|
||||
@@ -126,52 +145,6 @@ async fn assert_access_denied(client: &Client, context: &str) -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_list_buckets_access_denied(
|
||||
env: &RustFSTestEnvironment,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
context: &str,
|
||||
) -> TestResult {
|
||||
let error = aws_sdk_s3::Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
access_key,
|
||||
secret_key,
|
||||
None,
|
||||
"e2e-list-buckets-opa-unavailable",
|
||||
))
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("ListBuckets must be denied while OPA is unavailable");
|
||||
let service_error = error
|
||||
.as_service_error()
|
||||
.ok_or_else(|| format!("{context} should deserialize as an S3 service error: {error:?}"))?;
|
||||
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(403));
|
||||
assert_eq!(service_error.code(), Some("AccessDenied"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_opa_unavailable_denies_sts_and_list_buckets(env: &RustFSTestEnvironment, context: &str) -> TestResult {
|
||||
let user = "opaunavailable";
|
||||
let secret = "stsOpaUnavailableSecret123";
|
||||
create_user_with_policy(
|
||||
env,
|
||||
user,
|
||||
secret,
|
||||
"sts-opa-unavailable-local-policy",
|
||||
serde_json::json!([{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListAllMyBuckets"],
|
||||
"Resource": ["arn:aws:s3:::*"],
|
||||
}]),
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_access_denied(&sts_client(&env.url, user, secret, None), context).await?;
|
||||
assert_list_buckets_access_denied(env, user, secret, context).await
|
||||
}
|
||||
|
||||
async fn handle_opa_request(
|
||||
request: Request<Incoming>,
|
||||
requests: mpsc::UnboundedSender<Value>,
|
||||
@@ -213,15 +186,12 @@ async fn handle_opa_request(
|
||||
};
|
||||
if payload.is_none() {
|
||||
let _ = validation_started.send(());
|
||||
match validation_mode {
|
||||
OpaValidationMode::Blocked => std::future::pending::<()>().await,
|
||||
OpaValidationMode::Unavailable => {
|
||||
return Ok(Response::builder()
|
||||
.status(503)
|
||||
.body(Full::new(Bytes::new()))
|
||||
.expect("static OPA unavailable response must be valid"));
|
||||
}
|
||||
OpaValidationMode::Ready => {}
|
||||
if let OpaValidationMode::DelayedUnavailable(release) = validation_mode {
|
||||
release.notified().await;
|
||||
return Ok(Response::builder()
|
||||
.status(503)
|
||||
.body(Full::new(Bytes::new()))
|
||||
.expect("static OPA unavailable response must be valid"));
|
||||
}
|
||||
}
|
||||
let allow = match payload.as_ref().and_then(|value| value.pointer("/input/identity/account")) {
|
||||
@@ -231,25 +201,6 @@ async fn handle_opa_request(
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false),
|
||||
Some(Value::String(account)) if account == "opadeny" => false,
|
||||
Some(Value::String(account))
|
||||
if account == "opaunavailable" && matches!(validation_mode, OpaValidationMode::Unavailable) =>
|
||||
{
|
||||
true
|
||||
}
|
||||
Some(Value::String(account)) if account == "opalistbuckets" => {
|
||||
let action = payload
|
||||
.as_ref()
|
||||
.and_then(|value| value.pointer("/input/action"))
|
||||
.and_then(Value::as_str);
|
||||
let bucket = payload
|
||||
.as_ref()
|
||||
.and_then(|value| value.pointer("/input/resource/bucket"))
|
||||
.and_then(Value::as_str);
|
||||
matches!(
|
||||
(action, bucket),
|
||||
(Some("s3:ListBucket"), Some("opa-list-visible")) | (Some("s3:GetBucketLocation"), Some("opa-list-location"))
|
||||
)
|
||||
}
|
||||
None => true,
|
||||
_ => false,
|
||||
};
|
||||
@@ -264,17 +215,17 @@ async fn handle_opa_request(
|
||||
.expect("static OPA response must be valid"))
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
#[derive(Clone)]
|
||||
enum OpaValidationMode {
|
||||
Ready,
|
||||
Blocked,
|
||||
Unavailable,
|
||||
DelayedUnavailable(Arc<Notify>),
|
||||
}
|
||||
|
||||
struct OpaMock {
|
||||
url: String,
|
||||
requests: mpsc::UnboundedReceiver<Value>,
|
||||
validation_started: mpsc::UnboundedReceiver<()>,
|
||||
validation_release: Option<Arc<Notify>>,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
|
||||
@@ -283,8 +234,9 @@ impl OpaMock {
|
||||
Self::start_with_mode(OpaValidationMode::Ready, Some(OPA_AUTH_TOKEN)).await
|
||||
}
|
||||
|
||||
async fn start_blocked() -> Result<Self, BoxError> {
|
||||
Self::start_with_mode(OpaValidationMode::Blocked, None).await
|
||||
async fn start_delayed_unavailable() -> Result<Self, BoxError> {
|
||||
let release = Arc::new(Notify::new());
|
||||
Self::start_with_mode(OpaValidationMode::DelayedUnavailable(release), None).await
|
||||
}
|
||||
|
||||
async fn start_with_mode(validation_mode: OpaValidationMode, auth_token: Option<&str>) -> Result<Self, BoxError> {
|
||||
@@ -293,6 +245,10 @@ impl OpaMock {
|
||||
let (requests_tx, requests) = mpsc::unbounded_channel();
|
||||
let (validation_started_tx, validation_started) = mpsc::unbounded_channel();
|
||||
let expected_authorization = auth_token.map(|token| format!("Bearer {token}"));
|
||||
let validation_release = match &validation_mode {
|
||||
OpaValidationMode::Ready => None,
|
||||
OpaValidationMode::DelayedUnavailable(release) => Some(Arc::clone(release)),
|
||||
};
|
||||
let task = tokio::spawn(async move {
|
||||
let mut connections = JoinSet::new();
|
||||
loop {
|
||||
@@ -301,7 +257,7 @@ impl OpaMock {
|
||||
let Ok((stream, _)) = accepted else { break };
|
||||
let requests = requests_tx.clone();
|
||||
let validation_started = validation_started_tx.clone();
|
||||
let validation_mode = validation_mode;
|
||||
let validation_mode = validation_mode.clone();
|
||||
let expected_authorization = expected_authorization.clone();
|
||||
connections.spawn(async move {
|
||||
let handler = service_fn(move |request| {
|
||||
@@ -309,7 +265,7 @@ impl OpaMock {
|
||||
request,
|
||||
requests.clone(),
|
||||
validation_started.clone(),
|
||||
validation_mode,
|
||||
validation_mode.clone(),
|
||||
expected_authorization.clone(),
|
||||
)
|
||||
});
|
||||
@@ -326,6 +282,7 @@ impl OpaMock {
|
||||
url,
|
||||
requests,
|
||||
validation_started,
|
||||
validation_release,
|
||||
task,
|
||||
})
|
||||
}
|
||||
@@ -341,6 +298,12 @@ impl OpaMock {
|
||||
.await?
|
||||
.ok_or_else(|| "OPA validation channel closed".into())
|
||||
}
|
||||
|
||||
fn release_validation(&self) {
|
||||
if let Some(release) = &self.validation_release {
|
||||
release.notify_one();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for OpaMock {
|
||||
@@ -560,119 +523,35 @@ async fn test_sts_assume_role_opa_contract() -> TestResult {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_list_buckets_opa_contract() -> TestResult {
|
||||
async fn test_sts_assume_role_fails_closed_while_opa_is_unavailable() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut opa = OpaMock::start().await?;
|
||||
let mut opa = OpaMock::start_delayed_unavailable().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
|
||||
],
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
|
||||
.await?;
|
||||
opa.wait_for_validation().await?;
|
||||
|
||||
let user = "opaunavailable";
|
||||
let secret = "stsOpaUnavailableSecret123";
|
||||
create_user_with_policy(
|
||||
&env,
|
||||
user,
|
||||
secret,
|
||||
"sts-opa-unavailable-local-policy",
|
||||
serde_json::json!([{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListAllMyBuckets"],
|
||||
"Resource": ["arn:aws:s3:::*"],
|
||||
}]),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
for bucket in ["opa-list-hidden", "opa-list-location", "opa-list-visible"] {
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
}
|
||||
assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA initialization").await?;
|
||||
|
||||
let user = "opalistbuckets";
|
||||
let secret = "opaListBucketsSecret123";
|
||||
create_user(&env, user, secret).await?;
|
||||
|
||||
let output = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, user, secret, None, "e2e-list-buckets-opa"))
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?;
|
||||
let mut names = output
|
||||
.buckets()
|
||||
.iter()
|
||||
.filter_map(|bucket| bucket.name().map(str::to_owned))
|
||||
.collect::<Vec<_>>();
|
||||
names.sort();
|
||||
assert_eq!(names, ["opa-list-location", "opa-list-visible"]);
|
||||
|
||||
let mut evaluations = BTreeSet::new();
|
||||
for _ in 0..6 {
|
||||
let request = opa.next_request().await?;
|
||||
assert_eq!(request.pointer("/input/identity/account").and_then(Value::as_str), Some(user));
|
||||
assert_eq!(request.pointer("/input/context/deny_only").and_then(Value::as_bool), Some(false));
|
||||
|
||||
let action = request
|
||||
.pointer("/input/action")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("OPA ListBuckets input should include action")?;
|
||||
let bucket = request
|
||||
.pointer("/input/resource/bucket")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("OPA ListBuckets input should include resource.bucket")?;
|
||||
if bucket.is_empty() {
|
||||
assert_eq!(action, "s3:ListAllMyBuckets");
|
||||
assert!(request.pointer("/input/context/conditions/prefix").is_none());
|
||||
assert!(request.pointer("/input/context/conditions/delimiter").is_none());
|
||||
} else {
|
||||
let expected_arn = format!("arn:aws:s3:::{bucket}");
|
||||
assert_eq!(request.pointer("/input/context/conditions/prefix"), Some(&serde_json::json!([""])));
|
||||
assert_eq!(request.pointer("/input/context/conditions/delimiter"), Some(&serde_json::json!(["/"])));
|
||||
assert_eq!(
|
||||
request.pointer("/input/resource/arn").and_then(Value::as_str),
|
||||
Some(expected_arn.as_str())
|
||||
);
|
||||
}
|
||||
evaluations.insert((action.to_owned(), bucket.to_owned()));
|
||||
}
|
||||
assert_eq!(
|
||||
evaluations,
|
||||
BTreeSet::from([
|
||||
("s3:GetBucketLocation".to_owned(), "opa-list-hidden".to_owned()),
|
||||
("s3:GetBucketLocation".to_owned(), "opa-list-location".to_owned()),
|
||||
("s3:ListAllMyBuckets".to_owned(), String::new()),
|
||||
("s3:ListBucket".to_owned(), "opa-list-hidden".to_owned()),
|
||||
("s3:ListBucket".to_owned(), "opa-list-location".to_owned()),
|
||||
("s3:ListBucket".to_owned(), "opa-list-visible".to_owned()),
|
||||
])
|
||||
);
|
||||
assert!(
|
||||
matches!(opa.requests.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
|
||||
"ListBuckets should not make redundant OPA evaluations"
|
||||
);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_sts_and_list_buckets_fail_closed_while_opa_is_initializing() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut opa = OpaMock::start_blocked().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
|
||||
.await?;
|
||||
opa.wait_for_validation().await?;
|
||||
|
||||
assert_opa_unavailable_denies_sts_and_list_buckets(&env, "configured OPA initialization").await?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_sts_and_list_buckets_fail_closed_after_opa_validation_failure() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut opa = OpaMock::start_with_mode(OpaValidationMode::Unavailable, None).await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
|
||||
.await?;
|
||||
opa.wait_for_validation().await?;
|
||||
|
||||
assert_opa_unavailable_denies_sts_and_list_buckets(&env, "configured OPA validation failure").await?;
|
||||
opa.release_validation();
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA validation failure").await?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
|
||||
@@ -144,13 +144,11 @@ rustfs-lifecycle.workspace = true
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-data-usage.workspace = true
|
||||
rustfs-object-capacity.workspace = true
|
||||
rustfs-object-data-cache = { workspace = true, features = ["runtime-memory"] }
|
||||
arc-swap.workspace = true
|
||||
async-trait.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
byteorder = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
glob = { workspace = true }
|
||||
thiserror.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
|
||||
@@ -185,19 +185,19 @@ pub mod bucket {
|
||||
MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
|
||||
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
||||
REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo,
|
||||
ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigStructureError, ReplicationConfigurationExt,
|
||||
ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge,
|
||||
ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission,
|
||||
ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage,
|
||||
ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog,
|
||||
TargetReplicationResyncStatus, VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
|
||||
ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
|
||||
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
|
||||
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
||||
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
||||
VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||
replication_statuses_map, replication_target_arns, resync_start_conflict_id, should_remove_replication_target,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
||||
validate_replication_config_structure, validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -206,9 +206,7 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod target {
|
||||
pub use crate::bucket::target::{
|
||||
ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials, LatencyStat, duration_from_secs_or_nanos,
|
||||
};
|
||||
pub use crate::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials, LatencyStat};
|
||||
}
|
||||
|
||||
pub mod utils {
|
||||
@@ -310,8 +308,7 @@ pub mod config {
|
||||
pub mod data_usage {
|
||||
pub use crate::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
|
||||
init_compression_total_memory_from_backend, invalidate_data_usage_snapshot_cache, live_bucket_usage_computations,
|
||||
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached,
|
||||
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
|
||||
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
|
||||
@@ -440,7 +437,7 @@ pub mod rpc {
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
|
||||
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
|
||||
@@ -1424,37 +1424,6 @@ fn resolve_delete_api_version_id(version_id: Option<String>, opts: &RemoveObject
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the S3 `versionId` query parameter for a replication PUT /
|
||||
/// CreateMultipartUpload against a remote target.
|
||||
///
|
||||
/// MinIO reads the replicated version only from the query string
|
||||
/// (`putOptsFromReq`); the internal `x-*-source-version-id` headers do not
|
||||
/// exist there, so without the query a MinIO target mints fresh version ids
|
||||
/// and the deployments drift apart. RustFS represents the null version
|
||||
/// internally as the nil UUID while the S3 API addresses it as the literal
|
||||
/// "null" (the delete path already maps it via `target_delete_version_id`),
|
||||
/// and an empty id means the source object carries no version: send no query
|
||||
/// so an unversioned target stays valid.
|
||||
fn resolve_put_api_version_id(source_version_id: &str) -> Option<&str> {
|
||||
if source_version_id.is_empty() {
|
||||
None
|
||||
} else if Uuid::parse_str(source_version_id).is_ok_and(|uuid| uuid.is_nil()) {
|
||||
Some(rustfs_filemeta::NULL_VERSION_ID)
|
||||
} else {
|
||||
Some(source_version_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Append `versionId=<id>` to an already-built request URI. aws-sdk-s3's
|
||||
/// `PutObjectInput` / `CreateMultipartUploadInput` expose no version id
|
||||
/// member, so the query is spliced in via `map_request`, which runs at
|
||||
/// `modify_before_signing`: the parameter becomes part of the SigV4 canonical
|
||||
/// request.
|
||||
fn append_version_id_query(uri: &str, version_id: &str) -> String {
|
||||
let separator = if uri.contains('?') { '&' } else { '?' };
|
||||
format!("{uri}{separator}versionId={}", urlencoding::encode(version_id))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AdvancedPutOptions {
|
||||
pub source_version_id: String,
|
||||
@@ -1862,7 +1831,6 @@ impl TargetClient {
|
||||
if !version_id.is_empty() {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, &version_id);
|
||||
}
|
||||
let api_version_id = resolve_put_api_version_id(&version_id).map(ToOwned::to_owned);
|
||||
|
||||
match builder
|
||||
.bucket(bucket)
|
||||
@@ -1877,11 +1845,6 @@ impl TargetClient {
|
||||
req.headers_mut().insert(key_str, value_str);
|
||||
}
|
||||
}
|
||||
if let Some(version_id) = &api_version_id {
|
||||
let uri = append_version_id_query(req.uri(), version_id);
|
||||
req.set_uri(uri)
|
||||
.map_err(aws_smithy_types::error::operation::BuildError::other)?;
|
||||
}
|
||||
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
@@ -1930,9 +1893,6 @@ impl TargetClient {
|
||||
if opts.internal.replication_request {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
}
|
||||
// The remote version of a multipart replication is decided at initiate
|
||||
// time; CompleteMultipartUpload does not read a versionId.
|
||||
let api_version_id = resolve_put_api_version_id(&version_id).map(ToOwned::to_owned);
|
||||
|
||||
match self
|
||||
.client
|
||||
@@ -1947,11 +1907,6 @@ impl TargetClient {
|
||||
req.headers_mut().insert(key_str, value_str);
|
||||
}
|
||||
}
|
||||
if let Some(version_id) = &api_version_id {
|
||||
let uri = append_version_id_query(req.uri(), version_id);
|
||||
req.set_uri(uri)
|
||||
.map_err(aws_smithy_types::error::operation::BuildError::other)?;
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.send()
|
||||
@@ -2724,91 +2679,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_sends_source_version_id_query_to_target() {
|
||||
// MinIO reads the replicated version only from the `versionId` query
|
||||
// parameter (its receive path ignores the x-*-source-version-id
|
||||
// headers), so the query must carry the source version: a real UUID
|
||||
// as-is, the internal nil-UUID null-version representation as the
|
||||
// literal "null", and no query at all when the source object has no
|
||||
// version (P0-5 RustFS->MinIO version drift).
|
||||
let (client, request_uris) = recording_target_client();
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
let nil_version = Uuid::nil().to_string();
|
||||
for source_version in [version_id.as_str(), nil_version.as_str(), ""] {
|
||||
let mut opts = PutObjectOptions::default();
|
||||
opts.internal.source_version_id = source_version.to_string();
|
||||
opts.internal.replication_request = true;
|
||||
client
|
||||
.put_object("target-bucket", "object", 4, ByteStream::from_static(b"data"), &opts)
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
}
|
||||
|
||||
let request_uris = request_uris.lock().expect("recorded request lock should not be poisoned");
|
||||
assert_eq!(request_uris.len(), 3);
|
||||
assert!(
|
||||
request_uris[0].contains(&format!("versionId={version_id}")),
|
||||
"replication put_object must carry the source version as a versionId query: {}",
|
||||
request_uris[0]
|
||||
);
|
||||
assert!(
|
||||
request_uris[1].contains("versionId=null"),
|
||||
"a nil-UUID (null) source version must be sent as the literal null: {}",
|
||||
request_uris[1]
|
||||
);
|
||||
assert!(
|
||||
!request_uris[2].contains("versionId="),
|
||||
"put_object without a source version must omit the versionId query: {}",
|
||||
request_uris[2]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_multipart_upload_sends_source_version_id_query_to_target() {
|
||||
// The remote version of a multipart replication is decided at initiate
|
||||
// time: CreateMultipartUpload must carry the source version in the
|
||||
// `versionId` query (CompleteMultipartUpload does not read one).
|
||||
let (client, request_uris) = recording_target_client();
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
let nil_version = Uuid::nil().to_string();
|
||||
for source_version in [version_id.as_str(), nil_version.as_str()] {
|
||||
let mut opts = PutObjectOptions::default();
|
||||
opts.internal.source_version_id = source_version.to_string();
|
||||
opts.internal.replication_request = true;
|
||||
let _ = client.create_multipart_upload("target-bucket", "object", &opts).await;
|
||||
}
|
||||
|
||||
let request_uris = request_uris.lock().expect("recorded request lock should not be poisoned");
|
||||
assert_eq!(request_uris.len(), 2);
|
||||
assert!(
|
||||
request_uris[0].contains(&format!("versionId={version_id}")),
|
||||
"replication create_multipart_upload must carry the source version as a versionId query: {}",
|
||||
request_uris[0]
|
||||
);
|
||||
assert!(
|
||||
request_uris[1].contains("versionId=null"),
|
||||
"a nil-UUID (null) source version must be sent as the literal null: {}",
|
||||
request_uris[1]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_headers_keep_source_version_id_for_legacy_receivers() {
|
||||
// Older RustFS receivers have no versionId query support and fall back
|
||||
// to the internal source-version-id headers (rolling-upgrade path);
|
||||
// the query addition must never remove them.
|
||||
let mut opts = PutObjectOptions::default();
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
opts.internal.source_version_id = version_id.clone();
|
||||
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&opts.header(), SUFFIX_SOURCE_VERSION_ID).as_deref(),
|
||||
Some(version_id.as_str()),
|
||||
"replication put requests must keep the internal source-version-id headers"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_headers_include_non_empty_source_etag_only() {
|
||||
let mut opts = PutObjectOptions::default();
|
||||
|
||||
@@ -10774,18 +10774,7 @@ mod tests {
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_real_enqueue_failure_retries_same_object() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("recovery-enqueue-failure-{}", Uuid::new_v4());
|
||||
let object = "free-version-b";
|
||||
let start_marker = "free-version-a0";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, None).await;
|
||||
|
||||
let runtime_state = install_unconsumed_runtime_expiry_worker(&ecstore, 1).await;
|
||||
let recovery_rx = {
|
||||
let state = runtime_state.read().await;
|
||||
Arc::clone(&state.tasks_rx[0])
|
||||
};
|
||||
let mut recovery_rx = recovery_rx.lock().await;
|
||||
assert!(
|
||||
super::enqueue_recovered_free_version(ObjectInfo {
|
||||
bucket: "prefill".to_string(),
|
||||
@@ -10795,29 +10784,20 @@ mod tests {
|
||||
.await,
|
||||
"the production recovery queue should accept its first task"
|
||||
);
|
||||
let bucket = format!("recovery-enqueue-failure-{}", Uuid::new_v4());
|
||||
let object = "free-version";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, None).await;
|
||||
|
||||
let first = recover_tier_free_versions_with_cancel(
|
||||
Arc::clone(&ecstore),
|
||||
1,
|
||||
Some(bucket.clone()),
|
||||
Some(start_marker.to_string()),
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
.expect("queue failure should return retry markers");
|
||||
let first = recover_tier_free_versions_with_cancel(Arc::clone(&ecstore), 1, None, None, CancellationToken::new())
|
||||
.await
|
||||
.expect("queue failure should return retry markers");
|
||||
assert_eq!(first.scanned, 1);
|
||||
assert_eq!(first.enqueued, 0);
|
||||
assert_eq!(first.failed, 1);
|
||||
assert!(first.truncated);
|
||||
assert_eq!(first.next_bucket_marker.as_deref(), Some(bucket.as_str()));
|
||||
assert_eq!(first.next_object_marker.as_deref(), Some(start_marker));
|
||||
|
||||
drop(
|
||||
recovery_rx
|
||||
.try_recv()
|
||||
.expect("the failed recovery attempt must leave the prefilled task queued")
|
||||
.expect("the prefilled recovery queue entry should contain a task"),
|
||||
);
|
||||
assert!(first.next_object_marker.is_none());
|
||||
|
||||
let retried = recover_tier_free_versions_with_cancel(
|
||||
Arc::clone(&ecstore),
|
||||
@@ -10829,19 +10809,7 @@ mod tests {
|
||||
.await
|
||||
.expect("retry markers should revisit the failed free version");
|
||||
assert_eq!(retried.scanned, 1);
|
||||
assert_eq!(retried.enqueued, 1);
|
||||
assert_eq!(retried.failed, 0);
|
||||
|
||||
let retried_task = recovery_rx
|
||||
.try_recv()
|
||||
.expect("the retry should enqueue the recovered free-version task")
|
||||
.expect("the recovered queue entry should contain a task");
|
||||
let retried_task = retried_task
|
||||
.as_any()
|
||||
.downcast_ref::<FreeVersionTask>()
|
||||
.expect("the recovered queue entry should be a free-version task");
|
||||
assert_eq!(retried_task.0.bucket, bucket);
|
||||
assert_eq!(retried_task.0.name, object);
|
||||
assert_eq!(retried.failed, 1);
|
||||
|
||||
remove_seeded_free_version(&disk_paths, &bucket, object).await;
|
||||
ecstore
|
||||
|
||||
@@ -198,36 +198,11 @@ impl QuotaChecker {
|
||||
}
|
||||
|
||||
pub async fn get_real_time_usage(&self, bucket: &str) -> Result<u64, QuotaError> {
|
||||
if let Some(usage) = get_bucket_usage_memory(bucket).await {
|
||||
return Ok(usage);
|
||||
}
|
||||
|
||||
// Degraded window (issue #5716): with no authoritative usage — most
|
||||
// prominently after upgrading from a pre-v2 release, whose legacy
|
||||
// `.usage.json` is demoted to non-authoritative until the scanner's
|
||||
// first complete cycle persists `.usage.v2.json` — failing closed
|
||||
// turned every write to a quota-enabled bucket into a retryable 503
|
||||
// for the whole window. Quota admission instead degrades to the last
|
||||
// persisted per-bucket size. That baseline is static between snapshot
|
||||
// loads (live writes do not advance it), so hard-quota enforcement is
|
||||
// advisory for the duration of the window: the overrun is bounded by
|
||||
// the writes issued before the next complete scanner cycle. Buckets
|
||||
// with no persisted baseline anywhere keep failing closed.
|
||||
let store = self.metadata_sys.read().await.object_store();
|
||||
// Box the fallback: it embeds the whole snapshot-load future, and every
|
||||
// object write nests a quota check several futures deep, so keeping it
|
||||
// inline would grow each write's state machine by the loader's full
|
||||
// size — the debug-build 2MiB worker-stack overflow class fixed for
|
||||
// bucket-config writes in #5648. The allocation only happens on the
|
||||
// degraded path; the authoritative fast path returns above.
|
||||
if let Some(baseline) = Box::pin(crate::data_usage::lookup_degraded_bucket_usage_baseline(store, bucket)).await {
|
||||
debug!(bucket, baseline, "Bucket quota admission using degraded persisted usage baseline");
|
||||
return Ok(baseline);
|
||||
}
|
||||
|
||||
Err(QuotaError::UsageUnavailable {
|
||||
bucket: bucket.to_string(),
|
||||
})
|
||||
get_bucket_usage_memory(bucket)
|
||||
.await
|
||||
.ok_or_else(|| QuotaError::UsageUnavailable {
|
||||
bucket: bucket.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,76 +232,6 @@ mod tests {
|
||||
assert_eq!(result.quota_limit, None);
|
||||
}
|
||||
|
||||
/// Regression (issue #5716): an upgrade from a pre-v2 release leaves only
|
||||
/// the legacy `.usage.json` snapshot, which has no completeness marker and
|
||||
/// is demoted to non-authoritative, and the scanner's first complete cycle
|
||||
/// can be a long way off. Quota admission must degrade to that persisted
|
||||
/// baseline instead of failing every write to a quota-enabled bucket with
|
||||
/// a retryable 503 for the whole window.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_falls_back_to_legacy_snapshot_baseline() {
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore.clone())));
|
||||
let checker = QuotaChecker::new(sys);
|
||||
let bucket = format!("quota-legacy-{}", Uuid::new_v4().simple());
|
||||
|
||||
let mut legacy = rustfs_data_usage::DataUsageInfo {
|
||||
last_update: Some(std::time::SystemTime::now()),
|
||||
buckets_count: 1,
|
||||
..Default::default()
|
||||
};
|
||||
legacy.buckets_usage.insert(
|
||||
bucket.clone(),
|
||||
rustfs_data_usage::BucketUsageInfo {
|
||||
size: 1_234,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
legacy.bucket_sizes.insert(bucket.clone(), 1_234);
|
||||
// usage_snapshot_complete stays false: pre-v2 snapshots do not carry
|
||||
// the field at all, so they always deserialize as incomplete.
|
||||
let legacy_path = format!("{}/{}", crate::disk::BUCKET_META_PREFIX, rustfs_data_usage::LEGACY_DATA_USAGE_OBJECT_NAME);
|
||||
crate::config::com::save_config(
|
||||
ecstore.clone(),
|
||||
&legacy_path,
|
||||
serde_json::to_vec(&legacy).expect("legacy snapshot should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("legacy snapshot fixture should be stored");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
|
||||
let usage = checker
|
||||
.get_real_time_usage(&bucket)
|
||||
.await
|
||||
.expect("quota admission must degrade to the persisted legacy baseline");
|
||||
assert_eq!(usage, 1_234);
|
||||
|
||||
// A bucket absent from every persisted snapshot still has no grounded
|
||||
// baseline and must keep failing closed.
|
||||
let unknown = format!("quota-unknown-{}", Uuid::new_v4().simple());
|
||||
assert!(matches!(
|
||||
checker.get_real_time_usage(&unknown).await,
|
||||
Err(QuotaError::UsageUnavailable { .. })
|
||||
));
|
||||
|
||||
// Deleting the bucket's usage from the backend must purge the
|
||||
// baseline: a recreated bucket may not inherit the dead incarnation's
|
||||
// size, so with no persisted trace left it fails closed again.
|
||||
crate::data_usage::remove_bucket_usage_from_backend(ecstore.clone(), &bucket)
|
||||
.await
|
||||
.expect("bucket usage removal should succeed");
|
||||
assert!(matches!(
|
||||
checker.get_real_time_usage(&bucket).await,
|
||||
Err(QuotaError::UsageUnavailable { .. })
|
||||
));
|
||||
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
|
||||
.await
|
||||
.expect("test usage cache cleanup should succeed");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_usage_rejects_an_unknown_mutation_baseline() {
|
||||
|
||||
@@ -100,41 +100,11 @@ paths.
|
||||
behind the ECStore replication facade; only `rustfs/src/app/storage_api.rs`
|
||||
may retain direct object/delete replication helper calls.
|
||||
|
||||
## Completion Criteria
|
||||
## First Code-Bearing Step
|
||||
|
||||
The split is complete when the "Current dependency to remove" column in the
|
||||
Required Contracts table above is empty: every row is either deleted because
|
||||
the dependency is gone, or reduced to "none". No other signal — file count,
|
||||
boundary count, line count — measures completion.
|
||||
|
||||
Target end state:
|
||||
|
||||
- `replication_pool.rs`, `replication_resyncer.rs`, and `replication_state.rs`
|
||||
move into `crates/replication` behind the contracts above;
|
||||
- the `*_boundary.rs` and `*_bridge.rs` micro-files dissolve naturally as the
|
||||
code they fence moves across the crate boundary. They are the mechanical
|
||||
seams of the migration ratchet — the architecture guard scripts anchor on
|
||||
their file names — so batch-merging them beforehand is explicitly rejected:
|
||||
it forces synchronized guard-script/mod/import churn with zero functional
|
||||
gain;
|
||||
- the only module that can retire early is `datatypes.rs`: delete it once its
|
||||
facade consumers import the resync status enums through `rustfs-replication`
|
||||
directly.
|
||||
|
||||
## Milestones
|
||||
|
||||
| Milestone | Scope | Status |
|
||||
|---|---|---|
|
||||
| M0 | Record the completion criteria and end state (this section). | Done |
|
||||
| M1 | Contract extraction: resync/queue/stats/object-decision/filemeta/storage wire contracts owned by `crates/replication`; ECStore imports concentrated in `*_boundary.rs`; event sink and runtime access behind local contracts. | Done — see Required Contracts |
|
||||
| M2 | Move resyncer pure decision logic (no IO) into `crates/replication`. | Pending; sequence after splitting the oversized resyncer/pool functions (`resync_bucket`, `replicate_all`, `start_mrf_processor`) so moves stay mechanical |
|
||||
| M3 | Move the worker runtime (`replication_pool.rs`, the IO paths of `replication_resyncer.rs`, `replication_state.rs`) once the contract traits are stable. Highest-risk step of the whole plan; do it last. | Pending |
|
||||
| M4 | Retire the boundary modules together with their guard-script entries; delete `datatypes.rs`. | Pending |
|
||||
|
||||
The original first code-bearing step (narrow `ReplicationEventSink` /
|
||||
`ReplicationRuntime` contracts) has landed — `replication_event_sink.rs`
|
||||
exists and runtime access goes through local boundary aliases — so new work
|
||||
starts from M2.
|
||||
Start with `ReplicationRuntime` or `ReplicationEventSink`. Both can be added as
|
||||
narrow internal contracts while keeping current queue, MRF, resync, and target
|
||||
behavior unchanged. Do not start with a crate move.
|
||||
|
||||
Current compatibility guard: `crates/ecstore/tests/replication_facade_compat_test.rs`
|
||||
keeps the ECStore replication facade types covered while architecture rules
|
||||
|
||||
@@ -47,9 +47,9 @@ pub use datatypes::ResyncStatusType;
|
||||
pub use replication_config_boundary::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
|
||||
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
ReplicationConfigurationExt, ReplicationTargetValidationError, invalid_replication_config_status_field,
|
||||
replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
|
||||
validate_replication_config_target_arns,
|
||||
};
|
||||
pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
|
||||
pub use replication_filemeta_boundary::{
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
pub use rustfs_replication::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
|
||||
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError, invalid_replication_config_status_field,
|
||||
replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
|
||||
validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
@@ -89,116 +89,3 @@ pub fn replication_state_to_filemeta(state: &ReplicationState) -> rustfs_filemet
|
||||
target_delete_marker_version_ids_corrupt: state.target_delete_marker_version_ids_corrupt,
|
||||
}
|
||||
}
|
||||
|
||||
// Reconciliation tests for the deliberately duplicated wire types.
|
||||
//
|
||||
// `rustfs-filemeta` (xl.meta disk format) and `rustfs-replication` (MRF/resync
|
||||
// persistence format) each own a copy of `ReplicationStatusType`,
|
||||
// `VersionPurgeStatusType` and `ReplicationState`; the conversions above hop
|
||||
// between them via `as_str()`, whose `From<&str>` impls fall back to `Empty`
|
||||
// on any unknown token. That fallback silently degrades data the moment one
|
||||
// side gains a variant the other lacks, so these tests pin the two sides
|
||||
// together:
|
||||
//
|
||||
// - the `match` statements are exhaustive with no `_` arm — adding a variant
|
||||
// on either side fails compilation here until the mapping is reconsidered;
|
||||
// - the round-trips assert the string token survives both directions — a
|
||||
// variant whose token the other side does not recognize fails the assert
|
||||
// instead of quietly becoming `Empty`.
|
||||
//
|
||||
// Struct-shaped drift on `ReplicationState` is already compile-guarded by the
|
||||
// exhaustive struct literals in the two conversion functions above; the
|
||||
// round-trip test below additionally pins value fidelity for every field.
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[test]
|
||||
fn replication_status_variants_round_trip_across_boundary() {
|
||||
use rustfs_replication::ReplicationStatusType as Repl;
|
||||
|
||||
let all = [
|
||||
Repl::Pending,
|
||||
Repl::Completed,
|
||||
Repl::CompletedLegacy,
|
||||
Repl::Failed,
|
||||
Repl::Replica,
|
||||
Repl::Empty,
|
||||
];
|
||||
for status in all {
|
||||
// Exhaustive on the replication side: a new variant breaks this match.
|
||||
match status {
|
||||
Repl::Pending | Repl::Completed | Repl::CompletedLegacy | Repl::Failed | Repl::Replica | Repl::Empty => {}
|
||||
}
|
||||
let filemeta = replication_status_to_filemeta(status.clone());
|
||||
assert_eq!(
|
||||
filemeta.as_str(),
|
||||
status.as_str(),
|
||||
"replication->filemeta conversion must not degrade {status:?} (unknown tokens fall back to Empty)"
|
||||
);
|
||||
assert_eq!(replication_status_from_filemeta(filemeta), status);
|
||||
}
|
||||
|
||||
// Exhaustive on the filemeta side: a new variant breaks this match.
|
||||
fn _filemeta_side_is_covered(status: rustfs_filemeta::ReplicationStatusType) {
|
||||
use rustfs_filemeta::ReplicationStatusType as Meta;
|
||||
match status {
|
||||
Meta::Pending | Meta::Completed | Meta::CompletedLegacy | Meta::Failed | Meta::Replica | Meta::Empty => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_purge_status_variants_round_trip_across_boundary() {
|
||||
use rustfs_replication::VersionPurgeStatusType as Repl;
|
||||
|
||||
let all = [Repl::Pending, Repl::Complete, Repl::Failed, Repl::Empty];
|
||||
for status in all {
|
||||
// Exhaustive on the replication side: a new variant breaks this match.
|
||||
match status {
|
||||
Repl::Pending | Repl::Complete | Repl::Failed | Repl::Empty => {}
|
||||
}
|
||||
let filemeta = version_purge_status_to_filemeta(status.clone());
|
||||
assert_eq!(
|
||||
filemeta.as_str(),
|
||||
status.as_str(),
|
||||
"replication->filemeta conversion must not degrade {status:?} (unknown tokens fall back to Empty)"
|
||||
);
|
||||
assert_eq!(version_purge_status_from_filemeta(filemeta), status);
|
||||
}
|
||||
|
||||
// Exhaustive on the filemeta side: a new variant breaks this match.
|
||||
fn _filemeta_side_is_covered(status: rustfs_filemeta::VersionPurgeStatusType) {
|
||||
use rustfs_filemeta::VersionPurgeStatusType as Meta;
|
||||
match status {
|
||||
Meta::Pending | Meta::Complete | Meta::Failed | Meta::Empty => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_state_round_trips_every_field_across_boundary() {
|
||||
let timestamp = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid timestamp");
|
||||
let state = ReplicationState {
|
||||
replica_timestamp: Some(timestamp),
|
||||
replica_status: ReplicationStatusType::Replica,
|
||||
delete_marker: true,
|
||||
replication_timestamp: Some(timestamp),
|
||||
replication_status_internal: Some("arn:a=PENDING;".to_string()),
|
||||
version_purge_status_internal: Some("arn:a=FAILED;".to_string()),
|
||||
replicate_decision_str: "arn:a=true;false;;".to_string(),
|
||||
targets: HashMap::from([
|
||||
("arn:a".to_string(), ReplicationStatusType::Completed),
|
||||
("arn:b".to_string(), ReplicationStatusType::Failed),
|
||||
]),
|
||||
purge_targets: HashMap::from([("arn:a".to_string(), VersionPurgeStatusType::Pending)]),
|
||||
reset_statuses_map: HashMap::from([("reset-arn:a".to_string(), "reset-id;ts".to_string())]),
|
||||
target_delete_marker_version_ids: HashMap::from([("arn:a".to_string(), "version-1".to_string())]),
|
||||
target_delete_marker_version_ids_corrupt: true,
|
||||
};
|
||||
|
||||
let round_tripped = replication_state_from_filemeta(&replication_state_to_filemeta(&state));
|
||||
assert_eq!(round_tripped, state);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2550,12 +2550,6 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
|
||||
Ok((put_opts, is_mp)) => (put_opts, is_mp),
|
||||
Err(e) => {
|
||||
// Unsupported source metadata (e.g. managed SSE) is a fail-closed
|
||||
// condition: report FAILED so the composite status and the
|
||||
// OperationFailedReplication event reflect that nothing reached
|
||||
// the target, instead of leaking the optimistic Completed above.
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -2960,11 +2954,6 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
|
||||
Ok((put_opts, is_mp)) => (put_opts, is_mp),
|
||||
Err(e) => {
|
||||
// Unsupported source metadata (e.g. managed SSE) is a fail-closed
|
||||
// condition: report FAILED so the composite status and the
|
||||
// OperationFailedReplication event reflect that nothing reached
|
||||
// the target, instead of leaking the optimistic Completed above.
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
|
||||
@@ -56,59 +56,11 @@ impl FromStr for ARN {
|
||||
if parts.len() != 6 {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
|
||||
}
|
||||
// Display emits `arn:rustfs:{type}:{region}:{id}:{bucket}`; read the
|
||||
// segments back in the same order so parse(display(a)) == a.
|
||||
Ok(ARN {
|
||||
arn_type: BucketTargetType::from_str(parts[2]).unwrap_or_default(),
|
||||
region: parts[3].to_string(),
|
||||
id: parts[4].to_string(),
|
||||
id: parts[3].to_string(),
|
||||
region: parts[4].to_string(),
|
||||
bucket: parts[5].to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Display emits `arn:rustfs:{type}:{region}:{id}:{bucket}` (madmin layout);
|
||||
/// FromStr must read the same positions back so parse(display(a)) == a.
|
||||
#[test]
|
||||
fn from_str_round_trips_display_with_region_and_id() {
|
||||
let arn = ARN::new(
|
||||
BucketTargetType::ReplicationService,
|
||||
"depl-123".to_string(),
|
||||
"us-east-1".to_string(),
|
||||
"bucket-a".to_string(),
|
||||
);
|
||||
|
||||
let parsed = ARN::from_str(&arn.to_string()).expect("display output must parse");
|
||||
|
||||
assert_eq!(parsed.arn_type, arn.arn_type);
|
||||
assert_eq!(parsed.region, arn.region, "region must survive display->parse round-trip");
|
||||
assert_eq!(parsed.id, arn.id, "id must survive display->parse round-trip");
|
||||
assert_eq!(parsed.bucket, arn.bucket);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_str_reads_region_then_id_in_display_order() {
|
||||
let parsed = ARN::from_str("arn:rustfs:replication:us-east-1:depl-123:bucket-a").expect("valid ARN must parse");
|
||||
|
||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||
assert_eq!(parsed.region, "us-east-1");
|
||||
assert_eq!(parsed.id, "depl-123");
|
||||
assert_eq!(parsed.bucket, "bucket-a");
|
||||
}
|
||||
|
||||
/// RustFS commonly generates ARNs with an empty region:
|
||||
/// `arn:rustfs:replication::<deployment_id>:<bucket>`.
|
||||
#[test]
|
||||
fn from_str_handles_empty_region_segment() {
|
||||
let parsed = ARN::from_str("arn:rustfs:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
||||
|
||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||
assert_eq!(parsed.region, "", "region segment is empty in this form");
|
||||
assert_eq!(parsed.id, "depl-123");
|
||||
assert_eq!(parsed.bucket, "bucket-a");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use jiff::Timestamp;
|
||||
use rmp_serde::Serializer as rmpSerializer;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
@@ -33,7 +32,7 @@ pub struct Credentials {
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub session_token: Option<String>,
|
||||
pub expiration: Option<Timestamp>,
|
||||
pub expiration: Option<chrono::DateTime<chrono::Utc>>,
|
||||
}
|
||||
|
||||
impl Credentials {
|
||||
@@ -94,21 +93,6 @@ mod duration_milliseconds {
|
||||
}
|
||||
}
|
||||
|
||||
/// Defensive decode for the two integer wire encodings of these duration
|
||||
/// fields: RustFS persists (and legacy RustFS clients sent) plain seconds,
|
||||
/// while Go `time.Duration` JSON — madmin/mc requests and MinIO-written
|
||||
/// bucket-targets metadata — is nanoseconds. No meaningful interval lies
|
||||
/// between 10^7 seconds (~115 days) and 10^7 nanoseconds (10ms), so the
|
||||
/// magnitude disambiguates the unit.
|
||||
pub fn duration_from_secs_or_nanos(value: u64) -> Duration {
|
||||
const NANOS_THRESHOLD: u64 = 10_000_000;
|
||||
if value < NANOS_THRESHOLD {
|
||||
Duration::from_secs(value)
|
||||
} else {
|
||||
Duration::from_nanos(value)
|
||||
}
|
||||
}
|
||||
|
||||
mod duration_seconds {
|
||||
use serde::{Deserialize, Deserializer, Serializer};
|
||||
use std::time::Duration;
|
||||
@@ -124,8 +108,8 @@ mod duration_seconds {
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let value = u64::deserialize(deserializer)?;
|
||||
Ok(super::duration_from_secs_or_nanos(value))
|
||||
let secs = u64::deserialize(deserializer)?;
|
||||
Ok(Duration::from_secs(secs))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -424,11 +408,7 @@ mod tests {
|
||||
assert_eq!(credentials.access_key, "test-access-key");
|
||||
assert_eq!(credentials.secret_key, "test-secret-key");
|
||||
assert_eq!(credentials.session_token, Some("test-session-token".to_string()));
|
||||
assert_eq!(
|
||||
serde_json::to_value(credentials.expiration.expect("expiration should parse"))
|
||||
.expect("expiration should serialize to JSON"),
|
||||
serde_json::json!("2024-12-31T23:59:59Z")
|
||||
);
|
||||
assert!(credentials.expiration.is_some());
|
||||
|
||||
// Verify latency statistics
|
||||
assert_eq!(target.latency.curr, Duration::from_millis(100));
|
||||
@@ -504,29 +484,6 @@ mod tests {
|
||||
assert_eq!(original.offline_count, deserialized.offline_count);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_target_reads_go_nanosecond_durations_defensively() {
|
||||
// MinIO-written bucket-targets metadata and madmin clients encode
|
||||
// these fields as Go `time.Duration` nanoseconds; RustFS has always
|
||||
// persisted seconds. Both encodings must decode to the same interval.
|
||||
let target: BucketTarget = serde_json::from_value(serde_json::json!({
|
||||
"endpoint": "localhost:9000",
|
||||
"targetbucket": "target",
|
||||
"type": "replication",
|
||||
"healthCheckDuration": 60_000_000_000u64,
|
||||
"totalDowntime": 90_000_000_000u64
|
||||
}))
|
||||
.expect("nanosecond durations should deserialize");
|
||||
|
||||
assert_eq!(target.health_check_duration, Duration::from_secs(60));
|
||||
assert_eq!(target.total_downtime, Duration::from_secs(90));
|
||||
|
||||
// The persisted wire format stays seconds for existing RustFS readers.
|
||||
let value = serde_json::to_value(&target).expect("target should serialize");
|
||||
assert_eq!(value["healthCheckDuration"], 60);
|
||||
assert_eq!(value["totalDowntime"], 90);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_target_debug_redacts_credentials() {
|
||||
let target = BucketTarget {
|
||||
@@ -605,15 +562,12 @@ mod tests {
|
||||
.and_then(|credentials| credentials.session_token.as_deref()),
|
||||
Some("legacy-session-token")
|
||||
);
|
||||
assert_eq!(
|
||||
assert!(
|
||||
target
|
||||
.credentials
|
||||
.as_ref()
|
||||
.and_then(|credentials| credentials.expiration)
|
||||
.map(serde_json::to_value)
|
||||
.transpose()
|
||||
.expect("expiration should serialize to JSON"),
|
||||
Some(serde_json::json!("2024-12-31T23:59:59Z"))
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -655,11 +609,7 @@ mod tests {
|
||||
credentials.session_token,
|
||||
Some("AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(credentials.expiration.expect("expiration should parse"))
|
||||
.expect("expiration should serialize to JSON"),
|
||||
serde_json::json!("2024-12-31T23:59:59Z")
|
||||
);
|
||||
assert!(credentials.expiration.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -269,32 +269,6 @@ pub fn check_del_obj_args(bucket: &str, object: &str) -> Result<()> {
|
||||
check_bucket_and_object_names(bucket, object)
|
||||
}
|
||||
|
||||
/// Filesystem `NAME_MAX`: every object-key path segment becomes one on-disk
|
||||
/// directory entry, so a longer segment can never be stored and previously
|
||||
/// escaped as an `ENAMETOOLONG` io error → `InternalError` 500 (rustfs#5785).
|
||||
const MAX_OBJECT_KEY_SEGMENT_BYTES: usize = 255;
|
||||
|
||||
/// Reject object keys whose on-disk directory names would exceed `NAME_MAX`.
|
||||
///
|
||||
/// Middle segments map to their raw bytes; the final segment of a
|
||||
/// directory-object key (trailing `/`) is stored with the `__XLDIR__` suffix
|
||||
/// appended, shrinking its budget accordingly.
|
||||
fn object_key_segments_fit_on_disk(object: &str) -> bool {
|
||||
let trailing_dir = object.ends_with('/');
|
||||
let segments: Vec<&str> = object.split('/').collect();
|
||||
let last_nonempty = segments.iter().rposition(|s| !s.is_empty());
|
||||
for (index, segment) in segments.iter().enumerate() {
|
||||
let mut budget = MAX_OBJECT_KEY_SEGMENT_BYTES;
|
||||
if trailing_dir && Some(index) == last_nonempty {
|
||||
budget = budget.saturating_sub(rustfs_utils::path::GLOBAL_DIR_SUFFIX.len());
|
||||
}
|
||||
if segment.len() > budget {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub fn check_bucket_and_object_names(bucket: &str, object: &str) -> Result<()> {
|
||||
if !is_meta_bucketname(bucket) && check_valid_bucket_name_strict(bucket).is_err() {
|
||||
return Err(StorageError::BucketNameInvalid(bucket.to_string()));
|
||||
@@ -308,10 +282,6 @@ pub fn check_bucket_and_object_names(bucket: &str, object: &str) -> Result<()> {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
|
||||
if !object_key_segments_fit_on_disk(object) {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
|
||||
// if cfg!(target_os = "windows") && object.contains('\\') {
|
||||
// return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
|
||||
// }
|
||||
@@ -409,14 +379,6 @@ pub fn check_put_object_args(bucket: &str, object: &str) -> Result<()> {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
|
||||
// The write path validates arguments here rather than through
|
||||
// check_bucket_and_object_names, so the on-disk segment budget has to be
|
||||
// enforced in both places or an over-NAME_MAX key still reaches the disk
|
||||
// layer and escapes as ENAMETOOLONG → InternalError 500 (rustfs#5785).
|
||||
if !object_key_segments_fit_on_disk(object) {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -425,62 +387,6 @@ mod tests {
|
||||
use super::*;
|
||||
use proptest::prelude::*;
|
||||
|
||||
/// rustfs#5785: keys whose path segments exceed the on-disk NAME_MAX
|
||||
/// budget must be rejected up front as ObjectNameInvalid (4xx), not leak
|
||||
/// ENAMETOOLONG as InternalError 500 from the disk layer.
|
||||
#[test]
|
||||
fn object_key_segment_name_max_budget() {
|
||||
// 255-byte single segment: exactly at the on-disk limit.
|
||||
assert!(check_bucket_and_object_names("bucket", &"a".repeat(255)).is_ok());
|
||||
// 256 bytes: one over.
|
||||
assert!(matches!(
|
||||
check_bucket_and_object_names("bucket", &"a".repeat(256)),
|
||||
Err(StorageError::ObjectNameInvalid(_, _))
|
||||
));
|
||||
// Long keys are fine as long as every segment fits.
|
||||
let segmented = ["b".repeat(200), "c".repeat(200), "d".repeat(200)].join("/");
|
||||
assert!(check_bucket_and_object_names("bucket", &segmented).is_ok());
|
||||
// The budget counts bytes, not characters (100 CJK chars = 300 bytes).
|
||||
assert!(matches!(
|
||||
check_bucket_and_object_names("bucket", &"中".repeat(100)),
|
||||
Err(StorageError::ObjectNameInvalid(_, _))
|
||||
));
|
||||
assert!(check_bucket_and_object_names("bucket", &"中".repeat(85)).is_ok());
|
||||
// Directory-object keys spend GLOBAL_DIR_SUFFIX bytes of the final
|
||||
// segment's budget on the on-disk __XLDIR__ encoding.
|
||||
let dir_budget = 255 - rustfs_utils::path::GLOBAL_DIR_SUFFIX.len();
|
||||
assert!(check_bucket_and_object_names("bucket", &format!("{}/", "e".repeat(dir_budget))).is_ok());
|
||||
assert!(matches!(
|
||||
check_bucket_and_object_names("bucket", &format!("{}/", "e".repeat(dir_budget + 1))),
|
||||
Err(StorageError::ObjectNameInvalid(_, _))
|
||||
));
|
||||
}
|
||||
|
||||
/// rustfs#5785 follow-up: the write path validates through
|
||||
/// check_put_object_args, not check_bucket_and_object_names, so the same
|
||||
/// budget has to hold there — otherwise an over-NAME_MAX PUT still reached
|
||||
/// the disk layer and came back as InternalError 500.
|
||||
#[test]
|
||||
fn put_object_args_enforce_the_same_segment_budget() {
|
||||
assert!(check_put_object_args("bucket", &"a".repeat(255)).is_ok());
|
||||
assert!(matches!(
|
||||
check_put_object_args("bucket", &"a".repeat(256)),
|
||||
Err(StorageError::ObjectNameInvalid(_, _))
|
||||
));
|
||||
assert!(matches!(
|
||||
check_put_object_args("bucket", &"\u{4e2d}".repeat(100)),
|
||||
Err(StorageError::ObjectNameInvalid(_, _))
|
||||
));
|
||||
let segmented = ["b".repeat(200), "c".repeat(200), "d".repeat(200)].join("/");
|
||||
assert!(check_put_object_args("bucket", &segmented).is_ok());
|
||||
let dir_budget = 255 - rustfs_utils::path::GLOBAL_DIR_SUFFIX.len();
|
||||
assert!(check_put_object_args("bucket", &format!("{}/", "e".repeat(dir_budget))).is_ok());
|
||||
assert!(matches!(
|
||||
check_put_object_args("bucket", &format!("{}/", "e".repeat(dir_budget + 1))),
|
||||
Err(StorageError::ObjectNameInvalid(_, _))
|
||||
));
|
||||
}
|
||||
|
||||
// Test validation functions
|
||||
#[test]
|
||||
fn test_is_valid_object_name() {
|
||||
|
||||
@@ -36,20 +36,15 @@ use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
#[cfg(test)]
|
||||
use rustfs_credentials::{DEFAULT_SECRET_KEY, RPC_SECRET_REQUIRED_MESSAGE};
|
||||
use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_GRPC_OTHER, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics,
|
||||
};
|
||||
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use rustfs_utils::get_env_bool;
|
||||
use sha2::Digest as _;
|
||||
use sha2::Sha256;
|
||||
use std::collections::{HashSet, VecDeque};
|
||||
use std::sync::{LazyLock, Mutex, Once};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{error, info, warn};
|
||||
use tracing::error;
|
||||
use uuid::Uuid;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
@@ -75,11 +70,6 @@ const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
|
||||
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
||||
const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
|
||||
const REPLAY_CACHE_RETENTION_SECS: usize = 601;
|
||||
const REPLAY_CACHE_ENTRY_BYTES_ESTIMATE: u64 = 128;
|
||||
const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 8;
|
||||
const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 2048;
|
||||
const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 16_777_216;
|
||||
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
|
||||
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
|
||||
static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
@@ -101,211 +91,18 @@ static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
)
|
||||
});
|
||||
// Sized for peak legitimate authenticated RPC RPS x the retention window once replay scope is
|
||||
// active; overflow fails closed and increments the replay-cache overflow counter. Explicit operator
|
||||
// values and auto-sizing are both floored at the historical default so under-sizing cannot turn
|
||||
// legitimate high-throughput traffic into `No valid auth token` failures.
|
||||
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(resolve_replay_cache_capacity);
|
||||
// active; overflow fails closed and increments the replay-cache overflow counter. Clamped to at
|
||||
// least 1 so a misconfigured zero cannot disable replay protection by rejecting every request.
|
||||
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
)
|
||||
.max(1)
|
||||
});
|
||||
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
|
||||
static RPC_BOOT_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ReplayCacheCapacitySource {
|
||||
Env,
|
||||
EnvClampedToDefault,
|
||||
Auto,
|
||||
AutoClampedToDefault,
|
||||
AutoInvalidEnv,
|
||||
AutoInvalidEnvClampedToDefault,
|
||||
}
|
||||
|
||||
impl ReplayCacheCapacitySource {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Env => "env",
|
||||
Self::EnvClampedToDefault => "env_clamped_to_default",
|
||||
Self::Auto => "auto",
|
||||
Self::AutoClampedToDefault => "auto_clamped_to_default",
|
||||
Self::AutoInvalidEnv => "auto_invalid_env",
|
||||
Self::AutoInvalidEnvClampedToDefault => "auto_invalid_env_clamped_to_default",
|
||||
}
|
||||
}
|
||||
|
||||
fn is_env_clamped(self) -> bool {
|
||||
matches!(self, Self::EnvClampedToDefault)
|
||||
}
|
||||
|
||||
fn is_env(self) -> bool {
|
||||
matches!(self, Self::Env)
|
||||
}
|
||||
|
||||
fn is_invalid_env(self) -> bool {
|
||||
matches!(self, Self::AutoInvalidEnv | Self::AutoInvalidEnvClampedToDefault)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct ReplayCacheCapacityDecision {
|
||||
capacity: usize,
|
||||
source: ReplayCacheCapacitySource,
|
||||
cpu_count: usize,
|
||||
memory_limit_bytes: Option<u64>,
|
||||
memory_basis: Option<MemoryBasis>,
|
||||
memory_based_capacity: usize,
|
||||
cpu_based_capacity: usize,
|
||||
}
|
||||
|
||||
fn saturating_usize_from_u64(value: u64) -> usize {
|
||||
usize::try_from(value).unwrap_or(usize::MAX)
|
||||
}
|
||||
|
||||
fn replay_cache_capacity_from_resources(cpu_count: usize, memory_limit_bytes: Option<u64>) -> (usize, usize, usize) {
|
||||
let cpu_count = cpu_count.max(1);
|
||||
let cpu_based_capacity = cpu_count
|
||||
.saturating_mul(REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU)
|
||||
.saturating_mul(REPLAY_CACHE_RETENTION_SECS);
|
||||
let memory_based_capacity = memory_limit_bytes
|
||||
.map(|bytes| {
|
||||
let budget = bytes.saturating_mul(REPLAY_CACHE_AUTO_MEMORY_PERCENT) / 100;
|
||||
saturating_usize_from_u64(budget / REPLAY_CACHE_ENTRY_BYTES_ESTIMATE)
|
||||
})
|
||||
.unwrap_or(REPLAY_CACHE_AUTO_MAX_CAPACITY);
|
||||
let capacity = memory_based_capacity
|
||||
.min(cpu_based_capacity)
|
||||
.clamp(rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, REPLAY_CACHE_AUTO_MAX_CAPACITY);
|
||||
(capacity, memory_based_capacity, cpu_based_capacity)
|
||||
}
|
||||
|
||||
fn replay_cache_capacity_decision(
|
||||
env: rustfs_utils::EnvParseOutcome<usize>,
|
||||
cpu_count: usize,
|
||||
memory_limit_bytes: Option<u64>,
|
||||
memory_basis: Option<MemoryBasis>,
|
||||
) -> ReplayCacheCapacityDecision {
|
||||
let default = rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY;
|
||||
match env {
|
||||
rustfs_utils::EnvParseOutcome::Parsed(configured) => {
|
||||
let capacity = configured.max(default);
|
||||
let source = if configured < default {
|
||||
ReplayCacheCapacitySource::EnvClampedToDefault
|
||||
} else {
|
||||
ReplayCacheCapacitySource::Env
|
||||
};
|
||||
ReplayCacheCapacityDecision {
|
||||
capacity,
|
||||
source,
|
||||
cpu_count: cpu_count.max(1),
|
||||
memory_limit_bytes,
|
||||
memory_basis,
|
||||
memory_based_capacity: 0,
|
||||
cpu_based_capacity: 0,
|
||||
}
|
||||
}
|
||||
rustfs_utils::EnvParseOutcome::Absent | rustfs_utils::EnvParseOutcome::Invalid => {
|
||||
let (capacity, memory_based_capacity, cpu_based_capacity) =
|
||||
replay_cache_capacity_from_resources(cpu_count, memory_limit_bytes);
|
||||
let clamped_to_default = capacity == default && memory_based_capacity.min(cpu_based_capacity) < default;
|
||||
let invalid_env = matches!(env, rustfs_utils::EnvParseOutcome::Invalid);
|
||||
let source = match (invalid_env, clamped_to_default) {
|
||||
(true, true) => ReplayCacheCapacitySource::AutoInvalidEnvClampedToDefault,
|
||||
(true, false) => ReplayCacheCapacitySource::AutoInvalidEnv,
|
||||
(false, true) => ReplayCacheCapacitySource::AutoClampedToDefault,
|
||||
(false, false) => ReplayCacheCapacitySource::Auto,
|
||||
};
|
||||
ReplayCacheCapacityDecision {
|
||||
capacity,
|
||||
source,
|
||||
cpu_count: cpu_count.max(1),
|
||||
memory_limit_bytes,
|
||||
memory_basis,
|
||||
memory_based_capacity,
|
||||
cpu_based_capacity,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn detected_replay_cache_resources() -> (usize, Option<u64>, Option<MemoryBasis>) {
|
||||
let cpu_count = thread::available_parallelism().map(usize::from).unwrap_or(1).max(1);
|
||||
let memory = resolve_effective_memory();
|
||||
let memory_limit_bytes = (memory.total_bytes > 0).then_some(memory.total_bytes);
|
||||
(cpu_count, memory_limit_bytes, Some(memory.basis))
|
||||
}
|
||||
|
||||
fn log_replay_cache_capacity_decision(decision: ReplayCacheCapacityDecision) {
|
||||
let source = decision.source.as_str();
|
||||
if decision.source.is_env_clamped() {
|
||||
warn!(
|
||||
event = "internode_rpc_replay_cache_capacity_resolved",
|
||||
component = "ecstore",
|
||||
subsystem = "rpc_auth",
|
||||
capacity = decision.capacity,
|
||||
source,
|
||||
default_capacity = rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
env = rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
"internode rpc replay cache capacity clamped to default"
|
||||
);
|
||||
return;
|
||||
}
|
||||
if decision.source.is_env() {
|
||||
info!(
|
||||
event = "internode_rpc_replay_cache_capacity_resolved",
|
||||
component = "ecstore",
|
||||
subsystem = "rpc_auth",
|
||||
capacity = decision.capacity,
|
||||
source,
|
||||
default_capacity = rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
env = rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
"internode rpc replay cache capacity resolved from env"
|
||||
);
|
||||
return;
|
||||
}
|
||||
if decision.source.is_invalid_env() {
|
||||
warn!(
|
||||
event = "internode_rpc_replay_cache_capacity_resolved",
|
||||
component = "ecstore",
|
||||
subsystem = "rpc_auth",
|
||||
capacity = decision.capacity,
|
||||
source,
|
||||
cpu_count = decision.cpu_count,
|
||||
memory_limit_bytes = decision.memory_limit_bytes,
|
||||
memory_basis = decision.memory_basis.map(MemoryBasis::as_str),
|
||||
memory_based_capacity = decision.memory_based_capacity,
|
||||
cpu_based_capacity = decision.cpu_based_capacity,
|
||||
auto_max_capacity = REPLAY_CACHE_AUTO_MAX_CAPACITY,
|
||||
env = rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
"internode rpc replay cache capacity auto-sized after invalid env"
|
||||
);
|
||||
return;
|
||||
}
|
||||
info!(
|
||||
event = "internode_rpc_replay_cache_capacity_resolved",
|
||||
component = "ecstore",
|
||||
subsystem = "rpc_auth",
|
||||
capacity = decision.capacity,
|
||||
source,
|
||||
cpu_count = decision.cpu_count,
|
||||
memory_limit_bytes = decision.memory_limit_bytes,
|
||||
memory_basis = decision.memory_basis.map(MemoryBasis::as_str),
|
||||
memory_based_capacity = decision.memory_based_capacity,
|
||||
cpu_based_capacity = decision.cpu_based_capacity,
|
||||
auto_max_capacity = REPLAY_CACHE_AUTO_MAX_CAPACITY,
|
||||
"internode rpc replay cache capacity resolved"
|
||||
);
|
||||
}
|
||||
|
||||
fn resolve_replay_cache_capacity() -> usize {
|
||||
let (cpu_count, memory_limit_bytes, memory_basis) = detected_replay_cache_resources();
|
||||
let decision = replay_cache_capacity_decision(
|
||||
rustfs_utils::get_env_parse_outcome(rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY),
|
||||
cpu_count,
|
||||
memory_limit_bytes,
|
||||
memory_basis,
|
||||
);
|
||||
global_internode_metrics().record_replay_cache_state(0, decision.capacity);
|
||||
log_replay_cache_capacity_decision(decision);
|
||||
decision.capacity
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RpcNonceCache {
|
||||
nonces: HashSet<Uuid>,
|
||||
@@ -313,50 +110,8 @@ struct RpcNonceCache {
|
||||
max_wall_time: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct RpcReplayCacheMetricScope<'a> {
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
rpc_path: &'a str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct RpcNonceRecord<'a> {
|
||||
nonce: Uuid,
|
||||
signed_at: i64,
|
||||
now: Instant,
|
||||
wall_time: i64,
|
||||
expires_at: Instant,
|
||||
capacity: usize,
|
||||
metric_scope: RpcReplayCacheMetricScope<'a>,
|
||||
}
|
||||
|
||||
struct RpcNonceCacheMetrics<'a> {
|
||||
expired: usize,
|
||||
entries: usize,
|
||||
capacity: usize,
|
||||
overflow_scope: Option<RpcReplayCacheMetricScope<'a>>,
|
||||
}
|
||||
|
||||
fn publish_nonce_cache_metrics(metrics: Option<RpcNonceCacheMetrics<'_>>) {
|
||||
let Some(metrics) = metrics else {
|
||||
return;
|
||||
};
|
||||
let internode_metrics = global_internode_metrics();
|
||||
internode_metrics.record_replay_cache_evictions("expired", metrics.expired);
|
||||
internode_metrics.record_replay_cache_state(metrics.entries, metrics.capacity);
|
||||
if let Some(scope) = metrics.overflow_scope {
|
||||
internode_metrics.record_replay_cache_overflow_for_operation_and_backend_path(
|
||||
scope.operation,
|
||||
scope.backend,
|
||||
scope.rpc_path,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl RpcNonceCache {
|
||||
fn remove_expired(&mut self, now: Instant, wall_time: i64) -> usize {
|
||||
let mut removed = 0;
|
||||
fn remove_expired(&mut self, now: Instant, wall_time: i64) {
|
||||
while matches!(
|
||||
self.expirations.front(),
|
||||
Some((expires_at, valid_until, _)) if *expires_at < now && *valid_until < wall_time
|
||||
@@ -365,48 +120,37 @@ impl RpcNonceCache {
|
||||
break;
|
||||
};
|
||||
self.nonces.remove(&nonce);
|
||||
removed += 1;
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
fn check_and_record<'a>(&mut self, record: RpcNonceRecord<'a>) -> (std::io::Result<()>, Option<RpcNonceCacheMetrics<'a>>) {
|
||||
self.max_wall_time = self.max_wall_time.max(record.wall_time);
|
||||
if self.max_wall_time.saturating_sub(record.signed_at) > SIGNATURE_VALID_DURATION {
|
||||
return (Err(std::io::Error::other("RPC request timestamp expired after clock regression")), None);
|
||||
fn check_and_record(
|
||||
&mut self,
|
||||
nonce: Uuid,
|
||||
signed_at: i64,
|
||||
now: Instant,
|
||||
wall_time: i64,
|
||||
expires_at: Instant,
|
||||
capacity: usize,
|
||||
) -> std::io::Result<()> {
|
||||
self.max_wall_time = self.max_wall_time.max(wall_time);
|
||||
if self.max_wall_time.saturating_sub(signed_at) > SIGNATURE_VALID_DURATION {
|
||||
return Err(std::io::Error::other("RPC request timestamp expired after clock regression"));
|
||||
}
|
||||
let expired = self.remove_expired(record.now, self.max_wall_time);
|
||||
let metrics = RpcNonceCacheMetrics {
|
||||
expired,
|
||||
entries: self.nonces.len(),
|
||||
capacity: record.capacity,
|
||||
overflow_scope: None,
|
||||
};
|
||||
if self.nonces.contains(&record.nonce) {
|
||||
return (Err(std::io::Error::other("RPC request replay detected")), Some(metrics));
|
||||
self.remove_expired(now, self.max_wall_time);
|
||||
if self.nonces.contains(&nonce) {
|
||||
return Err(std::io::Error::other("RPC request replay detected"));
|
||||
}
|
||||
if self.nonces.len() >= record.capacity {
|
||||
if self.nonces.len() >= capacity {
|
||||
// Fail closed and alert: only legitimately signed traffic can fill the cache, so a
|
||||
// sustained overflow means RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY is undersized
|
||||
// for this node's peak mutation rate and writes are being refused.
|
||||
return (
|
||||
Err(std::io::Error::other("RPC replay cache capacity exceeded")),
|
||||
Some(RpcNonceCacheMetrics {
|
||||
overflow_scope: Some(record.metric_scope),
|
||||
..metrics
|
||||
}),
|
||||
);
|
||||
global_internode_metrics().record_replay_cache_overflow();
|
||||
return Err(std::io::Error::other("RPC replay cache capacity exceeded"));
|
||||
}
|
||||
self.nonces.insert(record.nonce);
|
||||
self.nonces.insert(nonce);
|
||||
self.expirations
|
||||
.push_back((record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce));
|
||||
(
|
||||
Ok(()),
|
||||
Some(RpcNonceCacheMetrics {
|
||||
entries: self.nonces.len(),
|
||||
..metrics
|
||||
}),
|
||||
)
|
||||
.push_back((expires_at, signed_at.saturating_add(SIGNATURE_VALID_DURATION), nonce));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -797,43 +541,18 @@ fn check_timestamp(timestamp: i64) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn tonic_rpc_metric_operation(path: &str) -> &'static str {
|
||||
match parse_tonic_rpc_path(path).ok().map(|(_, rpc_method)| rpc_method) {
|
||||
Some("ReadAll") => INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||
Some("ReadMultiple") => INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
Some("WriteAll") => INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
_ => INTERNODE_OPERATION_GRPC_OTHER,
|
||||
}
|
||||
}
|
||||
|
||||
fn check_and_record_nonce(nonce: Uuid, signed_at: i64, rpc_path: &str) -> std::io::Result<()> {
|
||||
fn check_and_record_nonce(nonce: Uuid, signed_at: i64) -> std::io::Result<()> {
|
||||
let wall_time = OffsetDateTime::now_utc().unix_timestamp();
|
||||
let (result, metrics) = {
|
||||
let mut cache = LOCAL_RPC_NONCE_CACHE
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("RPC replay cache unavailable"))?;
|
||||
// Take the monotonic timestamp after acquiring the lock so expiration
|
||||
// entries remain ordered by the same serialization point as insertion.
|
||||
let now = Instant::now();
|
||||
let expires_at = now
|
||||
.checked_add(REPLAY_CACHE_RETENTION)
|
||||
.ok_or_else(|| std::io::Error::other("RPC replay expiry overflow"))?;
|
||||
cache.check_and_record(RpcNonceRecord {
|
||||
nonce,
|
||||
signed_at,
|
||||
now,
|
||||
wall_time,
|
||||
expires_at,
|
||||
capacity: *REPLAY_CACHE_CAPACITY,
|
||||
metric_scope: RpcReplayCacheMetricScope {
|
||||
operation: tonic_rpc_metric_operation(rpc_path),
|
||||
backend: INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
rpc_path,
|
||||
},
|
||||
})
|
||||
};
|
||||
publish_nonce_cache_metrics(metrics);
|
||||
result
|
||||
let mut cache = LOCAL_RPC_NONCE_CACHE
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("RPC replay cache unavailable"))?;
|
||||
// Take the monotonic timestamp after acquiring the lock so expiration
|
||||
// entries remain ordered by the same serialization point as insertion.
|
||||
let now = Instant::now();
|
||||
let expires_at = now
|
||||
.checked_add(REPLAY_CACHE_RETENTION)
|
||||
.ok_or_else(|| std::io::Error::other("RPC replay expiry overflow"))?;
|
||||
cache.check_and_record(nonce, signed_at, now, wall_time, expires_at, *REPLAY_CACHE_CAPACITY)
|
||||
}
|
||||
|
||||
/// Build headers with authentication signature
|
||||
@@ -1095,7 +814,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
|
||||
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||
}
|
||||
check_and_record_nonce(nonce, signed_at, path)
|
||||
check_and_record_nonce(nonce, signed_at)
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
@@ -1128,46 +847,6 @@ pub fn verify_tonic_rpc_signature_with_bootstrap(
|
||||
)
|
||||
}
|
||||
|
||||
pub fn tonic_rpc_auth_failure_reason(error: &std::io::Error) -> &'static str {
|
||||
match error.to_string().as_str() {
|
||||
"Missing RPC audience" => "missing_audience",
|
||||
"Invalid RPC request path" => "invalid_request_path",
|
||||
"RPC replay-scoped authentication required" => "replay_scope_required",
|
||||
"Missing RPC replay scope version" => "missing_replay_scope_version",
|
||||
"Unsupported RPC replay scope version" => "unsupported_replay_scope_version",
|
||||
"Missing RPC replay scope signature" => "missing_replay_scope_signature",
|
||||
"Missing RPC replay scope nonce" => "missing_replay_scope_nonce",
|
||||
"Invalid RPC replay scope nonce" => "invalid_replay_scope_nonce",
|
||||
"Missing RPC boot epoch" => "missing_boot_epoch",
|
||||
"Invalid RPC boot epoch" => "invalid_boot_epoch",
|
||||
"Invalid RPC replay scope signature" => "invalid_replay_scope_signature",
|
||||
"RPC boot epoch is stale" => "stale_boot_epoch",
|
||||
"RPC request replay detected" => "replay_detected",
|
||||
"RPC replay cache capacity exceeded" => "replay_cache_capacity",
|
||||
"RPC replay cache unavailable" => "replay_cache_unavailable",
|
||||
"RPC replay expiry overflow" => "replay_expiry_overflow",
|
||||
"RPC request timestamp expired after clock regression" => "timestamp_expired_after_clock_regression",
|
||||
"RPC v2 authentication required" => "v2_required",
|
||||
"Missing RPC auth version" => "missing_v2_auth_version",
|
||||
"Unsupported RPC auth version" => "unsupported_v2_auth_version",
|
||||
"Missing RPC v2 signature" => "missing_v2_signature",
|
||||
"Invalid RPC v2 signature" => "invalid_v2_signature",
|
||||
"Missing timestamp header" => "missing_timestamp",
|
||||
"Invalid timestamp format" => "invalid_timestamp",
|
||||
"Request timestamp expired" => "timestamp_expired",
|
||||
"Missing RPC nonce" => "missing_v2_nonce",
|
||||
"Invalid RPC nonce" => "invalid_v2_nonce",
|
||||
"Invalid unsigned RPC nonce" => "invalid_unsigned_v2_nonce",
|
||||
"Missing RPC content SHA-256" => "missing_content_sha256",
|
||||
"Invalid RPC content SHA-256" => "invalid_content_sha256",
|
||||
"Missing signature header" => "missing_v1_signature",
|
||||
"Invalid signature" => "invalid_v1_signature",
|
||||
"Invalid RPC HMAC key" => "invalid_hmac_key",
|
||||
message if message.contains(RPC_SECRET_REQUIRED_OPERATOR_MESSAGE) => "missing_rpc_secret",
|
||||
_ => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
fn verify_tonic_rpc_signature_with_policy(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
@@ -1286,7 +965,7 @@ fn verify_tonic_rpc_signature_with_strictness(
|
||||
return Err(std::io::Error::other("Invalid RPC v2 signature"));
|
||||
}
|
||||
if let Some(nonce) = parsed_nonce {
|
||||
check_and_record_nonce(nonce, timestamp, path)?;
|
||||
check_and_record_nonce(nonce, timestamp)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -2020,31 +1699,6 @@ mod tests {
|
||||
assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_rpc_auth_failure_reason_maps_security_relevant_errors() {
|
||||
for (message, reason) in [
|
||||
("Invalid RPC v2 signature", "invalid_v2_signature"),
|
||||
("RPC replay-scoped authentication required", "replay_scope_required"),
|
||||
("Missing RPC replay scope signature", "missing_replay_scope_signature"),
|
||||
("RPC boot epoch is stale", "stale_boot_epoch"),
|
||||
("RPC request replay detected", "replay_detected"),
|
||||
("Request timestamp expired", "timestamp_expired"),
|
||||
("Missing RPC content SHA-256", "missing_content_sha256"),
|
||||
("Invalid RPC content SHA-256", "invalid_content_sha256"),
|
||||
] {
|
||||
assert_eq!(
|
||||
tonic_rpc_auth_failure_reason(&std::io::Error::other(message)),
|
||||
reason,
|
||||
"message {message:?} should map to a stable low-cardinality reason"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_rpc_auth_failure_reason_falls_back_for_unclassified_errors() {
|
||||
assert_eq!(tonic_rpc_auth_failure_reason(&std::io::Error::other("opaque failure")), "unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
|
||||
ensure_test_rpc_secret();
|
||||
@@ -2249,126 +1903,6 @@ mod tests {
|
||||
assert_eq!(error.to_string(), "RPC mutation requires v2 authentication");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_rpc_metric_operation_classifies_get_hot_path_methods() {
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/ReadAll"),
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/ReadMultiple"),
|
||||
INTERNODE_OPERATION_GRPC_READ_MULTIPLE
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/WriteAll"),
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/SignalService"),
|
||||
INTERNODE_OPERATION_GRPC_OTHER
|
||||
);
|
||||
assert_eq!(tonic_rpc_metric_operation("not-a-grpc-path"), INTERNODE_OPERATION_GRPC_OTHER);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_capacity_uses_env_with_default_floor() {
|
||||
let default = rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY;
|
||||
|
||||
let high = replay_cache_capacity_decision(
|
||||
rustfs_utils::EnvParseOutcome::Parsed(default * 16),
|
||||
2,
|
||||
Some(512 * 1024 * 1024),
|
||||
Some(MemoryBasis::Host),
|
||||
);
|
||||
assert_eq!(high.capacity, default * 16);
|
||||
assert_eq!(high.source, ReplayCacheCapacitySource::Env);
|
||||
|
||||
let low = replay_cache_capacity_decision(
|
||||
rustfs_utils::EnvParseOutcome::Parsed(1),
|
||||
64,
|
||||
Some(128 * 1024 * 1024 * 1024),
|
||||
Some(MemoryBasis::Host),
|
||||
);
|
||||
assert_eq!(low.capacity, default);
|
||||
assert_eq!(low.source, ReplayCacheCapacitySource::EnvClampedToDefault);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_capacity_auto_sizes_from_cpu_and_memory() {
|
||||
let gib = 1024_u64 * 1024 * 1024;
|
||||
let decision =
|
||||
replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Absent, 8, Some(16 * gib), Some(MemoryBasis::Host));
|
||||
|
||||
assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
|
||||
assert_eq!(decision.memory_basis, Some(MemoryBasis::Host));
|
||||
assert_eq!(decision.memory_based_capacity, 10_737_418);
|
||||
assert_eq!(decision.cpu_based_capacity, 9_846_784);
|
||||
assert_eq!(decision.capacity, 9_846_784);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_capacity_auto_reaches_hotpath_verified_capacity_on_larger_nodes() {
|
||||
let gib = 1024_u64 * 1024 * 1024;
|
||||
let decision =
|
||||
replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Absent, 16, Some(32 * gib), Some(MemoryBasis::Host));
|
||||
|
||||
assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
|
||||
assert_eq!(decision.memory_based_capacity, 21_474_836);
|
||||
assert_eq!(decision.cpu_based_capacity, 19_693_568);
|
||||
assert_eq!(decision.capacity, 16_777_216);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_capacity_auto_keeps_default_floor_for_small_nodes() {
|
||||
let decision = replay_cache_capacity_decision(
|
||||
rustfs_utils::EnvParseOutcome::Absent,
|
||||
1,
|
||||
Some(512 * 1024 * 1024),
|
||||
Some(MemoryBasis::Host),
|
||||
);
|
||||
|
||||
assert_eq!(decision.capacity, rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY);
|
||||
assert_eq!(decision.source, ReplayCacheCapacitySource::AutoClampedToDefault);
|
||||
assert!(decision.memory_based_capacity < rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_capacity_invalid_env_uses_auto_sizing() {
|
||||
let decision = replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Invalid, 8, None, None);
|
||||
|
||||
assert_eq!(decision.source, ReplayCacheCapacitySource::AutoInvalidEnv);
|
||||
assert_eq!(decision.capacity, 9_846_784);
|
||||
}
|
||||
|
||||
fn check_test_nonce_record(cache: &mut RpcNonceCache, record: RpcNonceRecord<'_>) -> std::io::Result<()> {
|
||||
let (result, metrics) = cache.check_and_record(record);
|
||||
publish_nonce_cache_metrics(metrics);
|
||||
result
|
||||
}
|
||||
|
||||
fn test_nonce_record(
|
||||
nonce: Uuid,
|
||||
signed_at: i64,
|
||||
now: Instant,
|
||||
wall_time: i64,
|
||||
expires_at: Instant,
|
||||
capacity: usize,
|
||||
) -> RpcNonceRecord<'static> {
|
||||
RpcNonceRecord {
|
||||
nonce,
|
||||
signed_at,
|
||||
now,
|
||||
wall_time,
|
||||
expires_at,
|
||||
capacity,
|
||||
metric_scope: RpcReplayCacheMetricScope {
|
||||
operation: INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||
backend: INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
rpc_path: "/node_service.NodeService/ReadAll",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_expires_by_monotonic_deadline_and_fails_closed_at_capacity() {
|
||||
let now = Instant::now();
|
||||
@@ -2378,12 +1912,15 @@ mod tests {
|
||||
let nonce_b = Uuid::new_v4();
|
||||
let mut cache = RpcNonceCache::default();
|
||||
|
||||
check_test_nonce_record(&mut cache, test_nonce_record(nonce_a, 100, now, 100, expiry, 1))
|
||||
cache
|
||||
.check_and_record(nonce_a, 100, now, 100, expiry, 1)
|
||||
.expect("first nonce should be recorded");
|
||||
let capacity = check_test_nonce_record(&mut cache, test_nonce_record(nonce_b, 100, now, 100, expiry, 1))
|
||||
let capacity = cache
|
||||
.check_and_record(nonce_b, 100, now, 100, expiry, 1)
|
||||
.expect_err("a full replay cache must fail closed");
|
||||
assert_eq!(capacity.to_string(), "RPC replay cache capacity exceeded");
|
||||
check_test_nonce_record(&mut cache, test_nonce_record(nonce_b, 702, after_expiry, 702, after_expiry, 1))
|
||||
cache
|
||||
.check_and_record(nonce_b, 702, after_expiry, 702, after_expiry, 1)
|
||||
.expect("expired nonce should release capacity");
|
||||
assert!(!cache.nonces.contains(&nonce_a));
|
||||
assert!(cache.nonces.contains(&nonce_b));
|
||||
@@ -2586,15 +2123,17 @@ mod tests {
|
||||
let nonce = Uuid::new_v4();
|
||||
let mut cache = RpcNonceCache::default();
|
||||
|
||||
check_test_nonce_record(&mut cache, test_nonce_record(nonce, 1_000, now, 1_000, expiry, 2))
|
||||
cache
|
||||
.check_and_record(nonce, 1_000, now, 1_000, expiry, 2)
|
||||
.expect("first nonce should be recorded");
|
||||
let replay = check_test_nonce_record(&mut cache, test_nonce_record(nonce, 1_000, after_expiry, 900, after_expiry, 2))
|
||||
let replay = cache
|
||||
.check_and_record(nonce, 1_000, after_expiry, 900, after_expiry, 2)
|
||||
.expect_err("wall clock regression must not make an old signature reusable");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
|
||||
let stale =
|
||||
check_test_nonce_record(&mut cache, test_nonce_record(Uuid::new_v4(), 600, after_expiry, 900, after_expiry, 2))
|
||||
.expect_err("the monotonic wall-clock high-water mark must fail closed");
|
||||
let stale = cache
|
||||
.check_and_record(Uuid::new_v4(), 600, after_expiry, 900, after_expiry, 2)
|
||||
.expect_err("the monotonic wall-clock high-water mark must fail closed");
|
||||
assert_eq!(stale.to_string(), "RPC request timestamp expired after clock regression");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,8 +34,8 @@ pub use client::{
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
|
||||
@@ -3017,14 +3017,7 @@ impl ECStore {
|
||||
&cleanup_preflight_allowed_missing,
|
||||
"decommission",
|
||||
)
|
||||
.await
|
||||
.map_err(|err| match err {
|
||||
data_movement::SourceCleanupError::SourceChanged => Error::other(format!(
|
||||
"decommission: source cleanup preflight failed for {}/{}: source versions changed after migration started",
|
||||
bucket, entry.name
|
||||
)),
|
||||
data_movement::SourceCleanupError::Storage(err) => err,
|
||||
});
|
||||
.await;
|
||||
resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())?
|
||||
} else if decommissioned != fivs.versions.len() || expired > 0 {
|
||||
warn!(
|
||||
|
||||
@@ -749,7 +749,7 @@ impl crate::storage_api_contracts::list::ListOperations for Sets {
|
||||
type WalkCancellation = CancellationToken;
|
||||
type WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>;
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(self))]
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn list_objects_v2(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
@@ -1093,7 +1093,7 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
#[tracing::instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1749,19 +1749,7 @@ mod tests {
|
||||
upload_id_marker = page.next_upload_id_marker;
|
||||
}
|
||||
|
||||
// Compare only the decoded `<uuid>x<timestamp>` suffixes: the full
|
||||
// upload id embeds the process-global deployment id, which a
|
||||
// concurrently running test can swap between create and list time.
|
||||
let normalize = |uploads: &[(String, String)]| {
|
||||
let mut normalized = uploads
|
||||
.iter()
|
||||
.map(|(key, upload_id)| (key.clone(), runtime_sources::upload_uuid_suffix(upload_id)))
|
||||
.collect::<Vec<_>>();
|
||||
normalized.sort();
|
||||
normalized
|
||||
};
|
||||
let actual = normalize(&actual);
|
||||
assert_eq!(actual, normalize(&expected), "set-level merge must return every upload exactly once");
|
||||
assert_eq!(actual, expected, "set-level merge must return every upload exactly once");
|
||||
let mut deduped = actual.clone();
|
||||
deduped.dedup();
|
||||
assert_eq!(deduped.len(), actual.len(), "set-level pagination must not duplicate uploads");
|
||||
|
||||
@@ -25,7 +25,7 @@ use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
|
||||
use crate::storage_api_contracts::{
|
||||
multipart::{CompletePart, MultipartOperations as _},
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{HTTPPreconditions, ObjectIO as _, ObjectOperations as _},
|
||||
object::{ObjectIO as _, ObjectOperations as _},
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use bytes::Bytes;
|
||||
@@ -228,7 +228,6 @@ fn data_movement_complete_multipart_opts(object_info: &ObjectInfo, src_pool_idx:
|
||||
ObjectOptions {
|
||||
versioned: object_info.version_id.is_some(),
|
||||
version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
|
||||
http_preconditions: data_movement_unversioned_target_precondition(object_info),
|
||||
data_movement: true,
|
||||
mod_time: object_info.mod_time,
|
||||
preserve_etag: object_info.etag.clone(),
|
||||
@@ -243,7 +242,6 @@ fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize)
|
||||
src_pool_idx,
|
||||
data_movement: true,
|
||||
version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
|
||||
http_preconditions: data_movement_unversioned_target_precondition(object_info),
|
||||
mod_time: object_info.mod_time,
|
||||
user_defined: data_movement_user_defined(object_info),
|
||||
preserve_etag: object_info.etag.clone(),
|
||||
@@ -251,17 +249,6 @@ fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize)
|
||||
}
|
||||
}
|
||||
|
||||
fn is_unversioned_data_movement_object(object_info: &ObjectInfo) -> bool {
|
||||
object_info.version_id.is_none_or(|version_id| version_id.is_nil())
|
||||
}
|
||||
|
||||
fn data_movement_unversioned_target_precondition(object_info: &ObjectInfo) -> Option<HTTPPreconditions> {
|
||||
is_unversioned_data_movement_object(object_info).then(|| HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
fn data_movement_put_object_reader(
|
||||
bucket: &str,
|
||||
object_info: &ObjectInfo,
|
||||
@@ -350,7 +337,7 @@ fn schedule_data_movement_multipart_abort_cleanup(
|
||||
}
|
||||
|
||||
fn should_check_data_movement_overwrite_resume(err: &Error) -> bool {
|
||||
is_err_data_movement_overwrite(err) || matches!(err, Error::PreconditionFailed)
|
||||
is_err_data_movement_overwrite(err)
|
||||
}
|
||||
|
||||
fn effective_actual_size(info: &ObjectInfo) -> Option<i64> {
|
||||
@@ -416,16 +403,6 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo)
|
||||
&& are_equivalent_data_movement_parts(&source.parts, &target.parts)
|
||||
}
|
||||
|
||||
fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
|
||||
is_unversioned_data_movement_object(source)
|
||||
&& is_unversioned_data_movement_object(target)
|
||||
&& !target.delete_marker
|
||||
&& source
|
||||
.mod_time
|
||||
.zip(target.mod_time)
|
||||
.is_some_and(|(source_time, target_time)| target_time > source_time)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
struct SourceCleanupPartIdentity {
|
||||
number: usize,
|
||||
@@ -437,15 +414,6 @@ struct SourceCleanupPartIdentity {
|
||||
checksums: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
struct SourceCleanupErasureIdentity {
|
||||
algorithm: String,
|
||||
data_blocks: usize,
|
||||
parity_blocks: usize,
|
||||
block_size: usize,
|
||||
distribution: Vec<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub(crate) struct SourceCleanupVersionIdentity {
|
||||
name: String,
|
||||
@@ -456,28 +424,10 @@ pub(crate) struct SourceCleanupVersionIdentity {
|
||||
etag: Option<String>,
|
||||
checksum: Option<Vec<u8>>,
|
||||
data_dir: Option<uuid::Uuid>,
|
||||
transition_status: String,
|
||||
transitioned_objname: String,
|
||||
transition_tier: String,
|
||||
transition_version_id: Option<uuid::Uuid>,
|
||||
transition_version: Option<String>,
|
||||
transition_version_state: u8,
|
||||
expire_restored: bool,
|
||||
erasure: SourceCleanupErasureIdentity,
|
||||
metadata: BTreeMap<String, String>,
|
||||
parts: Vec<SourceCleanupPartIdentity>,
|
||||
}
|
||||
|
||||
fn source_cleanup_erasure_identity(erasure: &rustfs_filemeta::ErasureInfo) -> SourceCleanupErasureIdentity {
|
||||
SourceCleanupErasureIdentity {
|
||||
algorithm: erasure.algorithm.clone(),
|
||||
data_blocks: erasure.data_blocks,
|
||||
parity_blocks: erasure.parity_blocks,
|
||||
block_size: erasure.block_size,
|
||||
distribution: erasure.distribution.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn source_cleanup_part_identity(part: &ObjectPartInfo) -> SourceCleanupPartIdentity {
|
||||
SourceCleanupPartIdentity {
|
||||
number: part.number,
|
||||
@@ -507,19 +457,6 @@ pub(crate) fn source_cleanup_version_identity(version: &FileInfo) -> SourceClean
|
||||
etag: version.get_etag(),
|
||||
checksum: version.checksum.as_ref().map(|checksum| checksum.to_vec()),
|
||||
data_dir: version.data_dir,
|
||||
transition_status: version.transition_status.clone(),
|
||||
transitioned_objname: version.transitioned_objname.clone(),
|
||||
transition_tier: version.transition_tier.clone(),
|
||||
transition_version_id: version.transition_version_id,
|
||||
transition_version: version.transition_version.clone(),
|
||||
transition_version_state: match version.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => 0,
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled => 1,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull => 2,
|
||||
rustfs_filemeta::TransitionVersionState::Exact => 3,
|
||||
},
|
||||
expire_restored: version.expire_restored,
|
||||
erasure: source_cleanup_erasure_identity(&version.erasure),
|
||||
metadata: version
|
||||
.metadata
|
||||
.iter()
|
||||
@@ -535,6 +472,10 @@ fn source_cleanup_version_identities(fivs: &FileInfoVersions) -> Vec<SourceClean
|
||||
identities
|
||||
}
|
||||
|
||||
fn source_cleanup_versions_match(expected: &FileInfoVersions, current: &FileInfoVersions) -> bool {
|
||||
source_cleanup_versions_match_with_allowed_missing(expected, current, &[])
|
||||
}
|
||||
|
||||
fn source_cleanup_versions_match_with_allowed_missing(
|
||||
expected: &FileInfoVersions,
|
||||
current: &FileInfoVersions,
|
||||
@@ -566,26 +507,6 @@ fn source_cleanup_versions_match_with_allowed_missing(
|
||||
.all(|(identity, count)| allowed_counts.get(&identity).copied().unwrap_or_default() >= count)
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum SourceCleanupError {
|
||||
#[error("source versions changed after migration started")]
|
||||
SourceChanged,
|
||||
#[error(transparent)]
|
||||
Storage(#[from] Error),
|
||||
}
|
||||
|
||||
fn ensure_source_cleanup_versions_match(
|
||||
expected: &FileInfoVersions,
|
||||
current: &FileInfoVersions,
|
||||
allowed_missing: &[SourceCleanupVersionIdentity],
|
||||
) -> std::result::Result<(), SourceCleanupError> {
|
||||
if source_cleanup_versions_match_with_allowed_missing(expected, current, allowed_missing) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(SourceCleanupError::SourceChanged)
|
||||
}
|
||||
}
|
||||
|
||||
fn source_cleanup_preflight_error(op_label: &str, bucket: &str, object: &str, err: impl std::fmt::Display) -> Error {
|
||||
Error::other(format!("{op_label}: source cleanup preflight failed for {bucket}/{object}: {err}"))
|
||||
}
|
||||
@@ -608,87 +529,21 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged(
|
||||
expected: &FileInfoVersions,
|
||||
allowed_missing: &[SourceCleanupVersionIdentity],
|
||||
op_label: &str,
|
||||
) -> std::result::Result<(), SourceCleanupError> {
|
||||
) -> Result<()> {
|
||||
let Some(current) = load_source_cleanup_versions(set, bucket, object, op_label).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
ensure_source_cleanup_versions_match(expected, ¤t, allowed_missing)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct SourceCleanupDeleteBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct SourceCleanupDeleteBarrier {
|
||||
state: Arc<SourceCleanupDeleteBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<SourceCleanupDeleteBarrierState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl SourceCleanupDeleteBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(SourceCleanupDeleteBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison");
|
||||
assert!(slot.is_none(), "source cleanup delete barrier must be unique");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
Self { state }
|
||||
if source_cleanup_versions_match_with_allowed_missing(expected, ¤t, allowed_missing) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("source cleanup should reach the pre-delete barrier");
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for SourceCleanupDeleteBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) {
|
||||
let barrier = SOURCE_CLEANUP_DELETE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
Err(source_cleanup_preflight_error(
|
||||
op_label,
|
||||
bucket,
|
||||
object,
|
||||
"source versions changed after migration started",
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
@@ -698,32 +553,30 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
expected: &FileInfoVersions,
|
||||
allowed_missing: &[SourceCleanupVersionIdentity],
|
||||
op_label: &str,
|
||||
) -> std::result::Result<ObjectInfo, SourceCleanupError> {
|
||||
) -> Result<ObjectInfo> {
|
||||
let cleanup_key = encode_dir_object(object);
|
||||
let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
|
||||
let _guard = ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(Error::from)?;
|
||||
let _guard = ns_lock.get_write_lock(get_lock_acquire_timeout()).await?;
|
||||
|
||||
ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;
|
||||
|
||||
#[cfg(test)]
|
||||
pause_source_cleanup_before_delete(bucket, object).await;
|
||||
|
||||
let mut opts = ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
data_movement: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_namespace_lock_guard(&_guard);
|
||||
let result = set.delete_object(bucket, cleanup_key.as_str(), opts).await;
|
||||
let result = set
|
||||
.delete_object(
|
||||
bucket,
|
||||
cleanup_key.as_str(),
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
data_movement: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
result.map_err(SourceCleanupError::from)
|
||||
result
|
||||
}
|
||||
|
||||
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
|
||||
@@ -774,11 +627,7 @@ fn resolve_data_movement_overwrite_resume_result(
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
if is_equivalent_data_movement_object(source, &target) {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target))
|
||||
Ok(is_equivalent_data_movement_object(source, &target))
|
||||
}
|
||||
|
||||
async fn should_treat_data_movement_overwrite_as_complete(
|
||||
@@ -989,6 +838,7 @@ pub(crate) async fn migrate_object(
|
||||
bucket.as_str(),
|
||||
object_info.name.as_str()
|
||||
);
|
||||
mark_multipart_upload_completed(&abort_multipart_flag);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -1007,32 +857,6 @@ pub(crate) async fn migrate_object(
|
||||
}
|
||||
.await;
|
||||
|
||||
if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
|
||||
let abort_result = match store.pools.get(target_pool_idx) {
|
||||
Some(pool) => {
|
||||
pool.abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
}
|
||||
None => Err(Error::other(format!(
|
||||
"{op_label}: target pool {target_pool_idx} is out of range while aborting superseded multipart upload"
|
||||
))),
|
||||
};
|
||||
if let Err(abort_err) = abort_result
|
||||
&& !is_err_invalid_upload_id(&abort_err)
|
||||
{
|
||||
error!("{op_label}: abort superseded multipart upload err {:?}", &abort_err);
|
||||
schedule_data_movement_multipart_abort_cleanup(
|
||||
store.clone(),
|
||||
target_pool_idx,
|
||||
bucket.clone(),
|
||||
object_info.name.clone(),
|
||||
res.upload_id.clone(),
|
||||
op_label,
|
||||
);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Err(primary_err) = multipart_result {
|
||||
if should_abort_multipart_upload(&abort_multipart_flag) {
|
||||
return match store
|
||||
@@ -1232,7 +1056,7 @@ mod tests {
|
||||
let expected = cleanup_test_versions(vec![first.clone(), second.clone()]);
|
||||
let current = cleanup_test_versions(vec![second, first]);
|
||||
|
||||
assert!(source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &[]));
|
||||
assert!(source_cleanup_versions_match(&expected, ¤t));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1240,40 +1064,7 @@ mod tests {
|
||||
let expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]);
|
||||
let current = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "changed")]);
|
||||
|
||||
let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[])
|
||||
.expect_err("changed source metadata must defer cleanup");
|
||||
assert!(matches!(err, SourceCleanupError::SourceChanged));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_source_cleanup_preflight_rejects_changed_transition_or_erasure() {
|
||||
let expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]);
|
||||
let mut current = expected.clone();
|
||||
current.versions[0].transition_tier = "COLD".to_string();
|
||||
let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[])
|
||||
.expect_err("transition metadata changes must defer cleanup");
|
||||
assert!(matches!(err, SourceCleanupError::SourceChanged));
|
||||
|
||||
let mut current = expected.clone();
|
||||
current.versions[0].erasure.algorithm = "changed".to_string();
|
||||
let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[])
|
||||
.expect_err("erasure metadata changes must defer cleanup");
|
||||
assert!(matches!(err, SourceCleanupError::SourceChanged));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_source_cleanup_preflight_ignores_per_disk_erasure_fields() {
|
||||
let mut expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]);
|
||||
expected.versions[0].erasure.checksums = vec![rustfs_filemeta::ChecksumInfo {
|
||||
part_number: 1,
|
||||
hash: Bytes::from_static(b"disk-a-checksum"),
|
||||
..Default::default()
|
||||
}];
|
||||
let mut current = expected.clone();
|
||||
current.versions[0].erasure.index = 7;
|
||||
current.versions[0].erasure.checksums[0].hash = Bytes::from_static(b"disk-b-checksum");
|
||||
|
||||
assert!(source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &[]));
|
||||
assert!(!source_cleanup_versions_match(&expected, ¤t));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1284,9 +1075,7 @@ mod tests {
|
||||
cleanup_test_file_info("object.txt", Uuid::from_u128(2), "new-version"),
|
||||
]);
|
||||
|
||||
let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[])
|
||||
.expect_err("an added source version must defer cleanup");
|
||||
assert!(matches!(err, SourceCleanupError::SourceChanged));
|
||||
assert!(!source_cleanup_versions_match(&expected, ¤t));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1307,9 +1096,7 @@ mod tests {
|
||||
let expected = cleanup_test_versions(vec![migrated.clone(), protected]);
|
||||
let current = cleanup_test_versions(vec![migrated]);
|
||||
|
||||
let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[])
|
||||
.expect_err("an unexpected missing version must defer cleanup");
|
||||
assert!(matches!(err, SourceCleanupError::SourceChanged));
|
||||
assert!(!source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1321,9 +1108,7 @@ mod tests {
|
||||
let current = cleanup_test_versions(vec![migrated, new_version]);
|
||||
let allowed_missing = vec![source_cleanup_version_identity(&expired)];
|
||||
|
||||
let err = ensure_source_cleanup_versions_match(&expected, ¤t, &allowed_missing)
|
||||
.expect_err("a new source version must defer cleanup even when an expired version may be missing");
|
||||
assert!(matches!(err, SourceCleanupError::SourceChanged));
|
||||
assert!(!source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &allowed_missing));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1386,13 +1171,12 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_check_data_movement_overwrite_resume_accepts_conflict_errors() {
|
||||
fn test_should_check_data_movement_overwrite_resume_only_for_overwrite_error() {
|
||||
assert!(should_check_data_movement_overwrite_resume(&Error::DataMovementOverwriteErr(
|
||||
"bucket-a".to_string(),
|
||||
"object-a".to_string(),
|
||||
"version-a".to_string(),
|
||||
)));
|
||||
assert!(should_check_data_movement_overwrite_resume(&Error::PreconditionFailed));
|
||||
assert!(!should_check_data_movement_overwrite_resume(&Error::SlowDown));
|
||||
}
|
||||
|
||||
@@ -1767,7 +1551,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_data_movement_complete_multipart_opts_preserves_mod_time_version_and_etag() {
|
||||
let mod_time = OffsetDateTime::now_utc();
|
||||
let version_id = Uuid::from_u128(7);
|
||||
let version_id = Uuid::nil();
|
||||
let object_info = ObjectInfo {
|
||||
version_id: Some(version_id),
|
||||
mod_time: Some(mod_time),
|
||||
@@ -1783,12 +1567,11 @@ mod tests {
|
||||
assert_eq!(opts.version_id.as_deref(), Some(version_id.to_string().as_str()));
|
||||
assert_eq!(opts.preserve_etag.as_deref(), Some("etag-value"));
|
||||
assert_eq!(opts.src_pool_idx, 7);
|
||||
assert!(opts.http_preconditions.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_movement_put_object_opts_preserves_version_and_etag() {
|
||||
let version_id = Uuid::from_u128(9);
|
||||
let version_id = Uuid::nil();
|
||||
let object_info = ObjectInfo {
|
||||
version_id: Some(version_id),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
@@ -1806,35 +1589,6 @@ mod tests {
|
||||
assert_eq!(opts.src_pool_idx, 9);
|
||||
assert!(opts.data_movement);
|
||||
assert_eq!(opts.mod_time, object_info.mod_time);
|
||||
assert!(opts.http_preconditions.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_movement_unversioned_put_and_complete_require_absent_target() {
|
||||
for version_id in [None, Some(Uuid::nil())] {
|
||||
let object_info = ObjectInfo {
|
||||
version_id,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let put_opts = data_movement_put_object_opts(&object_info, 9);
|
||||
let complete_opts = data_movement_complete_multipart_opts(&object_info, 9);
|
||||
|
||||
assert_eq!(
|
||||
put_opts
|
||||
.http_preconditions
|
||||
.as_ref()
|
||||
.and_then(HTTPPreconditions::if_none_match_value),
|
||||
Some("*")
|
||||
);
|
||||
assert_eq!(
|
||||
complete_opts
|
||||
.http_preconditions
|
||||
.as_ref()
|
||||
.and_then(HTTPPreconditions::if_none_match_value),
|
||||
Some("*")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2083,154 +1837,6 @@ mod tests {
|
||||
assert!(should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_accepts_newer_unversioned_target() {
|
||||
for version_id in [None, Some(Uuid::nil())] {
|
||||
let source = ObjectInfo {
|
||||
version_id,
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
etag: Some("etag-client-write".to_string()),
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("precondition conflict target should be evaluated");
|
||||
|
||||
assert!(should_resume);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_accepts_equivalent_target() {
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(source.clone())), &source, 0, 1)
|
||||
.expect("equivalent precondition target should be evaluated");
|
||||
|
||||
assert!(should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_rejects_non_newer_unversioned_target() {
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
etag: Some("etag-conflict".to_string()),
|
||||
..source.clone()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("precondition conflict target should be evaluated");
|
||||
|
||||
assert!(!should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_rejects_newer_delete_marker() {
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
delete_marker: true,
|
||||
etag: None,
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("delete marker conflict should be evaluated");
|
||||
|
||||
assert!(!should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_overwrite_error_rejects_newer_unversioned_target() {
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
etag: Some("etag-client-write".to_string()),
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
|
||||
|
||||
let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("pool-selection overwrite must require target equivalence");
|
||||
|
||||
assert!(!should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_rejects_newer_versioned_target() {
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
version_id: Some(Uuid::from_u128(2)),
|
||||
etag: Some("etag-conflict".to_string()),
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("versioned conflict target should be evaluated");
|
||||
|
||||
assert!(!should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_rejects_versioned_source_with_unversioned_target() {
|
||||
let source = ObjectInfo {
|
||||
version_id: Some(Uuid::from_u128(1)),
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
version_id: None,
|
||||
etag: Some("etag-conflict".to_string()),
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("versioned source conflict should be evaluated");
|
||||
|
||||
assert!(!should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rebalance_overwrite_resume_accepts_equivalent_target_version() {
|
||||
let source = ObjectInfo {
|
||||
|
||||
@@ -20,7 +20,7 @@ pub mod local_snapshot;
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, BucketOptions},
|
||||
list::{ListOperations as _, StorageListObjectVersionsInfo},
|
||||
object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _, ObjectOperations as _},
|
||||
object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _},
|
||||
};
|
||||
use crate::{
|
||||
bucket::{metadata_sys::get_replication_config, versioning::VersioningApi as _, versioning_sys::BucketVersioningSys},
|
||||
@@ -33,9 +33,8 @@ use crate::{
|
||||
};
|
||||
pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapshot, snapshot_path};
|
||||
use rustfs_data_usage::{
|
||||
BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
|
||||
DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary,
|
||||
VersionsHistogram, observed_data_usage_is_newer,
|
||||
BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DataUsageCache, DataUsageEntry,
|
||||
DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary, VersionsHistogram,
|
||||
};
|
||||
use rustfs_io_metrics::record_system_path_failure;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
@@ -86,57 +85,12 @@ static USAGE_CACHE_UPDATING: OnceLock<CacheUpdating> = OnceLock::new();
|
||||
static LIVE_BUCKET_USAGE_CACHE: OnceLock<LiveBucketUsageCache> = OnceLock::new();
|
||||
static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
/// Best-available persisted usage for `bucket` when no authoritative source
|
||||
/// exists yet (issue #5716): after an upgrade from a pre-v2 release the only
|
||||
/// persisted usage data is the legacy `.usage.json`, which is demoted to
|
||||
/// non-authoritative, and the authoritative caches stay empty until the
|
||||
/// scanner's first complete cycle lands. Quota admission degrades to the
|
||||
/// pre-discard per-bucket sizes retained on the cached snapshot instead of
|
||||
/// failing every write closed.
|
||||
///
|
||||
/// The baseline is static between snapshot loads — live writes do not advance
|
||||
/// it — so hard-quota enforcement during the degraded window is advisory: the
|
||||
/// overrun is bounded only by the writes issued before the next complete
|
||||
/// scanner cycle replaces the baseline with authoritative usage. That is
|
||||
/// strictly tighter than beta.11 (usage treated as 0) and strictly more
|
||||
/// available than a blanket 503. The fallback applies to any window without
|
||||
/// authoritative usage, not only pre-v2 upgrades; the values always come from
|
||||
/// the last persisted scanner output. Loads go through the TTL-bounded
|
||||
/// snapshot cache, so the quota path adds at most one backend read per
|
||||
/// [`DATA_USAGE_CACHE_TTL_SECS`] window. Returns `None` for buckets absent
|
||||
/// from every persisted snapshot — those still fail closed.
|
||||
pub async fn lookup_degraded_bucket_usage_baseline(store: Arc<ECStore>, bucket: &str) -> Option<u64> {
|
||||
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||
{
|
||||
let cache = data_usage_snapshot_cache().read().await;
|
||||
if let Some(cached) = cache
|
||||
.as_ref()
|
||||
.filter(|cached| tokio::time::Instant::now().duration_since(cached.loaded_at) < ttl)
|
||||
{
|
||||
return cached.degraded_baseline.get(bucket).copied();
|
||||
}
|
||||
}
|
||||
|
||||
// Stale or empty cache: refresh through the TTL-bounded loader. A failed
|
||||
// refresh carries the previous baseline forward, so quota admission keeps
|
||||
// its last grounded values through a backend read outage.
|
||||
let _ = load_data_usage_from_backend_cached(store).await;
|
||||
let cache = data_usage_snapshot_cache().read().await;
|
||||
cache
|
||||
.as_ref()
|
||||
.and_then(|cached| cached.degraded_baseline.get(bucket).copied())
|
||||
}
|
||||
|
||||
/// Cached copy of the last persisted data usage snapshot, served to admin
|
||||
/// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
|
||||
#[derive(Debug, Clone)]
|
||||
struct CachedDataUsageSnapshot {
|
||||
info: Option<DataUsageInfo>,
|
||||
loaded_at: tokio::time::Instant,
|
||||
/// Pre-discard per-bucket sizes from the same load, retained even when the
|
||||
/// snapshot is incomplete and its bucket data is discarded. Consumed only
|
||||
/// by [`lookup_degraded_bucket_usage_baseline`] for quota admission.
|
||||
degraded_baseline: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
impl CachedDataUsageSnapshot {
|
||||
@@ -160,34 +114,24 @@ fn fresh_cached_data_usage_snapshot(
|
||||
|
||||
fn cache_data_usage_snapshot_result(
|
||||
cache: &mut Option<CachedDataUsageSnapshot>,
|
||||
result: Result<(DataUsageInfo, HashMap<String, u64>), Error>,
|
||||
result: Result<DataUsageInfo, Error>,
|
||||
loaded_at: tokio::time::Instant,
|
||||
refresh_generation: u64,
|
||||
current_generation: u64,
|
||||
) -> Option<Result<DataUsageInfo, Error>> {
|
||||
if current_generation != refresh_generation {
|
||||
if data_usage_snapshot_generation() != refresh_generation {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(match result {
|
||||
Ok((info, degraded_baseline)) => {
|
||||
Ok(info) => {
|
||||
*cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(info.clone()),
|
||||
loaded_at,
|
||||
degraded_baseline,
|
||||
});
|
||||
Ok(info)
|
||||
}
|
||||
Err(e) => {
|
||||
// Keep the previous baseline through a failed refresh: quota
|
||||
// admission must not lose its last grounded values because one
|
||||
// backend read errored.
|
||||
let degraded_baseline = cache.take().map(|cached| cached.degraded_baseline).unwrap_or_default();
|
||||
*cache = Some(CachedDataUsageSnapshot {
|
||||
info: None,
|
||||
loaded_at,
|
||||
degraded_baseline,
|
||||
});
|
||||
*cache = Some(CachedDataUsageSnapshot { info: None, loaded_at });
|
||||
Err(e)
|
||||
}
|
||||
})
|
||||
@@ -198,9 +142,6 @@ type DataUsageSnapshotCache = Arc<RwLock<Option<CachedDataUsageSnapshot>>>;
|
||||
static DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
|
||||
static DATA_USAGE_SNAPSHOT_REFRESH: OnceLock<Arc<TokioMutex<()>>> = OnceLock::new();
|
||||
static DATA_USAGE_SNAPSHOT_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
static ADMIN_DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
|
||||
static ADMIN_DATA_USAGE_SNAPSHOT_REFRESH: OnceLock<Arc<TokioMutex<()>>> = OnceLock::new();
|
||||
static ADMIN_DATA_USAGE_SNAPSHOT_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
// Always-on revert detector for rustfs/backlog#1306: one relaxed increment per
|
||||
// full-bucket version listing is negligible and lets tests prove that admin
|
||||
@@ -259,24 +200,11 @@ fn data_usage_snapshot_generation() -> u64 {
|
||||
DATA_USAGE_SNAPSHOT_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn admin_data_usage_snapshot_cache() -> &'static DataUsageSnapshotCache {
|
||||
ADMIN_DATA_USAGE_SNAPSHOT_CACHE.get_or_init(|| Arc::new(RwLock::new(None)))
|
||||
}
|
||||
|
||||
fn admin_data_usage_snapshot_generation() -> u64 {
|
||||
ADMIN_DATA_USAGE_SNAPSHOT_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn clear_data_usage_snapshot_cache(cache: &mut Option<CachedDataUsageSnapshot>) {
|
||||
DATA_USAGE_SNAPSHOT_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||
*cache = None;
|
||||
}
|
||||
|
||||
fn clear_admin_data_usage_snapshot_cache(cache: &mut Option<CachedDataUsageSnapshot>) {
|
||||
ADMIN_DATA_USAGE_SNAPSHOT_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||
*cache = None;
|
||||
}
|
||||
|
||||
fn live_bucket_usage_cache() -> &'static LiveBucketUsageCache {
|
||||
LIVE_BUCKET_USAGE_CACHE.get_or_init(|| {
|
||||
moka::future::Cache::builder()
|
||||
@@ -301,11 +229,6 @@ lazy_static::lazy_static! {
|
||||
SLASH_SEPARATOR,
|
||||
DATA_USAGE_OBJECT_NAME
|
||||
);
|
||||
pub static ref DATA_USAGE_OBSERVED_OBJ_NAME_PATH: String = format!("{}{}{}",
|
||||
crate::disk::BUCKET_META_PREFIX,
|
||||
SLASH_SEPARATOR,
|
||||
DATA_USAGE_OBSERVED_OBJECT_NAME
|
||||
);
|
||||
static ref DATA_USAGE_OBJ_BACKUP_PATH: String = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
static ref LEGACY_DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}",
|
||||
crate::disk::BUCKET_META_PREFIX,
|
||||
@@ -380,11 +303,6 @@ fn stale_data_usage_persist_reason_for_source(
|
||||
/// Store data usage info to backend storage
|
||||
#[instrument(skip(store))]
|
||||
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
|
||||
if data_usage_info.usage_snapshot_converged == Some(false) {
|
||||
return Err(Error::other(
|
||||
"nonconverged data usage observations cannot replace the quota-authoritative snapshot",
|
||||
));
|
||||
}
|
||||
// Prevent older data from overwriting newer persisted stats
|
||||
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
|
||||
&& source.is_authoritative()
|
||||
@@ -405,12 +323,10 @@ async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<E
|
||||
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
|
||||
|
||||
// Save to backend using the same mechanism as original code
|
||||
crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
|
||||
crate::config::com::save_config(store, &DATA_USAGE_OBJ_NAME_PATH, data)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
|
||||
cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await;
|
||||
|
||||
// Invalidate the cached snapshot so readers observe the new save on their
|
||||
// next request instead of waiting out the remaining TTL. The next cached
|
||||
// read reloads through `load_data_usage_from_backend`, keeping its
|
||||
@@ -420,64 +336,6 @@ async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<E
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
trait ObservedDataUsageSnapshotCleanup {
|
||||
async fn delete_observed_data_usage_snapshot(&self, revision: &str) -> Result<(), Error>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObservedDataUsageSnapshotCleanup for ECStore {
|
||||
async fn delete_observed_data_usage_snapshot(&self, revision: &str) -> Result<(), Error> {
|
||||
self.delete_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(revision.to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
|
||||
where
|
||||
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
|
||||
{
|
||||
let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
|
||||
Ok(Some(snapshot)) => snapshot,
|
||||
Ok(None) => return,
|
||||
Err(err) => {
|
||||
record_usage_snapshot_failure(
|
||||
"read_observed_before_authoritative_cleanup",
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
&err,
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
if observed_data_usage_is_newer(&observed, authoritative) {
|
||||
return;
|
||||
}
|
||||
|
||||
match store.delete_observed_data_usage_snapshot(&revision).await {
|
||||
Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {}
|
||||
Err(err) => {
|
||||
record_usage_snapshot_failure(
|
||||
"delete_observed_after_authoritative_save",
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
&err,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
|
||||
data_usage_info.buckets_count = u64::try_from(data_usage_info.buckets_usage.len()).unwrap_or(u64::MAX);
|
||||
}
|
||||
@@ -531,11 +389,6 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
|
||||
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
|
||||
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache cleanup")?;
|
||||
clear_data_usage_snapshot_cache(&mut snapshot_cache);
|
||||
drop(snapshot_cache);
|
||||
|
||||
let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
|
||||
ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache cleanup")?;
|
||||
clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -551,10 +404,6 @@ where
|
||||
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
|
||||
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
|
||||
clear_data_usage_snapshot_cache(&mut snapshot_cache);
|
||||
drop(snapshot_cache);
|
||||
let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
|
||||
ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?;
|
||||
clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
|
||||
result
|
||||
}
|
||||
|
||||
@@ -648,23 +497,6 @@ where
|
||||
)
|
||||
.await?;
|
||||
|
||||
ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?;
|
||||
if let Err(err) = remove_bucket_usage_from_object_with_retries(
|
||||
store,
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
bucket,
|
||||
DATA_USAGE_REMOVE_CAS_RETRIES,
|
||||
None,
|
||||
guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
// The authoritative timestamp was already advanced above, so admin
|
||||
// selection rejects this observation even if optional cleanup fails.
|
||||
// Never make an admin-only freshness artifact block DeleteBucket.
|
||||
record_usage_snapshot_failure("remove_bucket_from_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
|
||||
}
|
||||
|
||||
for object in [
|
||||
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
|
||||
@@ -929,72 +761,10 @@ async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo,
|
||||
/// Load data usage info from backend storage
|
||||
#[instrument(skip(store))]
|
||||
pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
Ok(load_data_usage_from_backend_with_baseline(store).await?.0)
|
||||
}
|
||||
|
||||
/// Like [`load_data_usage_from_backend`], but also returns the pre-discard
|
||||
/// per-bucket sizes so the cached loader can retain them as the degraded
|
||||
/// quota-admission baseline (issue #5716).
|
||||
async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Result<(DataUsageInfo, HashMap<String, u64>), Error> {
|
||||
let (data_usage_info, source) = load_data_usage_snapshot(store).await?;
|
||||
Ok(normalize_loaded_data_usage(data_usage_info, source.is_authoritative()).await)
|
||||
}
|
||||
|
||||
async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUsageInfo> {
|
||||
let data = match read_config_preserve_empty(store, &DATA_USAGE_OBSERVED_OBJ_NAME_PATH).await {
|
||||
Ok(data) => data,
|
||||
Err(Error::ConfigNotFound) => return None,
|
||||
Err(err) => {
|
||||
record_usage_snapshot_failure("read_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
match parse_usage_snapshot(&data) {
|
||||
Ok(info) if info.usage_snapshot_converged == Some(false) && info.is_complete_bucket_usage_snapshot() => Some(info),
|
||||
Ok(_) => {
|
||||
error!(
|
||||
event = "data_usage_snapshot_load_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "data_usage",
|
||||
state = "invalid_observed_snapshot",
|
||||
object = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
"observed data usage snapshot was not a structurally complete nonconverged view"
|
||||
);
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
record_usage_snapshot_decode_failure("parse_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn select_admin_data_usage_snapshot(
|
||||
mut authoritative: DataUsageInfo,
|
||||
authoritative_format: bool,
|
||||
observed: Option<DataUsageInfo>,
|
||||
) -> (DataUsageInfo, bool) {
|
||||
if authoritative_format
|
||||
&& authoritative.is_complete_bucket_usage_snapshot()
|
||||
&& authoritative.usage_snapshot_converged.is_none()
|
||||
{
|
||||
authoritative.usage_snapshot_converged = Some(true);
|
||||
}
|
||||
match observed {
|
||||
Some(observed) if observed_data_usage_is_newer(&observed, &authoritative) => (observed, true),
|
||||
_ => (authoritative, authoritative_format),
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
let (authoritative, source) = load_data_usage_snapshot(store.clone()).await?;
|
||||
let observed = load_observed_data_usage_snapshot(store).await;
|
||||
let (selected, selected_is_current_format) =
|
||||
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
|
||||
Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
|
||||
}
|
||||
|
||||
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
|
||||
if !data_usage_info.is_complete_bucket_usage_snapshot() {
|
||||
data_usage_info.usage_snapshot_complete = false;
|
||||
@@ -1037,13 +807,7 @@ fn populate_backward_compatible_usage_maps(data_usage_info: &mut DataUsageInfo)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the normalized snapshot plus the pre-discard per-bucket sizes: the
|
||||
/// degraded quota-admission baseline captured before an incomplete snapshot
|
||||
/// drops its bucket data (issue #5716).
|
||||
async fn normalize_loaded_data_usage(
|
||||
mut data_usage_info: DataUsageInfo,
|
||||
authoritative_format: bool,
|
||||
) -> (DataUsageInfo, HashMap<String, u64>) {
|
||||
async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo, authoritative_format: bool) -> DataUsageInfo {
|
||||
info!("Loaded data usage info from backend with {} buckets", data_usage_info.buckets_count);
|
||||
|
||||
if !authoritative_format {
|
||||
@@ -1051,7 +815,6 @@ async fn normalize_loaded_data_usage(
|
||||
}
|
||||
populate_backward_compatible_usage_maps(&mut data_usage_info);
|
||||
validate_complete_usage_snapshot(&mut data_usage_info);
|
||||
let degraded_baseline = data_usage_info.bucket_sizes.clone();
|
||||
discard_incomplete_bucket_usage(&mut data_usage_info);
|
||||
|
||||
// Handle replication info
|
||||
@@ -1077,7 +840,7 @@ async fn normalize_loaded_data_usage(
|
||||
}
|
||||
}
|
||||
|
||||
(data_usage_info, degraded_baseline)
|
||||
data_usage_info
|
||||
}
|
||||
|
||||
/// Load the persisted data usage snapshot through a small in-process cache.
|
||||
@@ -1110,58 +873,10 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
|
||||
}
|
||||
|
||||
let refresh_generation = data_usage_snapshot_generation();
|
||||
let result = load_data_usage_from_backend_with_baseline(store.clone()).await;
|
||||
let result = load_data_usage_from_backend(store.clone()).await;
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let mut cache = data_usage_snapshot_cache().write().await;
|
||||
if let Some(result) =
|
||||
cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation, data_usage_snapshot_generation())
|
||||
{
|
||||
return result;
|
||||
}
|
||||
drop(cache);
|
||||
drop(refresh_guard);
|
||||
}
|
||||
}
|
||||
|
||||
/// Load the freshest structurally complete snapshot for authenticated admin
|
||||
/// observability. A scan raced by namespace activity may be selected here, but
|
||||
/// never by [`load_data_usage_from_backend_cached`], which remains the
|
||||
/// converged source for quota admission.
|
||||
pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||
|
||||
loop {
|
||||
{
|
||||
let cache = admin_data_usage_snapshot_cache().read().await;
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
let refresh_guard = ADMIN_DATA_USAGE_SNAPSHOT_REFRESH
|
||||
.get_or_init(|| Arc::new(TokioMutex::new(())))
|
||||
.lock()
|
||||
.await;
|
||||
{
|
||||
let cache = admin_data_usage_snapshot_cache().read().await;
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
let refresh_generation = admin_data_usage_snapshot_generation();
|
||||
let result = load_admin_data_usage_from_backend(store.clone())
|
||||
.await
|
||||
.map(|info| (info, HashMap::new()));
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let mut cache = admin_data_usage_snapshot_cache().write().await;
|
||||
if let Some(result) = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
result,
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
admin_data_usage_snapshot_generation(),
|
||||
) {
|
||||
if let Some(result) = cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation) {
|
||||
return result;
|
||||
}
|
||||
drop(cache);
|
||||
@@ -1174,16 +889,6 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
|
||||
pub async fn invalidate_data_usage_snapshot_cache() {
|
||||
let mut cache = data_usage_snapshot_cache().write().await;
|
||||
clear_data_usage_snapshot_cache(&mut cache);
|
||||
|
||||
let mut admin_cache = admin_data_usage_snapshot_cache().write().await;
|
||||
clear_admin_data_usage_snapshot_cache(&mut admin_cache);
|
||||
}
|
||||
|
||||
/// Invalidate only the admin/console view after an observational save. Quota
|
||||
/// admission continues to use the independently cached converged snapshot.
|
||||
pub async fn invalidate_admin_data_usage_snapshot_cache() {
|
||||
let mut cache = admin_data_usage_snapshot_cache().write().await;
|
||||
clear_admin_data_usage_snapshot_cache(&mut cache);
|
||||
}
|
||||
|
||||
/// Aggregate usage information from local disk snapshots.
|
||||
@@ -2307,7 +2012,6 @@ mod tests {
|
||||
struct UsageCasState {
|
||||
object: Option<(Vec<u8>, u64)>,
|
||||
backup_object: Option<(Vec<u8>, u64)>,
|
||||
observed_object: Option<(Vec<u8>, u64)>,
|
||||
legacy_object: Option<(Vec<u8>, u64)>,
|
||||
legacy_backup_object: Option<(Vec<u8>, u64)>,
|
||||
interleaving_snapshot: Option<Vec<u8>>,
|
||||
@@ -2327,24 +2031,10 @@ mod tests {
|
||||
state: Mutex<UsageCasState>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObservedDataUsageSnapshotCleanup for UsageCasStore {
|
||||
async fn delete_observed_data_usage_snapshot(&self, revision: &str) -> Result<(), Error> {
|
||||
let mut state = self.state.lock().await;
|
||||
let current = state.observed_object.as_ref().ok_or(Error::FileNotFound)?.1;
|
||||
if revision != format!("usage-{current}") {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
state.observed_object = None;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum UsageObjectSlot {
|
||||
Primary,
|
||||
Backup,
|
||||
Observed,
|
||||
LegacyPrimary,
|
||||
LegacyBackup,
|
||||
}
|
||||
@@ -2373,7 +2063,6 @@ mod tests {
|
||||
let slot = match object {
|
||||
object if object == DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Primary,
|
||||
object if object == DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::Backup,
|
||||
object if object == DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Observed,
|
||||
object if object == LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::LegacyPrimary,
|
||||
object if object == LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::LegacyBackup,
|
||||
_ => return Err(Error::FileNotFound),
|
||||
@@ -2382,7 +2071,6 @@ mod tests {
|
||||
let stored = match slot {
|
||||
UsageObjectSlot::Primary => &state.object,
|
||||
UsageObjectSlot::Backup => &state.backup_object,
|
||||
UsageObjectSlot::Observed => &state.observed_object,
|
||||
UsageObjectSlot::LegacyPrimary => &state.legacy_object,
|
||||
UsageObjectSlot::LegacyBackup => &state.legacy_backup_object,
|
||||
};
|
||||
@@ -2422,7 +2110,6 @@ mod tests {
|
||||
let slot = match object {
|
||||
object if object == DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Primary,
|
||||
object if object == DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::Backup,
|
||||
object if object == DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Observed,
|
||||
object if object == LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::LegacyPrimary,
|
||||
object if object == LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::LegacyBackup,
|
||||
_ => return Err(Error::FileNotFound),
|
||||
@@ -2458,9 +2145,6 @@ mod tests {
|
||||
let revision = state.backup_object.as_ref().map_or(1, |(_, revision)| revision + 1);
|
||||
state.backup_object = Some((interleaving, revision));
|
||||
}
|
||||
if slot == UsageObjectSlot::Observed {
|
||||
return Err(Error::other("observed test fixture writes are injected directly"));
|
||||
}
|
||||
if slot == UsageObjectSlot::LegacyPrimary
|
||||
&& let Some(interleaving) = state.interleaving_legacy_snapshot.take()
|
||||
{
|
||||
@@ -2476,7 +2160,6 @@ mod tests {
|
||||
let current_revision = match slot {
|
||||
UsageObjectSlot::Primary => state.object.as_ref(),
|
||||
UsageObjectSlot::Backup => state.backup_object.as_ref(),
|
||||
UsageObjectSlot::Observed => state.observed_object.as_ref(),
|
||||
UsageObjectSlot::LegacyPrimary => state.legacy_object.as_ref(),
|
||||
UsageObjectSlot::LegacyBackup => state.legacy_backup_object.as_ref(),
|
||||
}
|
||||
@@ -2507,7 +2190,6 @@ mod tests {
|
||||
match slot {
|
||||
UsageObjectSlot::Primary => state.object = Some((buf, revision)),
|
||||
UsageObjectSlot::Backup => state.backup_object = Some((buf, revision)),
|
||||
UsageObjectSlot::Observed => state.observed_object = Some((buf, revision)),
|
||||
UsageObjectSlot::LegacyPrimary => state.legacy_object = Some((buf, revision)),
|
||||
UsageObjectSlot::LegacyBackup => state.legacy_backup_object = Some((buf, revision)),
|
||||
}
|
||||
@@ -2679,7 +2361,7 @@ mod tests {
|
||||
legacy.bucket_sizes.insert("large".to_string(), 0);
|
||||
legacy.buckets_count = 2;
|
||||
|
||||
let (normalized, degraded_baseline) = normalize_loaded_data_usage(legacy, false).await;
|
||||
let normalized = normalize_loaded_data_usage(legacy, false).await;
|
||||
|
||||
assert_eq!(normalized.buckets_count, 0);
|
||||
assert!(normalized.buckets_usage.is_empty());
|
||||
@@ -2687,10 +2369,6 @@ mod tests {
|
||||
assert_eq!(normalized.objects_total_count, 0);
|
||||
assert_eq!(normalized.objects_total_size, 0);
|
||||
assert!(!normalized.usage_snapshot_complete);
|
||||
// Issue #5716: the discarded sizes must survive as the degraded
|
||||
// quota-admission baseline.
|
||||
assert_eq!(degraded_baseline.get("control").copied(), Some(10_285));
|
||||
assert_eq!(degraded_baseline.get("large").copied(), Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2726,7 +2404,7 @@ mod tests {
|
||||
info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
|
||||
info.buckets_count = 2;
|
||||
|
||||
let (normalized, _) = normalize_loaded_data_usage(info, true).await;
|
||||
let normalized = normalize_loaded_data_usage(info, true).await;
|
||||
|
||||
assert_eq!(normalized.buckets_count, 2);
|
||||
assert!(normalized.usage_snapshot_complete);
|
||||
@@ -2761,7 +2439,7 @@ mod tests {
|
||||
info.bucket_sizes.insert("partial".to_string(), 196_870_144);
|
||||
info.buckets_count = 1;
|
||||
|
||||
let (normalized, _) = normalize_loaded_data_usage(info, true).await;
|
||||
let normalized = normalize_loaded_data_usage(info, true).await;
|
||||
|
||||
assert_eq!(normalized.buckets_count, 0);
|
||||
assert!(!normalized.buckets_usage.contains_key("control"));
|
||||
@@ -2776,7 +2454,7 @@ mod tests {
|
||||
info.buckets_count = 2;
|
||||
|
||||
assert!(!data_usage_contains_bucket(&info, "missing"));
|
||||
let (normalized, _) = normalize_loaded_data_usage(info, true).await;
|
||||
let normalized = normalize_loaded_data_usage(info, true).await;
|
||||
|
||||
assert!(!normalized.usage_snapshot_complete);
|
||||
assert!(normalized.buckets_usage.is_empty());
|
||||
@@ -2785,7 +2463,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_empty_snapshot_remains_authoritative() {
|
||||
let (normalized, _) = normalize_loaded_data_usage(
|
||||
let normalized = normalize_loaded_data_usage(
|
||||
DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
usage_snapshot_complete: true,
|
||||
@@ -2800,71 +2478,6 @@ mod tests {
|
||||
assert!(normalized.buckets_usage.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_snapshot_selection_requires_the_current_authoritative_baseline() {
|
||||
let authoritative = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
scanner_epoch: Some(4),
|
||||
scanner_cycle: Some(10),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let observed = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
|
||||
scanner_epoch: Some(4),
|
||||
scanner_cycle: Some(11),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (selected, _) = select_admin_data_usage_snapshot(authoritative.clone(), true, Some(observed.clone()));
|
||||
assert_eq!(selected.usage_snapshot_converged, Some(false));
|
||||
|
||||
let mut namespace_changed = authoritative;
|
||||
namespace_changed.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(2));
|
||||
let (selected, _) = select_admin_data_usage_snapshot(namespace_changed, true, Some(observed));
|
||||
assert_eq!(selected.usage_snapshot_converged, Some(true));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
|
||||
let store = UsageCasStore::default();
|
||||
let authoritative = data_usage_info_for_test("bucket", 1, 10, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
|
||||
let stale_observed = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
|
||||
scanner_epoch: Some(4),
|
||||
scanner_cycle: Some(10),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
..Default::default()
|
||||
};
|
||||
store.state.lock().await.observed_object =
|
||||
Some((serde_json::to_vec(&stale_observed).expect("observed snapshot should encode"), 1));
|
||||
|
||||
cleanup_observed_data_usage_after_authoritative_save(&store, &authoritative).await;
|
||||
assert!(store.state.lock().await.observed_object.is_none());
|
||||
|
||||
cleanup_observed_data_usage_after_authoritative_save(&store, &authoritative).await;
|
||||
assert!(store.state.lock().await.observed_object.is_none());
|
||||
|
||||
let newer_observed = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(3)),
|
||||
scanner_epoch: Some(4),
|
||||
scanner_cycle: Some(11),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
|
||||
..Default::default()
|
||||
};
|
||||
store.state.lock().await.observed_object =
|
||||
Some((serde_json::to_vec(&newer_observed).expect("observed snapshot should encode"), 2));
|
||||
|
||||
cleanup_observed_data_usage_after_authoritative_save(&store, &authoritative).await;
|
||||
assert!(store.state.lock().await.observed_object.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn cached_snapshot_failure_is_reused_until_ttl_expires() {
|
||||
@@ -2872,14 +2485,8 @@ mod tests {
|
||||
let mut cache = None;
|
||||
let refresh_generation = data_usage_snapshot_generation();
|
||||
|
||||
let first = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Err(Error::ErasureReadQuorum),
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
)
|
||||
.expect("an uninterrupted refresh should populate the cache");
|
||||
let first = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at, refresh_generation)
|
||||
.expect("an uninterrupted refresh should populate the cache");
|
||||
assert!(matches!(first, Err(Error::ErasureReadQuorum)));
|
||||
|
||||
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
|
||||
@@ -2897,15 +2504,9 @@ mod tests {
|
||||
let mut cache = None;
|
||||
let refresh_generation = data_usage_snapshot_generation();
|
||||
|
||||
let first = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok((expected, HashMap::new())),
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
)
|
||||
.expect("an uninterrupted refresh should populate the cache")
|
||||
.expect("successful load must be returned");
|
||||
let first = cache_data_usage_snapshot_result(&mut cache, Ok(expected), loaded_at, refresh_generation)
|
||||
.expect("an uninterrupted refresh should populate the cache")
|
||||
.expect("successful load must be returned");
|
||||
assert_snapshot_bucket(&first, "bucket");
|
||||
|
||||
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
|
||||
@@ -2922,16 +2523,14 @@ mod tests {
|
||||
let mut cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
clear_data_usage_snapshot_cache(&mut cache);
|
||||
|
||||
let stale_result = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok((data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH), HashMap::new())),
|
||||
Ok(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
);
|
||||
|
||||
assert!(stale_result.is_none());
|
||||
@@ -3934,7 +3533,6 @@ mod tests {
|
||||
*snapshot_cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(successor),
|
||||
loaded_at: tokio::time::Instant::now(),
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
memory_cache()
|
||||
.write()
|
||||
@@ -3997,7 +3595,6 @@ mod tests {
|
||||
*snapshot_cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(successor),
|
||||
loaded_at: tokio::time::Instant::now(),
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
|
||||
let store_for_cleanup = store.clone();
|
||||
@@ -4049,7 +3646,6 @@ mod tests {
|
||||
*data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
|
||||
info: Some(stale),
|
||||
loaded_at: tokio::time::Instant::now(),
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
|
||||
remove_bucket_usage_from_backend_with_guard(&store, BUCKET, None)
|
||||
|
||||
@@ -119,7 +119,7 @@ fn read_all_data_std(path: &Path) -> core::result::Result<(Vec<u8>, Option<Offse
|
||||
Ok((bytes, modtime))
|
||||
}
|
||||
|
||||
pub(crate) fn inline_metadata_rollback_dir(version_id: Uuid, meta: &FileMeta) -> Uuid {
|
||||
fn inline_metadata_rollback_dir(version_id: Uuid, meta: &FileMeta) -> Uuid {
|
||||
let used_data_dirs: HashSet<Uuid> = meta.get_data_dirs().unwrap_or_default().into_iter().flatten().collect();
|
||||
let base = version_id.as_u128() ^ INLINE_METADATA_ROLLBACK_DIR_XOR;
|
||||
let mut salt = 0u128;
|
||||
@@ -240,15 +240,8 @@ async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, d
|
||||
}
|
||||
|
||||
async fn restore_metadata_backup(object_dir: &Path, xl_path: &Path, rollback_dir: Uuid) -> Result<()> {
|
||||
let rollback_path = object_dir.join(rollback_dir.to_string());
|
||||
let backup_path = rollback_path.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
rename_all(&backup_path, xl_path, object_dir).await?;
|
||||
// A synthetic inline rollback dir held only the backup the rename above
|
||||
// just consumed; reclaim it so the object dir can empty out. A real data
|
||||
// dir still holds its parts, so the non-recursive remove is a benign
|
||||
// no-op there (mirrors restore_delete_rollback).
|
||||
let _ = fs::remove_dir(&rollback_path).await;
|
||||
Ok(())
|
||||
let backup_path = object_dir.join(rollback_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
rename_all(&backup_path, xl_path, object_dir).await
|
||||
}
|
||||
|
||||
async fn restore_delete_rollback(object_dir: &Path, xl_path: &Path, rollback_dir: Uuid) -> Result<()> {
|
||||
@@ -691,10 +684,6 @@ const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts";
|
||||
const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed";
|
||||
const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed";
|
||||
const EVENT_DISK_LOCAL_FORMAT_DECODE_FAILED: &str = "disk_local_format_decode_failed";
|
||||
/// A healing commit could not trash the stale destination data dir it is about
|
||||
/// to replace. Best effort — the rename that follows fails closed — but a
|
||||
/// recurring signal means heal is stuck on that drive.
|
||||
const EVENT_DISK_LOCAL_HEAL_PURGE_FAILED: &str = "disk_local_heal_purge_failed";
|
||||
const METRIC_GET_OBJECT_MMAP_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_mmap_page_faults_total";
|
||||
const METRIC_GET_OBJECT_DIRECT_READ_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_direct_read_page_faults_total";
|
||||
// io_uring read-backend gray-release observability (rustfs/backlog#1172).
|
||||
@@ -7857,9 +7846,6 @@ impl DiskAPI for LocalDisk {
|
||||
check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?;
|
||||
|
||||
let no_inline = fi.data.is_none() && fi.size > 0;
|
||||
// Captured before `fi` is consumed by add_version; gates the stale
|
||||
// destination purge below.
|
||||
let fi_healing = fi.is_healing();
|
||||
|
||||
// Resolved once for the whole commit so a concurrent configuration
|
||||
// change can never leave a single rename_data half-synced. The tier is
|
||||
@@ -7976,26 +7962,6 @@ impl DiskAPI for LocalDisk {
|
||||
shard_sync_res?;
|
||||
remove_dst_base_before_commit(dst_path).map_err(to_file_error)?;
|
||||
|
||||
// Heal reuses the version's data_dir, so for in-place corruption
|
||||
// the destination dir still exists — and rename(2) cannot replace
|
||||
// a non-empty directory (EEXIST on XFS, ENOTEMPTY on ext4). Purge
|
||||
// it first, healing commits only; fresh PUTs mint a new data_dir
|
||||
// and never collide. Best effort: a real failure surfaces in the
|
||||
// rename below.
|
||||
if fi_healing
|
||||
&& let Some((_, dst_data_path)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = self.move_to_trash(dst_data_path, true, false).await
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_HEAL_PURGE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
dst_path = ?dst_data_path,
|
||||
error = ?err,
|
||||
"Healing commit could not purge the stale destination data dir"
|
||||
);
|
||||
}
|
||||
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = rename_all(src_data_path, dst_data_path, &skip_parent).await
|
||||
{
|
||||
@@ -11254,86 +11220,6 @@ mod test {
|
||||
(disk, dir)
|
||||
}
|
||||
|
||||
// Stage the bitrot-heal collision: a committed version whose data_dir is
|
||||
// present and non-empty, plus a replacement shard staged in tmp for the
|
||||
// SAME data_dir (heal repairs in place, it does not mint a new data_dir).
|
||||
async fn stage_healing_collision(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
tmp_object: &str,
|
||||
) -> (LocalDisk, tempfile::TempDir, std::path::PathBuf, FileInfo) {
|
||||
use tempfile::tempdir;
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
|
||||
let version_id = Uuid::parse_str("dddddddd-dddd-dddd-dddd-dddddddddddd").expect("version id should parse");
|
||||
let data_dir = Uuid::parse_str("eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee").expect("data dir should parse");
|
||||
|
||||
let object_dir = dir.path().join(bucket).join(object);
|
||||
let dst_data_dir = object_dir.join(data_dir.to_string());
|
||||
fs::create_dir_all(&dst_data_dir)
|
||||
.await
|
||||
.expect("dst data dir should be created");
|
||||
fs::write(dst_data_dir.join("part.1"), b"stale-corrupt-shard")
|
||||
.await
|
||||
.expect("stale shard should be written");
|
||||
let old_fi = test_file_info(object, version_id, Some(data_dir), None);
|
||||
fs::write(object_dir.join(STORAGE_FORMAT_FILE), test_meta(old_fi))
|
||||
.await
|
||||
.expect("old metadata should be written");
|
||||
|
||||
let tmp_data_dir = dir
|
||||
.path()
|
||||
.join(RUSTFS_META_TMP_BUCKET)
|
||||
.join(tmp_object)
|
||||
.join(data_dir.to_string());
|
||||
fs::create_dir_all(&tmp_data_dir)
|
||||
.await
|
||||
.expect("tmp data dir should be created");
|
||||
fs::write(tmp_data_dir.join("part.1"), b"healed-shard")
|
||||
.await
|
||||
.expect("healed shard should be written");
|
||||
|
||||
let new_fi = test_file_info(object, version_id, Some(data_dir), None);
|
||||
(disk, dir, dst_data_dir.join("part.1"), new_fi)
|
||||
}
|
||||
|
||||
// A healing commit must replace a still-existing destination data dir;
|
||||
// without the purge it failed on every attempt and bitrot was never
|
||||
// repaired.
|
||||
#[tokio::test]
|
||||
async fn rename_data_healing_commit_replaces_stale_destination_data_dir() {
|
||||
let (disk, _dir, dst_part, mut new_fi) = stage_healing_collision("bucket", "bitrot-object", "tmp-heal-object").await;
|
||||
new_fi.set_healing();
|
||||
|
||||
disk.rename_data(RUSTFS_META_TMP_BUCKET, "tmp-heal-object", new_fi, "bucket", "bitrot-object")
|
||||
.await
|
||||
.expect("a healing rename_data must replace the stale destination data dir");
|
||||
|
||||
let content = fs::read(&dst_part).await.expect("healed shard should be readable");
|
||||
assert_eq!(content, b"healed-shard", "the healed shard must replace the stale corrupt content");
|
||||
}
|
||||
|
||||
// The purge is healing-gated: an ordinary commit colliding with a
|
||||
// non-empty data dir must keep failing loudly.
|
||||
#[tokio::test]
|
||||
async fn rename_data_non_healing_destination_collision_still_fails() {
|
||||
let (disk, _dir, dst_part, new_fi) = stage_healing_collision("bucket", "collision-object", "tmp-collision-object").await;
|
||||
|
||||
disk.rename_data(RUSTFS_META_TMP_BUCKET, "tmp-collision-object", new_fi, "bucket", "collision-object")
|
||||
.await
|
||||
.expect_err("a non-healing rename_data onto a non-empty destination data dir must fail");
|
||||
|
||||
let content = fs::read(&dst_part).await.expect("stale shard should still be readable");
|
||||
assert_eq!(
|
||||
content, b"stale-corrupt-shard",
|
||||
"a failed non-healing commit must leave the existing content untouched"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rename_data_new_object_fsyncs_new_ancestor_dirs() {
|
||||
// A first PUT under a new prefix must fsync every newly created ancestor
|
||||
@@ -12342,54 +12228,6 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
// The undo_write restore consumes `<rollback>/xl.meta.bkp` by rename; a
|
||||
// synthetic rollback dir is then empty and must be reclaimed so the object
|
||||
// dir can empty out (BucketNotEmpty leak). A real data dir still holds its
|
||||
// parts and must survive the non-recursive remove.
|
||||
#[tokio::test]
|
||||
async fn restore_metadata_backup_reclaims_empty_rollback_dir_only() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let object_dir = dir.path().join("bucket").join("obj");
|
||||
let xl_path = object_dir.join(STORAGE_FORMAT_FILE);
|
||||
let rollback_dir = Uuid::new_v4();
|
||||
let rollback_path = object_dir.join(rollback_dir.to_string());
|
||||
fs::create_dir_all(&rollback_path)
|
||||
.await
|
||||
.expect("rollback dir should be created");
|
||||
fs::write(rollback_path.join(STORAGE_FORMAT_FILE_BACKUP), b"old-meta")
|
||||
.await
|
||||
.expect("backup should be written");
|
||||
|
||||
restore_metadata_backup(&object_dir, &xl_path, rollback_dir)
|
||||
.await
|
||||
.expect("restore should succeed");
|
||||
assert_eq!(
|
||||
fs::read(&xl_path).await.expect("xl.meta should be restored"),
|
||||
b"old-meta",
|
||||
"restore must move the backup back onto xl.meta"
|
||||
);
|
||||
assert!(!rollback_path.exists(), "an emptied synthetic rollback dir must be reclaimed");
|
||||
|
||||
// Real data dir: parts remain, the dir must survive.
|
||||
let real_dir = Uuid::new_v4();
|
||||
let real_path = object_dir.join(real_dir.to_string());
|
||||
fs::create_dir_all(&real_path).await.expect("real data dir should be created");
|
||||
fs::write(real_path.join(STORAGE_FORMAT_FILE_BACKUP), b"older-meta")
|
||||
.await
|
||||
.expect("backup should be written");
|
||||
fs::write(real_path.join("part.1"), b"data")
|
||||
.await
|
||||
.expect("part should be written");
|
||||
|
||||
restore_metadata_backup(&object_dir, &xl_path, real_dir)
|
||||
.await
|
||||
.expect("restore should succeed");
|
||||
assert!(real_path.join("part.1").exists(), "a real data dir must keep its parts");
|
||||
assert!(real_path.exists(), "a non-empty data dir must not be removed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_commit_failure_cleans_local_rollback_backup() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
@@ -42,10 +42,6 @@ pub const PART_TRANSACTION_NEW_META: &str = "new.meta";
|
||||
pub const PART_TRANSACTION_OLD_META: &str = "old.meta";
|
||||
pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_DISK: &str = "disk";
|
||||
const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified";
|
||||
|
||||
pub fn part_transaction_path(part_path: &str) -> String {
|
||||
match part_path.rsplit_once('/') {
|
||||
Some((parent, name)) => format!("{parent}/.{name}.rustfs-txn"),
|
||||
@@ -1200,13 +1196,7 @@ pub fn conv_part_err_to_int(err: &Option<Error>) -> usize {
|
||||
Some(DiskError::DiskNotFound) => CHECK_PART_DISK_NOT_FOUND,
|
||||
None => CHECK_PART_SUCCESS,
|
||||
_ => {
|
||||
tracing::warn!(
|
||||
event = EVENT_DISK_PART_ERR_UNCLASSIFIED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK,
|
||||
error = ?err,
|
||||
"Part error has no check-part code and degrades to unknown"
|
||||
);
|
||||
tracing::warn!("conv_part_err_to_int: unknown error: {err:?}");
|
||||
CHECK_PART_UNKNOWN
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ use crate::io_support::rio::HashReader;
|
||||
use crate::object_api::{BLOCK_SIZE_V2, ObjectLockConfigSnapshot, ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::multipart::{CompletePart, MultipartOperations as _};
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use crate::store::init_format::save_format_file;
|
||||
@@ -207,28 +206,8 @@ async fn blackbox_get_restores_body_after_one_shard_file_is_removed() {
|
||||
#[tokio::test]
|
||||
// Serialized: forces the reader-setup strategy through a process-global env var.
|
||||
#[serial_test::serial]
|
||||
async fn blackbox_heal_requests_preserve_repair_scope() {
|
||||
use rustfs_common::heal_channel::{
|
||||
HealAdmissionResult, HealChannelCommand, HealChannelPriority, HealChannelReceiver, HealChannelRequest, HealRequestSource,
|
||||
};
|
||||
|
||||
async fn receive_matching_heal(rx: &mut HealChannelReceiver, bucket: &str, object: &str) -> HealChannelRequest {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), async {
|
||||
loop {
|
||||
match rx.recv().await.expect("heal channel should stay open") {
|
||||
HealChannelCommand::Start { request, response_tx }
|
||||
if request.bucket == bucket && request.object_prefix.as_deref() == Some(object) =>
|
||||
{
|
||||
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
|
||||
break request;
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("matching heal request should be submitted")
|
||||
}
|
||||
async fn blackbox_get_restores_body_and_enqueues_repair_after_one_corrupt_shard() {
|
||||
use rustfs_common::heal_channel::{HealAdmissionResult, HealChannelCommand, HealChannelPriority, HealRequestSource};
|
||||
|
||||
// Own the process-global heal channel so the read path's repair submission
|
||||
// becomes observable. init_heal_channel() succeeds exactly once per test
|
||||
@@ -241,126 +220,6 @@ async fn blackbox_heal_requests_preserve_repair_scope() {
|
||||
let mut heal_rx = rustfs_common::heal_channel::init_heal_channel()
|
||||
.expect("this must be the only ecstore test that owns the heal channel receiver");
|
||||
|
||||
// Ordinary PUTs use the same admission channel as read repair. A single
|
||||
// rename target failure still satisfies write quorum, so the committed
|
||||
// version must be queued for convergence without delaying the PUT ACK.
|
||||
let (_put_dirs, put_set) = make_local_set_disks(4, 2).await;
|
||||
let put_bucket = "bb-put-partial-convergence";
|
||||
let put_object = "object.bin";
|
||||
put_set
|
||||
.make_bucket(put_bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("PUT bucket should be created");
|
||||
let offline_disk = {
|
||||
let mut disks = put_set.disks.write().await;
|
||||
disks[0].take()
|
||||
};
|
||||
let mut put_reader = PutObjReader::from_vec(vec![0x42; BLOCK_SIZE_V2 + 1024]);
|
||||
let committed = put_set
|
||||
.put_object(
|
||||
put_bucket,
|
||||
put_object,
|
||||
&mut put_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("partial ordinary PUT should succeed at write quorum");
|
||||
let committed_version = committed
|
||||
.version_id
|
||||
.expect("versioned PUT should return a version id")
|
||||
.to_string();
|
||||
|
||||
let request = tokio::time::timeout(std::time::Duration::from_secs(30), async {
|
||||
loop {
|
||||
match heal_rx.recv().await.expect("heal channel should stay open") {
|
||||
HealChannelCommand::Start { request, response_tx }
|
||||
if request.bucket == put_bucket && request.object_prefix.as_deref() == Some(put_object) =>
|
||||
{
|
||||
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
|
||||
break request;
|
||||
}
|
||||
HealChannelCommand::Start { response_tx, .. } => {
|
||||
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("partial ordinary PUT should enqueue convergence heal");
|
||||
assert_eq!(request.object_version_id.as_deref(), Some(committed_version.as_str()));
|
||||
assert_eq!(request.pool_index, Some(0));
|
||||
assert_eq!(request.set_index, Some(0));
|
||||
|
||||
let duplicate_request = tokio::time::timeout(std::time::Duration::from_millis(100), async {
|
||||
loop {
|
||||
match heal_rx.recv().await.expect("heal channel should stay open") {
|
||||
HealChannelCommand::Start { request, response_tx }
|
||||
if request.bucket == put_bucket && request.object_prefix.as_deref() == Some(put_object) =>
|
||||
{
|
||||
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
|
||||
break Some(request);
|
||||
}
|
||||
HealChannelCommand::Start { response_tx, .. } => {
|
||||
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
assert!(duplicate_request.is_none(), "partial ordinary PUT must enqueue exactly one heal request");
|
||||
|
||||
{
|
||||
let mut disks = put_set.disks.write().await;
|
||||
disks[0] = offline_disk;
|
||||
}
|
||||
|
||||
let healthy_bucket = "bb-put-healthy-convergence";
|
||||
put_set
|
||||
.make_bucket(healthy_bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("healthy PUT bucket should be created");
|
||||
let mut healthy_reader = PutObjReader::from_vec(b"healthy".to_vec());
|
||||
put_set
|
||||
.put_object(
|
||||
healthy_bucket,
|
||||
put_object,
|
||||
&mut healthy_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("healthy ordinary PUT should succeed");
|
||||
let healthy_request = tokio::time::timeout(std::time::Duration::from_millis(100), async {
|
||||
loop {
|
||||
match heal_rx.recv().await.expect("heal channel should stay open") {
|
||||
HealChannelCommand::Start { request, response_tx }
|
||||
if request.bucket == healthy_bucket && request.object_prefix.as_deref() == Some(put_object) =>
|
||||
{
|
||||
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
|
||||
break Some(request);
|
||||
}
|
||||
HealChannelCommand::Start { response_tx, .. } => {
|
||||
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
assert!(healthy_request.is_none(), "fully converged ordinary PUT must not enqueue heal");
|
||||
|
||||
// Keep data-blocks-first reader setup explicit for this deterministic
|
||||
// repair assertion (see ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP in
|
||||
// set_disk/core/io_primitives.rs): if a caller opts back into all-shards,
|
||||
@@ -417,7 +276,19 @@ async fn blackbox_heal_requests_preserve_repair_scope() {
|
||||
|
||||
assert_eq!(restored, payload);
|
||||
|
||||
let request = receive_matching_heal(&mut heal_rx, bucket, object).await;
|
||||
let request = tokio::time::timeout(std::time::Duration::from_secs(30), async {
|
||||
loop {
|
||||
match heal_rx.recv().await.expect("heal channel should stay open") {
|
||||
HealChannelCommand::Start { request, response_tx } if request.bucket == bucket => {
|
||||
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
|
||||
break request;
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("corrupt-shard GET should enqueue a read-repair heal request");
|
||||
|
||||
assert_eq!(request.source, HealRequestSource::ReadRepair);
|
||||
assert_eq!(request.object_prefix.as_deref(), Some(object));
|
||||
@@ -426,142 +297,6 @@ async fn blackbox_heal_requests_preserve_repair_scope() {
|
||||
assert_eq!(request.set_index, Some(0));
|
||||
assert_eq!(request.priority, HealChannelPriority::Low);
|
||||
assert_eq!(request.recreate_missing, Some(true));
|
||||
|
||||
let mpu_bucket = "bb-mpu-convergence-heal";
|
||||
let partial_object = "partial.bin";
|
||||
let suspended_object = "suspended.bin";
|
||||
let payload = vec![0x5a; 1 << 20];
|
||||
let mpu_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(mpu_bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("multipart bucket should be created");
|
||||
|
||||
let stage_upload = async |object: &str| {
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(mpu_bucket, object, &mpu_opts)
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let mut reader = PutObjReader::new(
|
||||
HashReader::from_stream(
|
||||
Cursor::new(payload.clone()),
|
||||
payload.len() as i64,
|
||||
payload.len() as i64,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.expect("multipart reader should be constructed"),
|
||||
);
|
||||
let part = set_disks
|
||||
.put_object_part(mpu_bucket, object, &upload.upload_id, 1, &mut reader, &mpu_opts)
|
||||
.await
|
||||
.expect("multipart part should be written");
|
||||
(
|
||||
upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
)
|
||||
};
|
||||
|
||||
let (partial_upload_id, partial_parts) = stage_upload(partial_object).await;
|
||||
let offline_disk = {
|
||||
let mut disks = set_disks.disks.write().await;
|
||||
disks[3].take().expect("fourth disk should be online before completion")
|
||||
};
|
||||
crate::crash_inject::arm(crate::crash_inject::CrashPoint::MultipartAfterCommitBeforePartsCleanup, partial_object);
|
||||
let completed = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(mpu_bucket, partial_object, &partial_upload_id, partial_parts, &mpu_opts)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(completed, Err(Error::Unexpected)),
|
||||
"partial multipart completion should reach the post-commit crash point, got {completed:?}"
|
||||
);
|
||||
crate::crash_inject::disarm(crate::crash_inject::CrashPoint::MultipartAfterCommitBeforePartsCleanup, partial_object);
|
||||
|
||||
let request = receive_matching_heal(&mut heal_rx, mpu_bucket, partial_object).await;
|
||||
|
||||
let completed_version_id = request
|
||||
.object_version_id
|
||||
.clone()
|
||||
.expect("versioned multipart convergence heal must bind a version id");
|
||||
assert_eq!(request.pool_index, Some(0));
|
||||
assert_eq!(request.set_index, Some(0));
|
||||
assert_eq!(request.priority, HealChannelPriority::Normal);
|
||||
|
||||
let duplicate = tokio::time::timeout(std::time::Duration::from_millis(250), async {
|
||||
loop {
|
||||
match heal_rx.recv().await.expect("heal channel should stay open") {
|
||||
HealChannelCommand::Start { request, .. }
|
||||
if request.bucket == mpu_bucket && request.object_prefix.as_deref() == Some(partial_object) =>
|
||||
{
|
||||
break request;
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(duplicate.is_err(), "partial multipart completion must enqueue exactly one heal request");
|
||||
|
||||
{
|
||||
let mut disks = set_disks.disks.write().await;
|
||||
disks[3] = Some(offline_disk);
|
||||
}
|
||||
let committed = set_disks
|
||||
.get_object_info(
|
||||
mpu_bucket,
|
||||
partial_object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
version_id: Some(completed_version_id.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("heal-bound multipart version should be committed and addressable");
|
||||
assert_eq!(committed.version_id.map(|version_id| version_id.to_string()), Some(completed_version_id));
|
||||
|
||||
let (suspended_upload_id, suspended_parts) = stage_upload(suspended_object).await;
|
||||
let offline_disk = {
|
||||
let mut disks = set_disks.disks.write().await;
|
||||
disks[3]
|
||||
.take()
|
||||
.expect("fourth disk should be online before suspended completion")
|
||||
};
|
||||
let suspended_opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
};
|
||||
let suspended = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(mpu_bucket, suspended_object, &suspended_upload_id, suspended_parts, &suspended_opts)
|
||||
.await
|
||||
.expect("suspended multipart completion should succeed at write quorum");
|
||||
{
|
||||
let mut disks = set_disks.disks.write().await;
|
||||
disks[3] = Some(offline_disk);
|
||||
}
|
||||
assert!(
|
||||
suspended.version_id.is_some_and(|version_id| version_id.is_nil()),
|
||||
"suspended multipart completion should publish the null version"
|
||||
);
|
||||
|
||||
let request = receive_matching_heal(&mut heal_rx, mpu_bucket, suspended_object).await;
|
||||
|
||||
let null_version_id = uuid::Uuid::nil().to_string();
|
||||
assert_eq!(request.object_version_id.as_deref(), Some(null_version_id.as_str()));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -20,21 +20,12 @@ use bytes::{Bytes, BytesMut};
|
||||
use reed_solomon_erasure::galois_8::ReedSolomon;
|
||||
use reed_solomon_simd;
|
||||
use smallvec::SmallVec;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
io,
|
||||
sync::{Arc, OnceLock, RwLock},
|
||||
};
|
||||
use std::io;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
const MODERN_MAX_TOTAL_SHARDS: usize = <reed_solomon_erasure::galois_8::Field as reed_solomon_erasure::Field>::ORDER;
|
||||
const MODERN_REED_SOLOMON_CACHE_MAX_ENTRIES: usize = 64;
|
||||
|
||||
type ModernReedSolomonCache = RwLock<HashMap<(usize, usize), Arc<ReedSolomon>>>;
|
||||
|
||||
static MODERN_REED_SOLOMON_CACHE: OnceLock<ModernReedSolomonCache> = OnceLock::new();
|
||||
|
||||
/// Errors returned when constructing an [`Erasure`] codec.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
@@ -284,7 +275,7 @@ impl LegacyReedSolomonEncoder {
|
||||
pub struct ReedSolomonEncoder {
|
||||
data_shards: usize,
|
||||
parity_shards: usize,
|
||||
encoder: Option<Arc<ReedSolomon>>,
|
||||
encoder: Option<ReedSolomon>,
|
||||
}
|
||||
|
||||
impl Clone for ReedSolomonEncoder {
|
||||
@@ -300,7 +291,7 @@ impl Clone for ReedSolomonEncoder {
|
||||
impl ReedSolomonEncoder {
|
||||
fn try_new_typed(data_shards: usize, parity_shards: usize) -> Result<Self, reed_solomon_erasure::Error> {
|
||||
let encoder = if parity_shards > 0 {
|
||||
Some(cached_modern_reed_solomon(data_shards, parity_shards)?)
|
||||
Some(ReedSolomon::new(data_shards, parity_shards)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -371,30 +362,6 @@ impl ReedSolomonEncoder {
|
||||
}
|
||||
}
|
||||
|
||||
fn cached_modern_reed_solomon(data_shards: usize, parity_shards: usize) -> Result<Arc<ReedSolomon>, reed_solomon_erasure::Error> {
|
||||
let key = (data_shards, parity_shards);
|
||||
let cache = MODERN_REED_SOLOMON_CACHE.get_or_init(|| RwLock::new(HashMap::new()));
|
||||
|
||||
if let Some(encoder) = cache
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.get(&key)
|
||||
.cloned()
|
||||
{
|
||||
return Ok(encoder);
|
||||
}
|
||||
|
||||
let encoder = Arc::new(ReedSolomon::new(data_shards, parity_shards)?);
|
||||
let mut cache = cache.write().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(existing) = cache.get(&key) {
|
||||
return Ok(Arc::clone(existing));
|
||||
}
|
||||
if cache.len() < MODERN_REED_SOLOMON_CACHE_MAX_ENTRIES {
|
||||
cache.insert(key, Arc::clone(&encoder));
|
||||
}
|
||||
Ok(encoder)
|
||||
}
|
||||
|
||||
fn encode_parity_shards<F>(shards: &mut [Option<Vec<u8>>], data_shards: usize, parity_shards: usize, encode: F) -> io::Result<()>
|
||||
where
|
||||
F: FnOnce(SmallVec<[&mut [u8]; 16]>) -> io::Result<()>,
|
||||
@@ -1305,16 +1272,6 @@ mod tests {
|
||||
assert!(legacy.legacy_encoder.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn modern_encoder_construction_reuses_cached_codec() {
|
||||
let first = ReedSolomonEncoder::try_new_typed(31, 7).expect("modern codec should construct");
|
||||
let second = ReedSolomonEncoder::try_new_typed(31, 7).expect("modern codec should construct");
|
||||
|
||||
let first = first.encoder.as_ref().expect("modern codec should initialize an encoder");
|
||||
let second = second.encoder.as_ref().expect("modern codec should initialize an encoder");
|
||||
assert!(Arc::ptr_eq(first, second));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn construction_errors_preserve_encoder_sources() {
|
||||
let modern = ErasureConstructionError::ModernEncoder {
|
||||
|
||||
@@ -24,11 +24,7 @@ use std::io;
|
||||
use std::io::ErrorKind;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::{trace, warn};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_ERASURE: &str = "erasure";
|
||||
const EVENT_ERASURE_HEAL_STARTED: &str = "erasure_heal_started";
|
||||
use tracing::{info, warn};
|
||||
|
||||
async fn read_heal_shards<R>(
|
||||
readers: &mut [Option<BitrotReader<R>>],
|
||||
@@ -119,14 +115,11 @@ impl super::Erasure {
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync,
|
||||
{
|
||||
trace!(
|
||||
event = EVENT_ERASURE_HEAL_STARTED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE,
|
||||
writer_count = writers.len(),
|
||||
reader_count = readers.len(),
|
||||
total_length,
|
||||
"Erasure heal started"
|
||||
info!(
|
||||
"Erasure heal, writers len: {}, readers len: {}, total_length: {}",
|
||||
writers.len(),
|
||||
readers.len(),
|
||||
total_length
|
||||
);
|
||||
if writers.len() != self.parity_shards + self.data_shards {
|
||||
return Err(Error::other("invalid argument"));
|
||||
|
||||
@@ -380,12 +380,6 @@ impl WritePlan {
|
||||
}
|
||||
|
||||
pub fn apply(self, mut reader: HashReader, actual_size: i64) -> std::io::Result<HashReader> {
|
||||
// Transformations create new HashReaders around the plaintext reader. Keep
|
||||
// the request checksum metadata on the final reader for multipart/single
|
||||
// PUT persistence, but leave verification to the plaintext reader.
|
||||
let checksum = reader.content_hash().clone();
|
||||
let trailer = reader.get_trailer().cloned();
|
||||
|
||||
let encrypted = self.encryption.is_some();
|
||||
if let Some(algorithm) = self.compression {
|
||||
reader = HashReader::from_reader(
|
||||
@@ -444,12 +438,6 @@ impl WritePlan {
|
||||
};
|
||||
}
|
||||
|
||||
// `ignore_value` deliberately avoids a second hasher over compressed or
|
||||
// encrypted bytes. The inner reader still validates the plaintext request
|
||||
// checksum while this outer reader exposes the request checksum context.
|
||||
reader.add_non_trailing_checksum(checksum, true)?;
|
||||
reader.set_trailer(trailer);
|
||||
|
||||
Ok(reader)
|
||||
}
|
||||
}
|
||||
@@ -457,73 +445,10 @@ impl WritePlan {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use rustfs_rio::{Checksum, ChecksumType};
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
|
||||
let plaintext = b"checksum-context-through-write-plan".repeat(256);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
let checksum = Checksum::new_from_data(ChecksumType::CRC32, &plaintext).expect("create CRC32 checksum");
|
||||
let mut reader = HashReader::from_stream(Cursor::new(plaintext), actual_size, actual_size, None, None, false)
|
||||
.expect("create hash reader");
|
||||
reader
|
||||
.add_non_trailing_checksum(Some(checksum.clone()), false)
|
||||
.expect("attach plaintext checksum");
|
||||
|
||||
let mut transformed = plan.apply(reader, actual_size).expect("apply write plan");
|
||||
assert_eq!(transformed.content_crc_type(), Some(ChecksumType::CRC32));
|
||||
|
||||
let mut transformed_bytes = Vec::new();
|
||||
transformed
|
||||
.read_to_end(&mut transformed_bytes)
|
||||
.await
|
||||
.expect("stream transformed data without rehashing ciphertext");
|
||||
|
||||
assert!(!transformed_bytes.is_empty());
|
||||
assert_eq!(transformed.content_crc().get("CRC32"), Some(&checksum.encoded));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_plan_preserves_non_trailing_checksum_context_across_transforms() {
|
||||
assert_non_trailing_checksum_survives(WritePlan::new().with_compression(CompressionAlgorithm::default())).await;
|
||||
assert_non_trailing_checksum_survives(
|
||||
WritePlan::new().with_encryption(WriteEncryption::singlepart([0x5Au8; 32], [0xA5u8; 12])),
|
||||
)
|
||||
.await;
|
||||
assert_non_trailing_checksum_survives(
|
||||
WritePlan::new()
|
||||
.with_compression(CompressionAlgorithm::default())
|
||||
.with_encryption(WriteEncryption::singlepart([0x5Au8; 32], [0xA5u8; 12])),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_plan_preserves_trailing_checksum_type_across_transforms() {
|
||||
let plaintext = b"trailing-checksum-context".to_vec();
|
||||
let actual_size = plaintext.len() as i64;
|
||||
let mut reader = HashReader::from_stream(Cursor::new(plaintext), actual_size, actual_size, None, None, false)
|
||||
.expect("create hash reader");
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-trailer", HeaderValue::from_static("x-amz-checksum-crc32"));
|
||||
reader
|
||||
.add_checksum_from_s3s(&headers, None, false)
|
||||
.expect("attach trailing checksum metadata");
|
||||
|
||||
let transformed = WritePlan::new()
|
||||
.with_encryption(WriteEncryption::singlepart([0x5Au8; 32], [0xA5u8; 12]))
|
||||
.apply(reader, actual_size)
|
||||
.expect("apply encryption plan");
|
||||
|
||||
assert_eq!(
|
||||
transformed.content_crc_type(),
|
||||
Some(ChecksumType(ChecksumType::CRC32.0 | ChecksumType::TRAILING.0))
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
fn s2_chunk_types(stream: &[u8]) -> Vec<u8> {
|
||||
let mut chunk_types = Vec::new();
|
||||
|
||||
@@ -246,22 +246,6 @@ pub fn deployment_id() -> Option<String> {
|
||||
get_global_deployment_id()
|
||||
}
|
||||
|
||||
/// Test-only inverse of [`deployment_upload_id`]: returns the raw
|
||||
/// `<uuid>x<timestamp>` suffix without the deployment-id prefix. Under plain
|
||||
/// `cargo test` (thread-parallel, shared process globals) a concurrently
|
||||
/// running test that re-initializes a store can swap the global deployment id
|
||||
/// between create time and list time, so assertions must compare only this
|
||||
/// suffix, never the full encoded upload id.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn upload_uuid_suffix(upload_id: &str) -> String {
|
||||
base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(upload_id.as_bytes())
|
||||
.ok()
|
||||
.and_then(|decoded| String::from_utf8(decoded).ok())
|
||||
.and_then(|decoded| decoded.split_once('.').map(|(_, suffix)| suffix.to_owned()))
|
||||
.unwrap_or_else(|| upload_id.to_owned())
|
||||
}
|
||||
|
||||
pub(crate) fn replication_pool() -> Option<Arc<DynReplicationPool>> {
|
||||
crate::runtime::global::current_ctx().replication_pool()
|
||||
}
|
||||
@@ -565,13 +549,8 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
endpoint_pools: EndpointServerPools,
|
||||
opt: &DiskOption,
|
||||
) -> Result<()> {
|
||||
// Every caller passes the FULL topology, so (re)initialization must replace
|
||||
// any previous registration wholesale: appending would leave the pool/set
|
||||
// vectors sized for a stale topology and panic on wider disk indices (seen
|
||||
// as cross-test contamination under single-process `cargo test`).
|
||||
let set_drives = instance_ctx.local_disk_set_drives();
|
||||
let mut global_set_drives = set_drives.write().await;
|
||||
global_set_drives.clear();
|
||||
for pool_eps in endpoint_pools.as_ref().iter() {
|
||||
let mut set_count_drives = Vec::with_capacity(pool_eps.set_count);
|
||||
for _ in 0..pool_eps.set_count {
|
||||
@@ -583,7 +562,6 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
|
||||
let map = instance_ctx.local_disk_map();
|
||||
let mut global_local_disk_map = map.write().await;
|
||||
global_local_disk_map.clear();
|
||||
|
||||
for pool_eps in endpoint_pools.as_ref().iter() {
|
||||
for ep in pool_eps.endpoints.as_ref().iter() {
|
||||
@@ -749,69 +727,4 @@ mod tests {
|
||||
process_ctx.local_disk_id_map().write().await.remove(&process_sentinel);
|
||||
bootstrap_ctx.local_disk_id_map().write().await.remove(&bootstrap_sentinel);
|
||||
}
|
||||
|
||||
/// Re-initializing the same context with a WIDER topology must replace the
|
||||
/// previous registration, not append to it: the stale pool-0 drive vector
|
||||
/// (sized for the narrow topology) made `global_set_drives[0][0][disk_idx]`
|
||||
/// panic for the wider set's higher disk indices. CI's nextest
|
||||
/// process-per-test isolation never exercises re-init, so this pins it.
|
||||
#[tokio::test]
|
||||
async fn reinitializing_local_disk_maps_replaces_previous_topology() {
|
||||
use crate::disk::DiskOption;
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("reinit test directory should be created");
|
||||
let build_pools = |label: &str, disk_count: usize| {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_idx in 0..disk_count {
|
||||
let disk_path = temp_dir.path().join(format!("{label}-disk{disk_idx}"));
|
||||
std::fs::create_dir_all(&disk_path).expect("reinit test disk should be created");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_idx);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: disk_count,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("reinit-test-{label}"),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
}])
|
||||
};
|
||||
let opt = DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
};
|
||||
|
||||
let instance_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
super::initialize_local_disk_maps(&instance_ctx, build_pools("narrow", 2), &opt)
|
||||
.await
|
||||
.expect("narrow topology should initialize");
|
||||
super::initialize_local_disk_maps(&instance_ctx, build_pools("wide", 4), &opt)
|
||||
.await
|
||||
.expect("re-initializing with a wider topology must not panic or fail");
|
||||
|
||||
let set_drives = instance_ctx.local_disk_set_drives();
|
||||
let set_drives = set_drives.read().await;
|
||||
assert_eq!(set_drives.len(), 1, "stale pools must not accumulate across re-inits");
|
||||
assert_eq!(set_drives[0][0].len(), 4, "pool 0 set 0 must be sized for the new topology");
|
||||
assert!(
|
||||
set_drives[0][0].iter().all(Option::is_some),
|
||||
"every wide-topology drive slot must be registered"
|
||||
);
|
||||
drop(set_drives);
|
||||
|
||||
let disk_map = instance_ctx.local_disk_map();
|
||||
let disk_map = disk_map.read().await;
|
||||
assert_eq!(disk_map.len(), 4, "stale narrow-topology disk entries must be dropped");
|
||||
assert!(
|
||||
disk_map.keys().all(|path| path.contains("wide-disk")),
|
||||
"only the new topology's disks may remain registered: {:?}",
|
||||
disk_map.keys().collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,9 +15,7 @@
|
||||
use crate::diagnostics::admin_server_info::get_local_server_property;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::admin::StorageAdminApi;
|
||||
#[cfg(test)]
|
||||
use chrono::Utc;
|
||||
use jiff::Timestamp;
|
||||
use rustfs_common::{heal_channel::DriveState, metrics::global_metrics};
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use rustfs_madmin::metrics::{
|
||||
@@ -69,18 +67,6 @@ impl MetricType {
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_millis_to_jiff_timestamp(millis: u64, fallback: Timestamp) -> Timestamp {
|
||||
let millis = match i64::try_from(millis) {
|
||||
Ok(millis) => millis,
|
||||
Err(_) => return fallback,
|
||||
};
|
||||
|
||||
match Timestamp::from_millisecond(millis) {
|
||||
Ok(timestamp) => timestamp,
|
||||
Err(_) => fallback,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsReport) -> MadminScannerMetrics {
|
||||
MadminScannerMetrics {
|
||||
collected_at: metrics.collected_at,
|
||||
@@ -400,7 +386,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
|
||||
if types.contains(&MetricType::DISK) {
|
||||
debug!("start get disk metrics");
|
||||
let mut aggr = DiskMetric {
|
||||
collected_at: Timestamp::now(),
|
||||
collected_at: Utc::now(),
|
||||
..Default::default()
|
||||
};
|
||||
for (name, disk) in collect_local_disks_metrics(&opts.disks).await.into_iter() {
|
||||
@@ -426,7 +412,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
|
||||
if types.contains(&MetricType::NET) {
|
||||
let snapshot = global_internode_metrics().snapshot();
|
||||
real_time_metrics.aggregated.net = Some(NetMetrics {
|
||||
collected_at: Timestamp::now(),
|
||||
collected_at: Utc::now(),
|
||||
interface_name: "internode".to_string(),
|
||||
net_stats: NetDevLine {
|
||||
name: "internode".to_string(),
|
||||
@@ -442,9 +428,10 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
|
||||
// if types.contains(&MetricType::CPU) {}
|
||||
|
||||
if types.contains(&MetricType::RPC) {
|
||||
let collected_at = Timestamp::now();
|
||||
let collected_at = Utc::now();
|
||||
let snapshot = global_internode_metrics().snapshot();
|
||||
let last_connect_time = unix_millis_to_jiff_timestamp(snapshot.last_dial_unix_millis, collected_at);
|
||||
let last_connect_time =
|
||||
chrono::DateTime::<Utc>::from_timestamp_millis(snapshot.last_dial_unix_millis as i64).unwrap_or(collected_at);
|
||||
|
||||
real_time_metrics.aggregated.rpc = Some(RPCMetrics {
|
||||
collected_at,
|
||||
@@ -556,10 +543,6 @@ mod test {
|
||||
use serial_test::serial;
|
||||
use std::time::Duration;
|
||||
|
||||
fn chrono_to_jiff_timestamp(timestamp: chrono::DateTime<Utc>) -> jiff::Timestamp {
|
||||
jiff::Timestamp::try_from(std::time::SystemTime::from(timestamp)).expect("test timestamp should fit in jiff")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tes_types() {
|
||||
let t = MetricType::ALL;
|
||||
@@ -608,7 +591,7 @@ mod test {
|
||||
let current_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
|
||||
current_cycle_active: true,
|
||||
current_started: chrono_to_jiff_timestamp(current_started),
|
||||
current_started,
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
|
||||
@@ -619,7 +602,7 @@ mod test {
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
|
||||
assert_eq!(scanner.current_started, current_started);
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
let usage = scanner
|
||||
@@ -660,7 +643,7 @@ mod test {
|
||||
aggregated.merge(decoded);
|
||||
let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(cycle_started));
|
||||
assert_eq!(scanner.current_started, cycle_started);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -18,8 +18,8 @@ use super::meta::{
|
||||
};
|
||||
use super::migration::migrate_entry_version;
|
||||
use super::worker::{
|
||||
RebalanceEntryCleanupResult, RebalanceEntryTask, load_rebalance_bucket_configs, rebalance_max_attempts,
|
||||
resolve_rebalance_bucket_error, resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
|
||||
RebalanceEntryTask, load_rebalance_bucket_configs, rebalance_max_attempts, resolve_rebalance_bucket_error,
|
||||
resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
|
||||
resolve_rebalance_migrate_result_error, resolve_rebalance_stats_update_result, resolve_rebalance_worker_result,
|
||||
run_rebalance_listing_with_retry, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete,
|
||||
should_defer_rebalance_entry_failure, should_skip_rebalance_delete_marker, wait_rebalance_entry_tasks,
|
||||
@@ -27,7 +27,7 @@ use super::worker::{
|
||||
};
|
||||
use super::{
|
||||
EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_ENTRY, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
|
||||
ObjectInfo, REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome,
|
||||
REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome,
|
||||
};
|
||||
use crate::core::pools::ListCallback;
|
||||
use crate::data_movement;
|
||||
@@ -37,55 +37,13 @@ use crate::object_api::{GetObjectReader, ObjectOptions};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::object::ObjectOperations as _;
|
||||
use crate::store::ECStore;
|
||||
use rustfs_filemeta::{FileInfo, MetaCacheEntry};
|
||||
use rustfs_filemeta::MetaCacheEntry;
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
impl ECStore {
|
||||
async fn finish_rebalance_entry_after_cleanup(
|
||||
&self,
|
||||
pool_index: usize,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
stats_updates: &[&FileInfo],
|
||||
cleanup: impl std::future::Future<Output = std::result::Result<ObjectInfo, data_movement::SourceCleanupError>>,
|
||||
) -> Result<RebalanceEntryCleanupResult> {
|
||||
// Persisted stats can complete a pool on restart, so source cleanup must resolve first.
|
||||
let cleanup_result = resolve_rebalance_entry_cleanup_delete_result(cleanup.await, bucket, object);
|
||||
let RebalanceEntryCleanupResult::Completed { warning } = cleanup_result else {
|
||||
return Ok(cleanup_result);
|
||||
};
|
||||
if let Some(message) = warning.as_ref()
|
||||
&& let Err(err) = self
|
||||
.record_rebalance_cleanup_warning(pool_index, bucket, object, message.clone())
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket,
|
||||
object,
|
||||
stage = "cleanup_source",
|
||||
error = ?err,
|
||||
"Failed to record rebalance source cleanup warning"
|
||||
);
|
||||
}
|
||||
|
||||
resolve_rebalance_stats_update_result(
|
||||
self.update_pool_stats_batch(pool_index, bucket.to_string(), stats_updates)
|
||||
.await,
|
||||
pool_index,
|
||||
bucket,
|
||||
object,
|
||||
)?;
|
||||
|
||||
Ok(RebalanceEntryCleanupResult::Completed { warning })
|
||||
}
|
||||
|
||||
#[allow(unused_assignments)]
|
||||
#[tracing::instrument(skip(self, set))]
|
||||
async fn rebalance_entry(
|
||||
@@ -258,7 +216,7 @@ impl ECStore {
|
||||
);
|
||||
if should_defer_rebalance_entry_failure(&err) {
|
||||
let deferred_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} {err}");
|
||||
debug!(
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
@@ -302,40 +260,46 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
resolve_rebalance_stats_update_result(
|
||||
self.update_pool_stats_batch(pool_index, bucket.clone(), stats_updates.as_slice())
|
||||
.await,
|
||||
pool_index,
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
)?;
|
||||
|
||||
if should_cleanup_rebalance_source_entry(rebalanced, fivs.versions.len(), expired) {
|
||||
let cleanup_result = self
|
||||
.finish_rebalance_entry_after_cleanup(
|
||||
pool_index,
|
||||
let cleanup_warning = resolve_rebalance_entry_cleanup_delete_result(
|
||||
data_movement::cleanup_source_entry_if_unchanged(
|
||||
set.clone(),
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
stats_updates.as_slice(),
|
||||
data_movement::cleanup_source_entry_if_unchanged(
|
||||
set.clone(),
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
&fivs,
|
||||
&cleanup_preflight_allowed_missing,
|
||||
"rebalance",
|
||||
),
|
||||
&fivs,
|
||||
&cleanup_preflight_allowed_missing,
|
||||
"rebalance",
|
||||
)
|
||||
.await?;
|
||||
match cleanup_result {
|
||||
RebalanceEntryCleanupResult::Deferred { last_error } => {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
state = "deferred",
|
||||
error = %last_error,
|
||||
"Deferred rebalance entry after source cleanup conflict"
|
||||
);
|
||||
return Ok(RebalanceEntryOutcome::Deferred { last_error });
|
||||
}
|
||||
RebalanceEntryCleanupResult::Completed { warning: Some(message) } => {
|
||||
warn!(
|
||||
.await,
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
)?;
|
||||
if let Some(message) = cleanup_warning {
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
stage = "cleanup_source",
|
||||
cleanup_status = "failed_ignored",
|
||||
error = %message,
|
||||
"Ignored rebalance source cleanup failure"
|
||||
);
|
||||
if let Err(err) = self
|
||||
.record_rebalance_cleanup_warning(pool_index, bucket.as_str(), entry.name.as_str(), message)
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
@@ -343,23 +307,21 @@ impl ECStore {
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
stage = "cleanup_source",
|
||||
cleanup_status = "failed_ignored",
|
||||
error = %message,
|
||||
"Ignored rebalance source cleanup failure"
|
||||
);
|
||||
}
|
||||
RebalanceEntryCleanupResult::Completed { warning: None } => {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
state = "source_deleted",
|
||||
"Deleted rebalance source entry"
|
||||
error = ?err,
|
||||
"Failed to record rebalance source cleanup warning"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
state = "source_deleted",
|
||||
"Deleted rebalance source entry"
|
||||
);
|
||||
}
|
||||
} else if rebalanced != fivs.versions.len() || expired > 0 {
|
||||
warn!(
|
||||
@@ -375,14 +337,6 @@ impl ECStore {
|
||||
state = "source_retained",
|
||||
"Rebalance source object retained"
|
||||
);
|
||||
|
||||
resolve_rebalance_stats_update_result(
|
||||
self.update_pool_stats_batch(pool_index, bucket.clone(), stats_updates.as_slice())
|
||||
.await,
|
||||
pool_index,
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(RebalanceEntryOutcome::Completed)
|
||||
@@ -539,23 +493,9 @@ impl ECStore {
|
||||
let entry_tasks = entry_tasks.clone();
|
||||
|
||||
let job = tokio::spawn(async move {
|
||||
let list_rx = rx.clone();
|
||||
let list_bucket = bucket.clone();
|
||||
let list_result = run_rebalance_listing_with_retry(
|
||||
rx,
|
||||
bucket,
|
||||
rebalance_entry,
|
||||
set_idx,
|
||||
rebalance_max_attempts(),
|
||||
entry_tasks.clone(),
|
||||
move |cb| {
|
||||
let set = set.clone();
|
||||
let rx = list_rx.clone();
|
||||
let bucket = list_bucket.clone();
|
||||
async move { set.list_objects_to_rebalance(rx, bucket, cb).await }
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let list_result =
|
||||
run_rebalance_listing_with_retry(set, rx, bucket.clone(), rebalance_entry, set_idx, rebalance_max_attempts())
|
||||
.await;
|
||||
let entry_result = wait_rebalance_entry_tasks(set_idx, entry_tasks).await;
|
||||
let result = list_result.and(entry_result);
|
||||
if let Err(err) = &result {
|
||||
@@ -608,124 +548,3 @@ impl ECStore {
|
||||
Ok(RebalanceBucketOutcome::Completed)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_stats_wait_for_source_cleanup_result() {
|
||||
let endpoint_pools: crate::layout::endpoints::EndpointServerPools = Vec::new().into();
|
||||
let store = Arc::new(ECStore {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
disk_map: std::collections::HashMap::new(),
|
||||
pools: Vec::new(),
|
||||
peer_sys: crate::cluster::rpc::S3PeerSys::new(&endpoint_pools),
|
||||
pool_meta: tokio::sync::RwLock::new(crate::core::pools::PoolMeta::default()),
|
||||
rebalance_meta: tokio::sync::RwLock::new(Some(RebalanceMeta {
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
start_time: Some(OffsetDateTime::now_utc()),
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
})),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
});
|
||||
let mut version = FileInfo::new("object.bin", 4, 2);
|
||||
version.name = "object.bin".to_string();
|
||||
version.size = 128;
|
||||
version.is_latest = true;
|
||||
let warning_version = version.clone();
|
||||
let (release_cleanup, cleanup_released) = tokio::sync::oneshot::channel();
|
||||
|
||||
let finish_store = Arc::clone(&store);
|
||||
let finish = tokio::spawn(async move {
|
||||
finish_store
|
||||
.finish_rebalance_entry_after_cleanup(0, "bucket", "object.bin", &[&version], async move {
|
||||
cleanup_released.await.expect("cleanup release sender should remain alive");
|
||||
Ok(ObjectInfo::default())
|
||||
})
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(
|
||||
store
|
||||
.rebalance_meta
|
||||
.read()
|
||||
.await
|
||||
.as_ref()
|
||||
.expect("rebalance metadata should exist")
|
||||
.pool_stats[0]
|
||||
.bytes,
|
||||
0,
|
||||
"stats must not become visible before source cleanup resolves"
|
||||
);
|
||||
|
||||
release_cleanup.send(()).expect("cleanup waiter should remain alive");
|
||||
assert_eq!(
|
||||
finish
|
||||
.await
|
||||
.expect("finish task should not panic")
|
||||
.expect("finish should succeed"),
|
||||
RebalanceEntryCleanupResult::Completed { warning: None }
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.rebalance_meta
|
||||
.read()
|
||||
.await
|
||||
.as_ref()
|
||||
.expect("rebalance metadata should exist")
|
||||
.pool_stats[0]
|
||||
.bytes
|
||||
> 0,
|
||||
"stats should become visible after source cleanup resolves"
|
||||
);
|
||||
|
||||
{
|
||||
let mut meta = store.rebalance_meta.write().await;
|
||||
meta.as_mut().expect("rebalance metadata should exist").pool_stats[0].bytes = 0;
|
||||
}
|
||||
let warning_result = store
|
||||
.finish_rebalance_entry_after_cleanup(0, "bucket", "object.bin", &[&warning_version], async {
|
||||
Err(Error::SlowDown.into())
|
||||
})
|
||||
.await
|
||||
.expect("cleanup warnings should not fail the completed migration");
|
||||
assert!(matches!(warning_result, RebalanceEntryCleanupResult::Completed { warning: Some(_) }));
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(pool_stats.cleanup_warnings.count, 1, "cleanup warning must block pool completion");
|
||||
assert!(pool_stats.bytes > 0, "completed migration bytes should still be recorded");
|
||||
drop(meta);
|
||||
|
||||
{
|
||||
let mut meta = store.rebalance_meta.write().await;
|
||||
meta.as_mut().expect("rebalance metadata should exist").pool_stats[0].bytes = 0;
|
||||
}
|
||||
let deferred = store
|
||||
.finish_rebalance_entry_after_cleanup(0, "bucket", "object.bin", &[&warning_version], async {
|
||||
Err(data_movement::SourceCleanupError::SourceChanged)
|
||||
})
|
||||
.await
|
||||
.expect("source changes should defer cleanup without failing the worker");
|
||||
assert!(matches!(deferred, RebalanceEntryCleanupResult::Deferred { .. }));
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(pool_stats.bytes, 0, "deferred cleanup must not commit completion stats");
|
||||
assert_eq!(pool_stats.cleanup_warnings.count, 1, "deferred cleanup must not add a permanent warning");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,14 +27,12 @@ const REBAL_META_FMT: u16 = 1; // Replace with actual format value
|
||||
const REBAL_META_VER: u16 = 1; // Replace with actual version value
|
||||
pub(crate) const REBAL_META_NAME: &str = "rebalance.bin";
|
||||
const DEFAULT_REBALANCE_MAX_ATTEMPTS: usize = 3;
|
||||
pub(crate) const REBALANCE_SOURCE_CLEANUP_MAX_DEFERS: usize = 3;
|
||||
const REBALANCE_MAX_ATTEMPTS_ENV: &str = "RUSTFS_REBALANCE_MAX_ATTEMPTS";
|
||||
const REBALANCE_STOP_PROPAGATION_ERROR_PREFIX: &str = "rebalance stop propagation incomplete: ";
|
||||
const REBALANCE_LISTING_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
|
||||
const REBALANCE_MIGRATION_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
|
||||
const REBALANCE_MIGRATION_LOCK_RETRY_CAP: Duration = Duration::from_secs(10);
|
||||
const REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX: &str = "deferred transient rebalance entry failure:";
|
||||
pub(crate) const REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX: &str = "deferred rebalance source cleanup conflict:";
|
||||
const REBALANCE_CLEANUP_WARNING_ENTRY_LIMIT: usize = 10;
|
||||
|
||||
mod control;
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX;
|
||||
use super::control::validate_rebalance_disk_stats_coverage;
|
||||
use super::meta::{
|
||||
RebalanceMetaMergeOutcome, RebalanceTerminalEvent, apply_rebalance_save_option, apply_rebalance_terminal_event,
|
||||
@@ -32,15 +33,13 @@ use super::migration::{
|
||||
MigrationBackend, MigrationVersionResult, migrate_entry_version, migrate_entry_version_with_retry_wait,
|
||||
rebalance_delete_marker_opts,
|
||||
};
|
||||
use super::runtime::{should_fail_repeated_rebalance_bucket_defer, source_cleanup_defer_attempt};
|
||||
use super::worker::{
|
||||
RebalanceEntryCleanupResult, ensure_rebalance_listing_disks_available, is_transient_rebalance_error,
|
||||
parse_rebalance_max_attempts, rebalance_listing_retry_delay, rebalance_migration_retry_delay,
|
||||
resolve_load_rebalance_stats_update_result, resolve_rebalance_bucket_error, resolve_rebalance_bucket_result,
|
||||
resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
|
||||
resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, resolve_rebalance_migrate_result_error,
|
||||
resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result, resolve_rebalance_stats_update_result,
|
||||
resolve_rebalance_terminal_error, resolve_rebalance_worker_result, run_rebalance_listing_with_retry,
|
||||
ensure_rebalance_listing_disks_available, is_transient_rebalance_error, parse_rebalance_max_attempts,
|
||||
rebalance_listing_retry_delay, rebalance_migration_retry_delay, resolve_load_rebalance_stats_update_result,
|
||||
resolve_rebalance_bucket_error, resolve_rebalance_bucket_result, resolve_rebalance_entry_cleanup_delete_result,
|
||||
resolve_rebalance_file_info_versions_result, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
|
||||
resolve_rebalance_migrate_result_error, resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result,
|
||||
resolve_rebalance_stats_update_result, resolve_rebalance_terminal_error, resolve_rebalance_worker_result,
|
||||
send_rebalance_done_signal, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete,
|
||||
should_defer_rebalance_entry_failure, should_retry_rebalance_listing, should_skip_rebalance_delete_marker,
|
||||
wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context,
|
||||
@@ -49,17 +48,15 @@ use super::{
|
||||
DiskStat, GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketConfigs,
|
||||
RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats,
|
||||
};
|
||||
use super::{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX};
|
||||
use crate::bucket::replication::{ReplicationState, ReplicationStatusType, replication_state_to_filemeta};
|
||||
use crate::data_movement;
|
||||
use crate::data_movement::SourceCleanupError;
|
||||
use crate::data_usage::DATA_USAGE_CACHE_NAME;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_filemeta::TRANSITION_COMPLETE;
|
||||
use rustfs_filemeta::{FileInfo, MetaCacheEntry};
|
||||
use rustfs_rio::Index;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use serde::Serialize;
|
||||
@@ -1668,63 +1665,26 @@ fn test_resolve_rebalance_meta_load_result_wraps_error_context() {
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_passthrough() {
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Ok(ObjectInfo::default()), "bucket-a", "obj.txt");
|
||||
assert_eq!(result, RebalanceEntryCleanupResult::Completed { warning: None });
|
||||
assert_eq!(result.expect("successful cleanup should pass through"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_ignores_not_found() {
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(
|
||||
Err(Error::ObjectNotFound("bucket-a".to_string(), "obj.txt".to_string()).into()),
|
||||
Err(Error::ObjectNotFound("bucket-a".to_string(), "obj.txt".to_string())),
|
||||
"bucket-a",
|
||||
"obj.txt",
|
||||
);
|
||||
assert_eq!(result, RebalanceEntryCleanupResult::Completed { warning: None });
|
||||
assert_eq!(result.expect("missing cleanup source should be ignored"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_returns_warning_for_failures() {
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::SlowDown.into()), "bucket-a", "obj.txt");
|
||||
assert!(matches!(
|
||||
result,
|
||||
RebalanceEntryCleanupResult::Completed { warning: Some(ref message) }
|
||||
if message.contains("rebalance cleanup delete failed for bucket-a/obj.txt")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_defers_source_change() {
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Err(SourceCleanupError::SourceChanged), "bucket-a", "obj.txt");
|
||||
assert!(matches!(
|
||||
result,
|
||||
RebalanceEntryCleanupResult::Deferred { ref last_error }
|
||||
if last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX)
|
||||
&& last_error.contains("source changed during cleanup preflight for bucket-a/obj.txt")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_does_not_defer_other_precondition_failure() {
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::PreconditionFailed.into()), "bucket-a", "obj.txt");
|
||||
assert!(matches!(
|
||||
result,
|
||||
RebalanceEntryCleanupResult::Completed { warning: Some(ref message) }
|
||||
if message.contains("rebalance cleanup delete failed for bucket-a/obj.txt")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_source_cleanup_defer_does_not_fail_repeated_bucket_retry() {
|
||||
let mut deferred_buckets = std::collections::HashSet::new();
|
||||
|
||||
assert!(!should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, "bucket-a", true));
|
||||
assert!(!should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, "bucket-a", true));
|
||||
assert!(!should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, "bucket-b", false));
|
||||
assert!(should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, "bucket-b", false));
|
||||
|
||||
let mut source_attempts = std::collections::HashMap::new();
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 1);
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 2);
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 3);
|
||||
let warning = resolve_rebalance_entry_cleanup_delete_result(Err(Error::SlowDown), "bucket-a", "obj.txt")
|
||||
.expect("cleanup delete failures should be downgraded to warnings")
|
||||
.expect("cleanup delete failure should return warning");
|
||||
let message = warning.as_str();
|
||||
assert!(message.contains("rebalance cleanup delete failed for bucket-a/obj.txt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1849,109 +1809,6 @@ fn test_should_retry_rebalance_listing_respects_attempt_limit_and_error_type() {
|
||||
assert!(!should_retry_rebalance_listing(&Error::FileAccessDenied, 0, 3));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_rebalance_listing_retry_waits_for_scheduled_entries() {
|
||||
let entry_tasks = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let task_registered = Arc::new(tokio::sync::Notify::new());
|
||||
let release_task = Arc::new(tokio::sync::Notify::new());
|
||||
let task_finished = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let callback: crate::core::pools::ListCallback = Arc::new({
|
||||
let entry_tasks = entry_tasks.clone();
|
||||
let task_registered = task_registered.clone();
|
||||
let release_task = release_task.clone();
|
||||
let task_finished = task_finished.clone();
|
||||
move |_| {
|
||||
let entry_tasks = entry_tasks.clone();
|
||||
let task_registered = task_registered.clone();
|
||||
let release_task = release_task.clone();
|
||||
let task_finished = task_finished.clone();
|
||||
Box::pin(async move {
|
||||
let task = tokio::spawn(async move {
|
||||
release_task.notified().await;
|
||||
task_finished.store(true, Ordering::SeqCst);
|
||||
Ok(RebalanceEntryOutcome::Completed)
|
||||
});
|
||||
entry_tasks.lock().await.push(task);
|
||||
task_registered.notify_one();
|
||||
})
|
||||
}
|
||||
});
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let runner = tokio::spawn(run_rebalance_listing_with_retry(
|
||||
CancellationToken::new(),
|
||||
"bucket-a".to_string(),
|
||||
callback,
|
||||
0,
|
||||
3,
|
||||
entry_tasks,
|
||||
{
|
||||
let attempts = attempts.clone();
|
||||
let task_finished = task_finished.clone();
|
||||
move |cb| {
|
||||
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
|
||||
let task_finished = task_finished.clone();
|
||||
async move {
|
||||
if attempt == 0 {
|
||||
cb(MetaCacheEntry::default()).await;
|
||||
return Err(Error::SlowDown);
|
||||
}
|
||||
assert!(task_finished.load(Ordering::SeqCst), "retry must wait for scheduled entries");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
},
|
||||
));
|
||||
|
||||
task_registered.notified().await;
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1, "retry must not overlap the scheduled entry task");
|
||||
release_task.notify_one();
|
||||
|
||||
runner
|
||||
.await
|
||||
.expect("listing retry task should join")
|
||||
.expect("listing retry should complete after the scheduled entry");
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_rebalance_listing_retry_propagates_scheduled_entry_failure() {
|
||||
let entry_tasks = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let callback: crate::core::pools::ListCallback = Arc::new({
|
||||
let entry_tasks = entry_tasks.clone();
|
||||
move |_| {
|
||||
let entry_tasks = entry_tasks.clone();
|
||||
Box::pin(async move {
|
||||
entry_tasks
|
||||
.lock()
|
||||
.await
|
||||
.push(tokio::spawn(async { Err(Error::other("scheduled entry failed")) }));
|
||||
})
|
||||
}
|
||||
});
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let err = run_rebalance_listing_with_retry(CancellationToken::new(), "bucket-a".to_string(), callback, 0, 3, entry_tasks, {
|
||||
let attempts = attempts.clone();
|
||||
move |cb| {
|
||||
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
|
||||
async move {
|
||||
if attempt == 0 {
|
||||
cb(MetaCacheEntry::default()).await;
|
||||
return Err(Error::SlowDown);
|
||||
}
|
||||
panic!("entry failure must stop listing retries")
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect_err("scheduled entry failure must be returned before retrying the listing");
|
||||
|
||||
assert!(err.to_string().contains("scheduled entry failed"));
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_rebalance_max_attempts_uses_positive_override_or_default() {
|
||||
assert_eq!(parse_rebalance_max_attempts(Some("5")), 5);
|
||||
@@ -2597,7 +2454,6 @@ async fn test_init_and_start_rebalance_rejects_second_start_after_gate() {
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
});
|
||||
|
||||
let err = store
|
||||
|
||||
@@ -10,14 +10,12 @@ use super::worker::{
|
||||
resolve_rebalance_terminal_error, send_rebalance_done_signal,
|
||||
};
|
||||
use super::{
|
||||
EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalSaveOpt, RebalStatus,
|
||||
EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, RebalSaveOpt, RebalStatus,
|
||||
RebalanceBucketOutcome,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::store::ECStore;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
@@ -25,20 +23,6 @@ use tokio::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
pub(super) fn should_fail_repeated_rebalance_bucket_defer(
|
||||
deferred_buckets: &mut HashSet<String>,
|
||||
bucket: &str,
|
||||
source_cleanup_deferred: bool,
|
||||
) -> bool {
|
||||
!source_cleanup_deferred && !deferred_buckets.insert(bucket.to_string())
|
||||
}
|
||||
|
||||
pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<String, usize>, bucket: &str) -> usize {
|
||||
let attempts = deferred_attempts.entry(bucket.to_string()).or_default();
|
||||
*attempts = attempts.saturating_add(1);
|
||||
*attempts
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
|
||||
@@ -314,7 +298,6 @@ impl ECStore {
|
||||
);
|
||||
let mut final_result: Result<()> = Ok(());
|
||||
let mut deferred_buckets = HashSet::new();
|
||||
let mut source_cleanup_deferred_attempts = HashMap::new();
|
||||
|
||||
loop {
|
||||
if rx.is_cancelled() {
|
||||
@@ -392,8 +375,7 @@ impl ECStore {
|
||||
};
|
||||
|
||||
if let RebalanceBucketOutcome::Deferred { last_error } = outcome {
|
||||
let source_cleanup_deferred = last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX);
|
||||
if should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, &bucket, source_cleanup_deferred) {
|
||||
if !deferred_buckets.insert(bucket.clone()) {
|
||||
let err = Error::other(format!(
|
||||
"rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
|
||||
));
|
||||
@@ -414,11 +396,6 @@ impl ECStore {
|
||||
break;
|
||||
}
|
||||
|
||||
let source_cleanup_attempt = if source_cleanup_deferred {
|
||||
source_cleanup_defer_attempt(&mut source_cleanup_deferred_attempts, &bucket)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -429,10 +406,7 @@ impl ECStore {
|
||||
error = %last_error,
|
||||
"Deferred rebalance bucket after transient object failures"
|
||||
);
|
||||
if let Err(err) = self
|
||||
.defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone())
|
||||
.await
|
||||
{
|
||||
if let Err(err) = self.defer_rebalance_bucket(pool_index, bucket.clone(), last_error).await {
|
||||
error!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -449,38 +423,6 @@ impl ECStore {
|
||||
));
|
||||
break;
|
||||
}
|
||||
if source_cleanup_deferred {
|
||||
if source_cleanup_attempt >= super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS {
|
||||
let err = Error::other(format!(
|
||||
"rebalance bucket {bucket} source cleanup remained unstable after {} deferrals: {last_error}",
|
||||
super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
));
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
pool_index,
|
||||
bucket = %bucket,
|
||||
state = "source_cleanup_defer_limit",
|
||||
error = ?err,
|
||||
"Rebalance bucket failed after repeated source cleanup conflicts"
|
||||
);
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
));
|
||||
break;
|
||||
}
|
||||
if let Err(err) =
|
||||
super::worker::wait_rebalance_listing_retry(&rx, REBALANCE_LISTING_RETRY_BASE_DELAY).await
|
||||
{
|
||||
final_result = Err(resolve_rebalance_terminal_error(
|
||||
err.clone(),
|
||||
send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
|
||||
));
|
||||
break;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -493,7 +435,6 @@ impl ECStore {
|
||||
state = "completed",
|
||||
"Completed rebalance bucket"
|
||||
);
|
||||
source_cleanup_deferred_attempts.remove(&bucket);
|
||||
if let Err(err) = self.bucket_rebalance_done(pool_index, bucket).await {
|
||||
error!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
|
||||
@@ -2,12 +2,10 @@ use super::migration::MigrationVersionResult;
|
||||
use super::{
|
||||
DEFAULT_REBALANCE_MAX_ATTEMPTS, EVENT_REBALANCE_LISTING, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_MAX_ATTEMPTS_ENV, REBALANCE_MIGRATION_LOCK_RETRY_CAP,
|
||||
REBALANCE_MIGRATION_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalanceBucketConfigs,
|
||||
RebalanceBucketOutcome, RebalanceEntryOutcome, Result,
|
||||
REBALANCE_MIGRATION_RETRY_BASE_DELAY, RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome, Result,
|
||||
};
|
||||
use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
|
||||
use crate::core::pools::ListCallback;
|
||||
use crate::data_movement::SourceCleanupError;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::error::{
|
||||
Error, is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found, is_network_or_host_down,
|
||||
@@ -38,12 +36,6 @@ pub(super) fn resolve_rebalance_worker_result<T>(
|
||||
|
||||
pub(super) type RebalanceEntryTask = tokio::task::JoinHandle<Result<RebalanceEntryOutcome>>;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(super) enum RebalanceEntryCleanupResult {
|
||||
Completed { warning: Option<String> },
|
||||
Deferred { last_error: String },
|
||||
}
|
||||
|
||||
pub(super) async fn wait_rebalance_entry_tasks(
|
||||
set_idx: usize,
|
||||
tasks: Arc<tokio::sync::Mutex<Vec<RebalanceEntryTask>>>,
|
||||
@@ -153,23 +145,14 @@ where
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_entry_cleanup_delete_result(
|
||||
result: std::result::Result<crate::object_api::ObjectInfo, SourceCleanupError>,
|
||||
result: Result<crate::object_api::ObjectInfo>,
|
||||
bucket: &str,
|
||||
object_name: &str,
|
||||
) -> RebalanceEntryCleanupResult {
|
||||
) -> Result<Option<String>> {
|
||||
match result {
|
||||
Ok(_) => RebalanceEntryCleanupResult::Completed { warning: None },
|
||||
Err(SourceCleanupError::Storage(err)) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
|
||||
RebalanceEntryCleanupResult::Completed { warning: None }
|
||||
}
|
||||
Err(SourceCleanupError::SourceChanged) => RebalanceEntryCleanupResult::Deferred {
|
||||
last_error: format!(
|
||||
"{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} source changed during cleanup preflight for {bucket}/{object_name}"
|
||||
),
|
||||
},
|
||||
Err(SourceCleanupError::Storage(err)) => RebalanceEntryCleanupResult::Completed {
|
||||
warning: Some(format!("rebalance cleanup delete failed for {bucket}/{object_name}: {err}")),
|
||||
},
|
||||
Ok(_) => Ok(None),
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
|
||||
Err(err) => Ok(Some(format!("rebalance cleanup delete failed for {bucket}/{object_name}: {err}"))),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -416,24 +399,19 @@ pub(super) async fn load_rebalance_bucket_configs(api: &ECStore, bucket: &str) -
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn run_rebalance_listing_with_retry<List, ListFuture>(
|
||||
pub(super) async fn run_rebalance_listing_with_retry(
|
||||
set: Arc<SetDisks>,
|
||||
rx: CancellationToken,
|
||||
bucket: String,
|
||||
cb: ListCallback,
|
||||
set_idx: usize,
|
||||
max_attempts: usize,
|
||||
entry_tasks: Arc<tokio::sync::Mutex<Vec<RebalanceEntryTask>>>,
|
||||
mut list: List,
|
||||
) -> Result<()>
|
||||
where
|
||||
List: FnMut(ListCallback) -> ListFuture,
|
||||
ListFuture: std::future::Future<Output = Result<()>>,
|
||||
{
|
||||
) -> Result<()> {
|
||||
let max_attempts = max_attempts.max(1);
|
||||
let mut last_error = None;
|
||||
|
||||
for attempt in 0..max_attempts {
|
||||
match list(cb.clone()).await {
|
||||
match set.list_objects_to_rebalance(rx.clone(), bucket.clone(), cb.clone()).await {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(err) if should_retry_rebalance_listing(&err, attempt, max_attempts) => {
|
||||
let next_attempt = attempt + 2;
|
||||
@@ -448,8 +426,6 @@ where
|
||||
delay
|
||||
);
|
||||
last_error = Some(err);
|
||||
// The full retry re-evaluates deferred entries; only task failures block the next attempt.
|
||||
let _ = wait_rebalance_entry_tasks(set_idx, entry_tasks.clone()).await?;
|
||||
wait_rebalance_listing_retry(&rx, delay).await?;
|
||||
info!(
|
||||
"rebalance listing retrying bucket {} set {} attempt {}/{}",
|
||||
|
||||
@@ -49,7 +49,7 @@ use crate::diagnostics::get::{
|
||||
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
|
||||
use crate::disk::{
|
||||
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
|
||||
PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
|
||||
PartTransactionAction, part_transaction_path,
|
||||
};
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::ShardReader;
|
||||
@@ -2950,68 +2950,6 @@ impl SetDisks {
|
||||
return Err(ret_err);
|
||||
}
|
||||
|
||||
// The write is authoritatively committed, so the per-disk rollback
|
||||
// backup (`object/<rollback_dir>/xl.meta.bkp`) is dead weight now.
|
||||
// When the rollback dir doubles as the real dereferenced data dir it
|
||||
// is reclaimed wholesale by `commit_rename_data_dir`; a rollback dir
|
||||
// reported separately (an overwrite of an inline version, whose dir is
|
||||
// synthetic) is excluded from that recursive reclamation for safety
|
||||
// (#5703) and must be reclaimed here instead — otherwise every inline
|
||||
// overwrite strands a backup file that keeps the object dir non-empty
|
||||
// and makes a later DeleteBucket fail with BucketNotEmpty forever.
|
||||
// Delete exactly the backup file, never the directory tree: the
|
||||
// synthetic UUID is a fixed, publicly-known constant for unversioned
|
||||
// objects, so `object/<rollback_dir>` can simultaneously be a
|
||||
// legitimate child key's directory — recursively deleting it would
|
||||
// reopen the authorization bypass #5703 closed. The non-recursive
|
||||
// delete removes the directory only when the backup was its sole
|
||||
// content. Best-effort space reclamation — like
|
||||
// `commit_rename_data_dir`, this must never negate the already-durable
|
||||
// ACK.
|
||||
let mut backup_reclaims = Vec::new();
|
||||
for (idx, disk) in disks.iter().enumerate() {
|
||||
if errs[idx].is_some() {
|
||||
continue;
|
||||
}
|
||||
let Some(rollback_dir) = data_dirs[idx] else {
|
||||
continue;
|
||||
};
|
||||
if cleanup_data_dirs[idx] == Some(rollback_dir) {
|
||||
continue;
|
||||
}
|
||||
let Some(disk) = disk.clone() else {
|
||||
continue;
|
||||
};
|
||||
let dst_bucket = dst_bucket.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
backup_reclaims.push(tokio::spawn(async move {
|
||||
let backup_path = format!("{dst_object}/{rollback_dir}/{STORAGE_FORMAT_FILE_BACKUP}");
|
||||
disk.delete(&dst_bucket, &backup_path, DeleteOptions::default()).await
|
||||
}));
|
||||
}
|
||||
for result in join_all(backup_reclaims).await {
|
||||
match result {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(DiskError::FileNotFound | DiskError::VolumeNotFound)) => {}
|
||||
Ok(Err(err)) => {
|
||||
warn!(
|
||||
dst_bucket = %dst_bucket,
|
||||
dst_object = %dst_object,
|
||||
error = %err,
|
||||
"rollback backup reclamation failed after committed rename"
|
||||
);
|
||||
}
|
||||
Err(join_err) => {
|
||||
warn!(
|
||||
dst_bucket = %dst_bucket,
|
||||
dst_object = %dst_object,
|
||||
error = %join_err,
|
||||
"rollback backup reclamation task failed after committed rename"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let data_dir = Self::reduce_common_data_dir(&cleanup_data_dirs, write_quorum);
|
||||
let convergence = Self::classify_rename_convergence(&disk_versions, &errs);
|
||||
let old_current_size = Self::reduce_common_old_current_size(&old_current_sizes, write_quorum);
|
||||
@@ -5265,84 +5203,6 @@ mod tests {
|
||||
assert!(stored.is_canonical_delete_marker());
|
||||
}
|
||||
|
||||
/// Overwriting an inline version with a non-inline one stages the old
|
||||
/// xl.meta as `<object>/<synthetic-rollback-dir>/xl.meta.bkp` for the
|
||||
/// quorum-failure undo. After a successful quorum commit that dir must be
|
||||
/// reclaimed — leftover residue keeps DeleteBucket failing with
|
||||
/// BucketNotEmpty long after the object itself is deleted.
|
||||
#[tokio::test]
|
||||
async fn rename_data_reclaims_synthetic_inline_rollback_dir_after_commit() {
|
||||
let bucket = "rename-inline-rollback-bucket";
|
||||
let object = "object";
|
||||
let (dirs, mut online_disks) = call_counter_local_disks(bucket, 1).await;
|
||||
let online_disk = online_disks.pop().expect("one test disk slot should be present");
|
||||
let disk = online_disk.as_ref().expect("test disk should be online");
|
||||
match disk.make_volume(RUSTFS_META_TMP_BUCKET).await {
|
||||
Ok(()) | Err(DiskError::VolumeExists) => {}
|
||||
Err(err) => panic!("temporary metadata volume should be available: {err:?}"),
|
||||
}
|
||||
|
||||
let disk_root = dirs[0].path();
|
||||
|
||||
// Commit an inline version (data carried in xl.meta, no data dir).
|
||||
let mut inline_fi = metadata_test_fileinfo(object);
|
||||
inline_fi.data = Some(Bytes::from_static(b"inline-body"));
|
||||
inline_fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
std::fs::create_dir_all(disk_root.join(RUSTFS_META_TMP_BUCKET).join("tmp-inline"))
|
||||
.expect("inline staging dir should be created");
|
||||
SetDisks::rename_data(
|
||||
std::slice::from_ref(&online_disk),
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"tmp-inline",
|
||||
std::slice::from_ref(&inline_fi),
|
||||
bucket,
|
||||
object,
|
||||
1,
|
||||
)
|
||||
.await
|
||||
.expect("inline version should commit");
|
||||
|
||||
// Overwrite the same (nil) version with a non-inline one.
|
||||
let new_data_dir = Uuid::new_v4();
|
||||
let mut streaming_fi = metadata_test_fileinfo(object);
|
||||
streaming_fi.data_dir = Some(new_data_dir);
|
||||
streaming_fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
let staged_data_dir = disk_root
|
||||
.join(RUSTFS_META_TMP_BUCKET)
|
||||
.join("tmp-streaming")
|
||||
.join(new_data_dir.to_string());
|
||||
std::fs::create_dir_all(&staged_data_dir).expect("streaming staging dir should be created");
|
||||
std::fs::write(staged_data_dir.join("part.1"), b"streamed-body").expect("staged part should be written");
|
||||
SetDisks::rename_data(
|
||||
std::slice::from_ref(&online_disk),
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"tmp-streaming",
|
||||
std::slice::from_ref(&streaming_fi),
|
||||
bucket,
|
||||
object,
|
||||
1,
|
||||
)
|
||||
.await
|
||||
.expect("non-inline overwrite should commit");
|
||||
|
||||
let mut leftovers: Vec<String> = std::fs::read_dir(disk_root.join(bucket).join(object))
|
||||
.expect("committed object dir should be readable")
|
||||
.map(|entry| {
|
||||
entry
|
||||
.expect("object dir entry should be readable")
|
||||
.file_name()
|
||||
.to_string_lossy()
|
||||
.into_owned()
|
||||
})
|
||||
.collect();
|
||||
leftovers.sort();
|
||||
assert_eq!(
|
||||
leftovers,
|
||||
vec![new_data_dir.to_string(), STORAGE_FORMAT_FILE.to_string()],
|
||||
"only the committed data dir and xl.meta may remain — synthetic rollback residue breaks DeleteBucket"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_delete_marker_quorum_failure_restores_existing_metadata() {
|
||||
let bucket = "rename-marker-quorum-bucket";
|
||||
|
||||
@@ -250,26 +250,14 @@ impl SetDisks {
|
||||
continue;
|
||||
}
|
||||
|
||||
// A parity count outside [0, total_shards] cannot describe a real
|
||||
// layout on this set: it comes from corrupt or foreign metadata
|
||||
// (e.g. stray leftovers, rustfs#5801). Treat the entry as invalid
|
||||
// instead of clamping to i32::MAX, which would poison
|
||||
// `common_parity`'s occurrence counting.
|
||||
let erasure_parity = i32::try_from(metadata.erasure.parity_blocks).unwrap_or(-1);
|
||||
let erasure_parity = if (0..=total_shards_i32).contains(&erasure_parity) {
|
||||
erasure_parity
|
||||
} else {
|
||||
-1
|
||||
};
|
||||
if metadata.is_canonical_delete_marker() || metadata.size == 0 {
|
||||
parities[index] = half;
|
||||
} else if erasure_parity < 0 {
|
||||
parities[index] = -1;
|
||||
} else if metadata.transition_status == TRANSITION_COMPLETE {
|
||||
let majority_metadata_parity = total_shards_i32 - (half + 1);
|
||||
let erasure_parity = i32::try_from(metadata.erasure.parity_blocks).unwrap_or(i32::MAX);
|
||||
parities[index] = majority_metadata_parity.max(erasure_parity);
|
||||
} else {
|
||||
parities[index] = erasure_parity;
|
||||
parities[index] = i32::try_from(metadata.erasure.parity_blocks).unwrap_or(i32::MAX);
|
||||
}
|
||||
}
|
||||
parities
|
||||
@@ -306,19 +294,6 @@ impl SetDisks {
|
||||
let parity_blocks = Self::common_parity(&parities, default_parity_count as i32);
|
||||
|
||||
if parity_blocks < 0 {
|
||||
// No parity value reached read quorum. Distinguish two cases:
|
||||
// enough disks answered with valid-looking metadata that simply
|
||||
// cannot be reconciled (corrupt/foreign entries — retrying cannot
|
||||
// help, and heal should see Corrupt, rustfs#5801) versus too few
|
||||
// healthy answers (a genuine quorum condition where retry may
|
||||
// succeed once disks recover).
|
||||
let healthy_replies = errs.iter().filter(|err| err.is_none()).count();
|
||||
if healthy_replies >= expected_rquorum {
|
||||
error!(
|
||||
"object_quorum_from_meta: irreconcilable parity across {healthy_replies} healthy replies (corrupt metadata), errs={errs:?}"
|
||||
);
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
error!("object_quorum_from_meta: parity_blocks < 0, errs={:?}", errs);
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
@@ -1161,9 +1136,7 @@ mod tests {
|
||||
let invalid = vec![FileInfo::default(); 4];
|
||||
let err = SetDisks::object_quorum_from_meta(&invalid, &vec![None; 4], 2)
|
||||
.expect_err("invalid metadata without a common parity must fail closed");
|
||||
// A full set of healthy replies whose metadata cannot be reconciled is
|
||||
// corrupt (heal-actionable), not a retryable quorum outage (rustfs#5801).
|
||||
assert_eq!(err, DiskError::FileCorrupt);
|
||||
assert_eq!(err, DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1495,55 +1468,4 @@ mod tests {
|
||||
"compatible prefixes carrying the same mapping must share one identity"
|
||||
);
|
||||
}
|
||||
|
||||
/// rustfs#5801: parity counts outside [0, total_shards] come from corrupt
|
||||
/// or foreign metadata and must be treated as invalid entries instead of
|
||||
/// clamped values that poison `common_parity`'s occurrence counting.
|
||||
#[test]
|
||||
fn out_of_range_parity_is_treated_as_invalid_entry() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
|
||||
let mut metas: Vec<FileInfo> = (0..4).map(|i| metadata_quorum_test_fileinfo(mod_time, i)).collect();
|
||||
for fi in &mut metas {
|
||||
fi.erasure.parity_blocks = usize::MAX;
|
||||
}
|
||||
let errs: Vec<Option<DiskError>> = vec![None; 4];
|
||||
|
||||
let parities = SetDisks::list_object_parities(&metas, &errs);
|
||||
assert_eq!(parities, vec![-1; 4], "garbage parity must not survive as a candidate");
|
||||
}
|
||||
|
||||
/// rustfs#5801: when a read quorum of healthy disks answers but their
|
||||
/// parity values are irreconcilable, the object metadata is corrupt —
|
||||
/// return `FileCorrupt` (heal-actionable, non-retryable) instead of the
|
||||
/// retryable-looking `ErasureReadQuorum`.
|
||||
#[test]
|
||||
fn irreconcilable_parity_with_healthy_quorum_is_file_corrupt() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
|
||||
let mut metas: Vec<FileInfo> = (0..4).map(|i| metadata_quorum_test_fileinfo(mod_time, i)).collect();
|
||||
for fi in &mut metas {
|
||||
fi.erasure.parity_blocks = usize::MAX;
|
||||
}
|
||||
let errs: Vec<Option<DiskError>> = vec![None; 4];
|
||||
|
||||
let err = SetDisks::object_quorum_from_meta(&metas, &errs, 2).expect_err("garbage parity cannot form a quorum");
|
||||
assert_eq!(err, DiskError::FileCorrupt);
|
||||
}
|
||||
|
||||
/// Too few healthy replies remains a genuine quorum condition where a
|
||||
/// retry may succeed once disks recover.
|
||||
#[test]
|
||||
fn insufficient_healthy_replies_stays_erasure_read_quorum() {
|
||||
let mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
|
||||
let mut metas: Vec<FileInfo> = (0..4).map(|i| metadata_quorum_test_fileinfo(mod_time, i)).collect();
|
||||
metas[0].erasure.parity_blocks = usize::MAX;
|
||||
let errs: Vec<Option<DiskError>> = vec![
|
||||
None,
|
||||
Some(DiskError::DiskNotFound),
|
||||
Some(DiskError::DiskNotFound),
|
||||
Some(DiskError::DiskNotFound),
|
||||
];
|
||||
|
||||
let err = SetDisks::object_quorum_from_meta(&metas, &errs, 2).expect_err("one healthy reply is below quorum");
|
||||
assert_eq!(err, DiskError::ErasureReadQuorum);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3964,15 +3964,9 @@ async fn disks_with_all_parts(
|
||||
};
|
||||
|
||||
if corrupted {
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object = %object_name,
|
||||
disk_index = index,
|
||||
state = "metadata_corrupt",
|
||||
"Set disk object metadata is corrupt"
|
||||
info!(
|
||||
"disks_with_all_partsv2: metadata is corrupted, object_name={}, index: {index}",
|
||||
object_name
|
||||
);
|
||||
meta_errs[index] = Some(DiskError::FileCorrupt);
|
||||
parts_metadata[index] = FileInfo::default();
|
||||
@@ -3983,15 +3977,9 @@ async fn disks_with_all_parts(
|
||||
|
||||
if erasure_distribution_reliable {
|
||||
if !file_info_is_valid_for_metadata(meta) {
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object = %object_name,
|
||||
disk_index = index,
|
||||
state = "metadata_invalid",
|
||||
"Set disk object metadata is invalid"
|
||||
info!(
|
||||
"disks_with_all_partsv2: metadata is not valid, object_name={}, index: {index}",
|
||||
object_name
|
||||
);
|
||||
parts_metadata[index] = FileInfo::default();
|
||||
meta_errs[index] = Some(DiskError::FileCorrupt);
|
||||
@@ -4003,15 +3991,9 @@ async fn disks_with_all_parts(
|
||||
// Erasure distribution is not the same as onlineDisks
|
||||
// attempt a fix if possible, assuming other entries
|
||||
// might have the right erasure distribution.
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object = %object_name,
|
||||
disk_index = index,
|
||||
state = "erasure_distribution_mismatch",
|
||||
"Set disk erasure distribution mismatched online disks"
|
||||
info!(
|
||||
"disks_with_all_partsv2: erasure distribution is not the same as onlineDisks, object_name={}, index: {index}",
|
||||
object_name
|
||||
);
|
||||
parts_metadata[index] = FileInfo::default();
|
||||
meta_errs[index] = Some(DiskError::FileCorrupt);
|
||||
@@ -4084,7 +4066,6 @@ async fn disks_with_all_parts(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object = %object_name,
|
||||
disk_index = index,
|
||||
state = "verify_failed",
|
||||
@@ -4104,7 +4085,6 @@ async fn disks_with_all_parts(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object = %object_name,
|
||||
disk_index = index,
|
||||
state = "check_parts_failed",
|
||||
@@ -4173,13 +4153,9 @@ pub fn should_heal_object_on_disk(
|
||||
}
|
||||
|
||||
if !meta.equals(latest_meta) {
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
object = %meta.name,
|
||||
state = "metadata_outdated",
|
||||
"Set disk object metadata is outdated"
|
||||
warn!(
|
||||
"should_heal_object_on_disk: metadata is outdated, object_name={}, meta: {:?}, latest_meta: {:?}",
|
||||
meta.name, meta, latest_meta
|
||||
);
|
||||
return (true, true, Some(DiskError::OutdatedXLMeta));
|
||||
}
|
||||
@@ -6309,136 +6285,6 @@ mod tests {
|
||||
assert_eq!(read_back.size, 9, "HEAD must observe the new version, not stale metadata");
|
||||
}
|
||||
|
||||
// Regression for the inline-overwrite rollback backup leak: #5703 stopped
|
||||
// reporting the synthetic rollback dir for recursive post-commit cleanup,
|
||||
// which stranded `object/<synthetic>/xl.meta.bkp` after every inline
|
||||
// overwrite. The object dir then never emptied, so the s3-tests teardown
|
||||
// sequence (delete object, delete bucket) failed with BucketNotEmpty
|
||||
// forever. After a committed overwrite the backup must be reclaimed and a
|
||||
// subsequent delete must leave nothing behind.
|
||||
#[tokio::test]
|
||||
async fn inline_overwrite_reclaims_synthetic_rollback_backup() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let bucket = "bucket-inline-rollback-leak";
|
||||
let object = "obj";
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
for body in [b"hello".to_vec(), b"goodbye".to_vec()] {
|
||||
let mut reader = PutObjReader::from_vec(body);
|
||||
set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..ObjectOptions::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("inline write should succeed");
|
||||
}
|
||||
|
||||
// The committed overwrite must leave only xl.meta in the object dir on
|
||||
// every disk; a stranded rollback dir keeps the bucket undeletable.
|
||||
for endpoint in &set_disks.set_endpoints {
|
||||
let object_dir = std::path::PathBuf::from(endpoint.get_file_path()).join(bucket).join(object);
|
||||
let mut entries: Vec<String> = std::fs::read_dir(&object_dir)
|
||||
.expect("object dir should exist")
|
||||
.map(|entry| entry.expect("entry should read").file_name().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
entries.sort();
|
||||
assert_eq!(
|
||||
entries,
|
||||
vec![STORAGE_FORMAT_FILE.to_string()],
|
||||
"only xl.meta may remain after an inline overwrite in {object_dir:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// With only xl.meta left, the s3-tests teardown (delete object, delete
|
||||
// bucket) empties the dir; the delete paths themselves are covered by
|
||||
// their own tests. This harness has no bucket metadata sys, so the
|
||||
// full delete_object flow cannot run here.
|
||||
}
|
||||
|
||||
// #5703's security property must survive the backup reclamation: the
|
||||
// synthetic rollback dir of key K maps to the directory `K/<uuid>`, which
|
||||
// can simultaneously be a legitimate child key. Reclaiming the backup must
|
||||
// remove exactly the backup file — never the child key's metadata.
|
||||
#[tokio::test]
|
||||
async fn inline_overwrite_backup_reclaim_spares_child_key_dir() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let bucket = "bucket-inline-rollback-child";
|
||||
let object = "obj";
|
||||
let synthetic = crate::disk::local::inline_metadata_rollback_dir(Uuid::nil(), &FileMeta::new());
|
||||
let child_object = format!("{object}/{synthetic}");
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
let mut reader = PutObjReader::from_vec(b"child".to_vec());
|
||||
set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
&child_object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..ObjectOptions::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("child write should succeed");
|
||||
|
||||
// Create then overwrite the parent key: the overwrite writes its
|
||||
// rollback backup into the child's directory and must afterwards
|
||||
// reclaim only that file.
|
||||
for body in [b"first".to_vec(), b"second".to_vec()] {
|
||||
let mut reader = PutObjReader::from_vec(body);
|
||||
set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..ObjectOptions::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("parent write should succeed");
|
||||
}
|
||||
|
||||
let child_info = set_disks
|
||||
.get_object_info(bucket, &child_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("child key must survive the parent's rollback backup reclamation");
|
||||
assert_eq!(child_info.size, 5, "child key content must be untouched");
|
||||
|
||||
for endpoint in &set_disks.set_endpoints {
|
||||
let child_dir = std::path::PathBuf::from(endpoint.get_file_path())
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(synthetic.to_string());
|
||||
let mut entries: Vec<String> = std::fs::read_dir(&child_dir)
|
||||
.expect("child object dir should exist")
|
||||
.map(|entry| entry.expect("entry should read").file_name().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
entries.sort();
|
||||
assert_eq!(
|
||||
entries,
|
||||
vec![STORAGE_FORMAT_FILE.to_string()],
|
||||
"the child dir must keep its xl.meta and lose only the stray backup in {child_dir:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn test_acquire_dist_delete_object_locks_batch_succeeds_with_two_healthy_lockers() {
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
use super::super::*;
|
||||
use crate::io_support::bitrot::object_mmap_read_enabled;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use tracing::trace;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_HEAL: &str = "heal";
|
||||
@@ -72,9 +71,7 @@ fn should_fail_heal_rename(bucket: &str, object: &str, disk_index: usize) -> boo
|
||||
.expect("heal rename failure registry should not poison");
|
||||
if let Some(position) = failures
|
||||
.iter()
|
||||
.position(|(registered_bucket, registered_object, registered_index)| {
|
||||
registered_bucket == bucket && registered_object == object && *registered_index == disk_index
|
||||
})
|
||||
.position(|entry| entry == &(bucket.to_string(), object.to_string(), disk_index))
|
||||
{
|
||||
failures.swap_remove(position);
|
||||
true
|
||||
@@ -88,67 +85,6 @@ fn should_fail_heal_rename(_bucket: &str, _object: &str, _disk_index: usize) ->
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static HEAL_WRITER_FAILURES: std::sync::Mutex<Vec<(String, String, usize, DiskError)>> = std::sync::Mutex::new(Vec::new());
|
||||
|
||||
#[cfg(test)]
|
||||
struct HealWriterFailureScope {
|
||||
bucket: String,
|
||||
object: String,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl HealWriterFailureScope {
|
||||
fn install(bucket: &str, object: &str, disk_indexes: &[usize], error: DiskError) -> Self {
|
||||
let mut failures = HEAL_WRITER_FAILURES
|
||||
.lock()
|
||||
.expect("heal writer failure registry should not poison");
|
||||
assert!(
|
||||
!failures.iter().any(|(registered_bucket, registered_object, _, _)| {
|
||||
registered_bucket == bucket && registered_object == object
|
||||
}),
|
||||
"heal writer failures must be installed once per object"
|
||||
);
|
||||
failures.extend(
|
||||
disk_indexes
|
||||
.iter()
|
||||
.map(|index| (bucket.to_string(), object.to_string(), *index, error.clone())),
|
||||
);
|
||||
Self {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for HealWriterFailureScope {
|
||||
fn drop(&mut self) {
|
||||
HEAL_WRITER_FAILURES
|
||||
.lock()
|
||||
.expect("heal writer failure registry should not poison")
|
||||
.retain(|(bucket, object, _, _)| bucket != &self.bucket || object != &self.object);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn injected_heal_writer_error(bucket: &str, object: &str, disk_index: usize) -> Option<DiskError> {
|
||||
let mut failures = HEAL_WRITER_FAILURES
|
||||
.lock()
|
||||
.expect("heal writer failure registry should not poison");
|
||||
failures
|
||||
.iter()
|
||||
.position(|(registered_bucket, registered_object, registered_index, _)| {
|
||||
registered_bucket == bucket && registered_object == object && *registered_index == disk_index
|
||||
})
|
||||
.map(|position| failures.swap_remove(position).3)
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn injected_heal_writer_error(_bucket: &str, _object: &str, _disk_index: usize) -> Option<DiskError> {
|
||||
None
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
struct PartFailureSummary {
|
||||
part_number: usize,
|
||||
@@ -297,41 +233,8 @@ fn first_unhealthy_part_summary(
|
||||
.map(|(_, summary)| summary)
|
||||
}
|
||||
|
||||
fn heal_writer_error_summary(error: &DiskError) -> String {
|
||||
match error {
|
||||
DiskError::Io(io_error) => format!("io::{:?}", io_error.kind()),
|
||||
_ => error.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn warn_heal_writer_failures(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
writer_failure_count: usize,
|
||||
result: &'static str,
|
||||
first_failure: &(usize, usize, String),
|
||||
) {
|
||||
let (first_part_number, first_disk_index, first_error) = first_failure;
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
writer_failure_count,
|
||||
first_part_number,
|
||||
first_disk_index,
|
||||
error = %first_error,
|
||||
result,
|
||||
state = "writer_unavailable",
|
||||
"Set disk object heal writer failures"
|
||||
);
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
#[tracing::instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
|
||||
#[tracing::instrument(skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
|
||||
pub(in crate::set_disk) async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -351,16 +254,7 @@ impl SetDisks {
|
||||
opts: &HealOpts,
|
||||
allow_explicit_version_regen: bool,
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
trace!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
scan_mode = %opts.scan_mode.as_str(),
|
||||
dry_run = opts.dry_run,
|
||||
remove = opts.remove,
|
||||
state = "started",
|
||||
"Set disk object heal started"
|
||||
);
|
||||
info!(?opts, "Starting heal_object");
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
@@ -396,26 +290,16 @@ impl SetDisks {
|
||||
let (mut parts_metadata, errs) =
|
||||
Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true, false).await?;
|
||||
|
||||
trace!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
info!(
|
||||
parts_count = parts_metadata.len(),
|
||||
error_count = errs.iter().flatten().count(),
|
||||
state = "metadata_read",
|
||||
"Set disk object metadata read"
|
||||
bucket = bucket,
|
||||
object = object,
|
||||
version_id = version_id,
|
||||
?errs,
|
||||
"File info read complete"
|
||||
);
|
||||
if DiskError::is_all_not_found(&errs) {
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
state = "missing_object_skipped",
|
||||
"Set disk heal skipped missing object"
|
||||
);
|
||||
debug!(bucket, object, version_id, "heal_object skipped missing object");
|
||||
let err = if !version_id.is_empty() {
|
||||
DiskError::FileVersionNotFound
|
||||
} else {
|
||||
@@ -429,14 +313,7 @@ impl SetDisks {
|
||||
));
|
||||
}
|
||||
|
||||
trace!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
parts_count = parts_metadata.len(),
|
||||
state = "quorum_check",
|
||||
"Set disk object quorum check started"
|
||||
);
|
||||
info!(parts_count = parts_metadata.len(), "heal_object Initiating quorum check");
|
||||
match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count) {
|
||||
Ok((read_quorum, _)) => {
|
||||
result.parity_blocks = result.disk_count - read_quorum as usize;
|
||||
@@ -448,35 +325,14 @@ impl SetDisks {
|
||||
(Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum as usize), disk_len)
|
||||
};
|
||||
|
||||
trace!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
metadata_count = parts_metadata.len(),
|
||||
error_count = errs.iter().flatten().count(),
|
||||
read_quorum,
|
||||
disk_count = disk_len,
|
||||
online_disk_count = online_disks.iter().flatten().count(),
|
||||
state = "disk_metadata_resolved",
|
||||
"Set disk object metadata resolved"
|
||||
);
|
||||
info!(?parts_metadata, ?errs, ?read_quorum, ?disk_len, "heal_object List disks metadata");
|
||||
|
||||
info!(?online_disks, ?quorum_mod_time, ?quorum_etag, "heal_object List online disks");
|
||||
|
||||
let filter_by_etag = quorum_etag.is_some();
|
||||
match Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag.clone(), read_quorum as usize) {
|
||||
Ok(latest_meta) => {
|
||||
trace!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
deleted = latest_meta.deleted,
|
||||
remote = latest_meta.is_remote(),
|
||||
inline = latest_meta.inline_data(),
|
||||
part_count = latest_meta.parts.len(),
|
||||
data_shards = latest_meta.erasure.data_blocks,
|
||||
parity_shards = latest_meta.erasure.parity_blocks,
|
||||
state = "canonical_metadata_selected",
|
||||
"Set disk canonical object metadata selected"
|
||||
);
|
||||
info!("heal_object latest_meta: {:?}", latest_meta);
|
||||
|
||||
let (data_errs_by_disk, data_errs_by_part) = disks_with_all_parts(
|
||||
&mut online_disks,
|
||||
@@ -490,14 +346,10 @@ impl SetDisks {
|
||||
)
|
||||
.await?;
|
||||
|
||||
trace!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
available_disk_count = online_disks.iter().flatten().count(),
|
||||
disk_count = online_disks.len(),
|
||||
state = "parts_checked",
|
||||
"Set disk object parts checked"
|
||||
info!(
|
||||
"disks_with_all_parts heal_object results: available_disks count={}, total_disks={}",
|
||||
online_disks.iter().filter(|d| d.is_some()).count(),
|
||||
online_disks.len()
|
||||
);
|
||||
|
||||
let erasure = if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
@@ -536,18 +388,7 @@ impl SetDisks {
|
||||
if is_meta {
|
||||
meta_to_heal_count += 1;
|
||||
}
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
disk_index = index,
|
||||
endpoint = %self.set_endpoints[index],
|
||||
state = "disk_marked_for_healing",
|
||||
"Set disk marked for healing"
|
||||
);
|
||||
debug!("heal_object Disk {} marked for healing (endpoint={})", index, self.set_endpoints[index]);
|
||||
}
|
||||
|
||||
let drive_state = match reason {
|
||||
@@ -631,8 +472,6 @@ impl SetDisks {
|
||||
latest_meta.erasure.data_blocks
|
||||
);
|
||||
error!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
@@ -651,8 +490,6 @@ impl SetDisks {
|
||||
latest_meta.erasure.parity_blocks
|
||||
);
|
||||
error!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
@@ -708,8 +545,6 @@ impl SetDisks {
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
@@ -723,23 +558,11 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
if !latest_meta.deleted && latest_meta.erasure.distribution.len() != online_disks.len() {
|
||||
let distribution_len = latest_meta.erasure.distribution.len();
|
||||
let disk_slot_count = online_disks.len();
|
||||
let err_str = format!(
|
||||
"unexpected file distribution length {distribution_len} for {disk_slot_count} disk slots; backend disks may have been manually modified; refusing to heal {bucket}/{object}({version_id})"
|
||||
);
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
distribution_len,
|
||||
disk_slot_count,
|
||||
state = "invalid_distribution",
|
||||
"Set disk object heal refused due to invalid erasure distribution"
|
||||
"unexpected file distribution ({:?}) from available disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
|
||||
latest_meta.erasure.distribution, online_disks, bucket, object, version_id
|
||||
);
|
||||
warn!(err_str);
|
||||
let err = DiskError::other(err_str);
|
||||
return Ok((
|
||||
self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await,
|
||||
@@ -749,23 +572,11 @@ impl SetDisks {
|
||||
|
||||
let latest_disks = Self::shuffle_disks(&online_disks, &latest_meta.erasure.distribution);
|
||||
if !latest_meta.deleted && latest_meta.erasure.distribution.len() != out_dated_disks.len() {
|
||||
let distribution_len = latest_meta.erasure.distribution.len();
|
||||
let disk_slot_count = out_dated_disks.len();
|
||||
let err_str = format!(
|
||||
"unexpected file distribution length {distribution_len} for {disk_slot_count} disk slots; backend disks may have been manually modified; refusing to heal {bucket}/{object}({version_id})"
|
||||
);
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
distribution_len,
|
||||
disk_slot_count,
|
||||
state = "invalid_distribution",
|
||||
"Set disk object heal refused due to invalid erasure distribution"
|
||||
"unexpected file distribution ({:?}) from outdated disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
|
||||
latest_meta.erasure.distribution, out_dated_disks, bucket, object, version_id
|
||||
);
|
||||
warn!(err_str);
|
||||
let err = DiskError::other(err_str);
|
||||
return Ok((
|
||||
self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await,
|
||||
@@ -774,23 +585,15 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
if !latest_meta.deleted && latest_meta.erasure.distribution.len() != parts_metadata.len() {
|
||||
let distribution_len = latest_meta.erasure.distribution.len();
|
||||
let metadata_count = parts_metadata.len();
|
||||
let err_str = format!(
|
||||
"unexpected file distribution length {distribution_len} for {metadata_count} metadata entries; backend disks may have been manually modified; refusing to heal {bucket}/{object}({version_id})"
|
||||
);
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
"unexpected file distribution ({:?}) from metadata entries ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})",
|
||||
latest_meta.erasure.distribution,
|
||||
parts_metadata.len(),
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
distribution_len,
|
||||
metadata_count,
|
||||
state = "invalid_distribution",
|
||||
"Set disk object heal refused due to invalid erasure distribution"
|
||||
version_id
|
||||
);
|
||||
warn!(err_str);
|
||||
let err = DiskError::other(err_str);
|
||||
return Ok((
|
||||
self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await,
|
||||
@@ -836,12 +639,8 @@ impl SetDisks {
|
||||
None => {
|
||||
if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
error!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
"Heal object latest metadata has no data_dir, cannot heal object data"
|
||||
"heal: latest metadata for {}/{} has no data_dir, cannot heal object data",
|
||||
bucket, object
|
||||
);
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
@@ -853,9 +652,6 @@ impl SetDisks {
|
||||
|
||||
if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
let erasure_info = latest_meta.erasure.clone();
|
||||
let mut writer_failure_count = 0usize;
|
||||
let mut first_writer_failure = None;
|
||||
let mut writer_failure_warned = false;
|
||||
|
||||
for (part_index, part) in latest_meta.parts.iter().enumerate() {
|
||||
let till_offset = erasure.shard_file_offset(0, part.size, part.size);
|
||||
@@ -870,15 +666,9 @@ impl SetDisks {
|
||||
let this_part_errs =
|
||||
Self::shuffle_check_parts(&data_errs_by_part[&part_index], &erasure_info.distribution);
|
||||
if this_part_errs[index] != CHECK_PART_SUCCESS {
|
||||
trace!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
part_number = part.number,
|
||||
disk_index = index,
|
||||
part_status = this_part_errs[index],
|
||||
state = "source_shard_skipped",
|
||||
"Set disk source shard skipped"
|
||||
info!(
|
||||
"reading part {}: index={}, part_errs={:?}, skipping",
|
||||
part.number, index, this_part_errs[index]
|
||||
);
|
||||
readers.push(None);
|
||||
continue;
|
||||
@@ -936,33 +726,28 @@ impl SetDisks {
|
||||
// create writers for all disk positions, but only for outdated disks
|
||||
for (index, disk_op) in out_dated_disks.iter().enumerate() {
|
||||
if let Some(outdated_disk) = disk_op {
|
||||
let writer_result = if let Some(error) = injected_heal_writer_error(bucket, object, index)
|
||||
let writer = match create_bitrot_writer(
|
||||
is_inline_buffer,
|
||||
Some(outdated_disk),
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&path_join_buf(&[
|
||||
&tmp_id.to_string(),
|
||||
&dst_data_dir.to_string(),
|
||||
&format!("part.{}", part.number),
|
||||
]),
|
||||
erasure.shard_file_size(part.size as i64),
|
||||
erasure.shard_size(),
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(error)
|
||||
} else {
|
||||
create_bitrot_writer(
|
||||
is_inline_buffer,
|
||||
Some(outdated_disk),
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&path_join_buf(&[
|
||||
&tmp_id.to_string(),
|
||||
&dst_data_dir.to_string(),
|
||||
&format!("part.{}", part.number),
|
||||
]),
|
||||
erasure.shard_file_size(part.size as i64),
|
||||
erasure.shard_size(),
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
)
|
||||
.await
|
||||
};
|
||||
let writer = match writer_result {
|
||||
Ok(writer) => writer,
|
||||
Err(err) => {
|
||||
writer_failure_count += 1;
|
||||
if first_writer_failure.is_none() {
|
||||
first_writer_failure =
|
||||
Some((part.number, index, heal_writer_error_summary(&err)));
|
||||
}
|
||||
info!(
|
||||
"create_bitrot_writer disk {}, err {:?}, skipping operation",
|
||||
outdated_disk.to_string(),
|
||||
err
|
||||
);
|
||||
writers.push(None);
|
||||
continue;
|
||||
}
|
||||
@@ -976,20 +761,6 @@ impl SetDisks {
|
||||
// Heal each part. erasure.Heal() will write the healed
|
||||
// part to .rustfs/tmp/uuid/ which needs to be renamed
|
||||
// later to the final location.
|
||||
if writer_failure_count > 0
|
||||
&& writers.iter().all(Option::is_none)
|
||||
&& let Some(first_failure) = first_writer_failure.as_ref()
|
||||
{
|
||||
warn_heal_writer_failures(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
writer_failure_count,
|
||||
"all_targets_unavailable",
|
||||
first_failure,
|
||||
);
|
||||
writer_failure_warned = true;
|
||||
}
|
||||
if let Err(e) = erasure.heal(&mut writers, readers, part.size, &prefer).await {
|
||||
// Don't leak the partially-written healed shards in
|
||||
// .rustfs/tmp when heal fails midway (backlog#799 B20).
|
||||
@@ -1034,16 +805,6 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
if disks_to_heal_count == 0 {
|
||||
if !writer_failure_warned && let Some(first_failure) = first_writer_failure.as_ref() {
|
||||
warn_heal_writer_failures(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
writer_failure_count,
|
||||
"all_targets_unavailable",
|
||||
first_failure,
|
||||
);
|
||||
}
|
||||
// Clean up healed shards written to .rustfs/tmp before bailing (B20).
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
return Ok((
|
||||
@@ -1054,17 +815,6 @@ impl SetDisks {
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if !writer_failure_warned && let Some(first_failure) = first_writer_failure.as_ref() {
|
||||
warn_heal_writer_failures(
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
writer_failure_count,
|
||||
"partial_targets_unavailable",
|
||||
first_failure,
|
||||
);
|
||||
}
|
||||
}
|
||||
// Rename from tmp location to the actual location.
|
||||
// MinIO stops on the first RenameData error. RustFS intentionally
|
||||
@@ -1325,8 +1075,6 @@ impl SetDisks {
|
||||
Ok(()) => wrote += 1,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
bucket,
|
||||
object,
|
||||
disk_index = index,
|
||||
@@ -1347,29 +1095,11 @@ impl SetDisks {
|
||||
async fn reclaim_orphan_data_dirs_best_effort(&self, bucket: &str, object: &str) {
|
||||
match self.reclaim_orphan_data_dirs(bucket, object).await {
|
||||
Ok(removed) if removed > 0 => {
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
removed,
|
||||
state = "orphan_data_reclaimed",
|
||||
"Set disk orphaned data reclaimed"
|
||||
);
|
||||
info!(bucket, object, removed, "heal_object: reclaimed orphaned data directories");
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
error = %e,
|
||||
state = "orphan_data_reclaim_failed",
|
||||
"Set disk orphan data-dir reclaim failed"
|
||||
);
|
||||
warn!(bucket, object, error = %e, "heal_object: orphan data-dir reclaim failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1608,7 +1338,7 @@ impl SetDisks {
|
||||
Ok((result, None))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(self), fields(bucket = %bucket, object = %object))]
|
||||
#[tracing::instrument(skip(self))]
|
||||
pub(in crate::set_disk) async fn heal_object_dir(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1773,13 +1503,7 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
||||
};
|
||||
|
||||
if count_errs(&errs, &DiskError::UnformattedDisk) == 0 {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
error_count = errs.iter().flatten().count(),
|
||||
result = "no_heal_required",
|
||||
"set disk formats success"
|
||||
);
|
||||
info!("set disk formats success, NoHealRequired, errs: {:?}", errs);
|
||||
return Ok((result, Some(StorageError::NoHealRequired)));
|
||||
}
|
||||
|
||||
@@ -1808,7 +1532,7 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1912,8 +1636,8 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
||||
|
||||
#[cfg(test)]
|
||||
mod heal_result_report_tests {
|
||||
use super::{DanglingCheckPartsFailure, DanglingDeleteFailure, DanglingDeleteSafety, SetDisks, heal_writer_error_summary};
|
||||
use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope, HealWriterFailureScope};
|
||||
use super::{DanglingCheckPartsFailure, DanglingDeleteFailure, DanglingDeleteSafety, SetDisks};
|
||||
use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::format::FormatV3;
|
||||
@@ -1930,166 +1654,12 @@ mod heal_result_report_tests {
|
||||
};
|
||||
use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode};
|
||||
use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
struct CapturedLogWriter {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl CapturedLogs {
|
||||
fn contents(&self) -> String {
|
||||
let buffer = self
|
||||
.buffer
|
||||
.lock()
|
||||
.expect("captured logs mutex should not be poisoned")
|
||||
.clone();
|
||||
String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::io::Write for CapturedLogWriter {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
self.buffer
|
||||
.lock()
|
||||
.expect("captured logs mutex should not be poisoned")
|
||||
.extend_from_slice(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for CapturedLogs {
|
||||
type Writer = CapturedLogWriter;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
CapturedLogWriter {
|
||||
buffer: Arc::clone(&self.buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_writer_error_summary_redacts_io_message() {
|
||||
let error = DiskError::Io(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "/sensitive/storage/path"));
|
||||
|
||||
let summary = heal_writer_error_summary(&error);
|
||||
|
||||
assert_eq!(summary, "io::PermissionDenied");
|
||||
assert!(!summary.contains("sensitive"));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[serial_test::serial]
|
||||
async fn heal_writer_failures_emit_one_aggregate_warning_per_object() {
|
||||
for (case, failed_target_count, expected_result, expect_error) in [
|
||||
("partial", 1usize, "partial_targets_unavailable", false),
|
||||
("all", 2usize, "all_targets_unavailable", true),
|
||||
] {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = format!("heal-writer-{case}");
|
||||
let object = "object.bin";
|
||||
for disk in &disks {
|
||||
disk.make_volume(&bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let mut reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]);
|
||||
set.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let source = disks[2]
|
||||
.read_version("", &bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("source metadata should be readable");
|
||||
let data_dir = source.data_dir.expect("non-inline source should have a data directory");
|
||||
let mut target_slots = [source.erasure.distribution[0] - 1, source.erasure.distribution[1] - 1];
|
||||
target_slots.sort_unstable();
|
||||
|
||||
for index in [0, 1] {
|
||||
tokio::fs::remove_file(
|
||||
temp_dirs[index]
|
||||
.path()
|
||||
.join(&bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1"),
|
||||
)
|
||||
.await
|
||||
.expect("target shard should be removed before heal");
|
||||
}
|
||||
|
||||
let failed_slots = &target_slots[..failed_target_count];
|
||||
let logs = CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_max_level(tracing::Level::WARN)
|
||||
.with_writer(logs.clone())
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let subscriber_guard = tracing::subscriber::set_default(subscriber);
|
||||
let failure_scope = HealWriterFailureScope::install(&bucket, object, failed_slots, DiskError::DiskFull);
|
||||
|
||||
let heal_outcome = set
|
||||
.heal_object(
|
||||
&bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
drop(failure_scope);
|
||||
drop(subscriber_guard);
|
||||
|
||||
assert_eq!(
|
||||
heal_outcome.is_err(),
|
||||
expect_error,
|
||||
"{case}: aggregate heal result should match writer outcomes"
|
||||
);
|
||||
let output = logs.contents();
|
||||
assert_eq!(
|
||||
output.matches("Set disk object heal writer failures").count(),
|
||||
1,
|
||||
"{case}: writer failures must emit one aggregate warning per object: {output}"
|
||||
);
|
||||
assert!(
|
||||
output.contains(&format!("writer_failure_count={failed_target_count}")),
|
||||
"{case}: warning must report the aggregate failure count: {output}"
|
||||
);
|
||||
assert!(
|
||||
output.contains(&format!("first_disk_index={}", failed_slots[0])),
|
||||
"{case}: warning must report the first failed target: {output}"
|
||||
);
|
||||
assert!(
|
||||
output.contains("first_part_number=1"),
|
||||
"{case}: warning must report the first failed part: {output}"
|
||||
);
|
||||
assert!(
|
||||
output.contains(&format!("result=\"{expected_result}\"")),
|
||||
"{case}: warning must distinguish partial from all-target failure: {output}"
|
||||
);
|
||||
assert!(
|
||||
output.contains("error=drive path full"),
|
||||
"{case}: warning must preserve a redacted failure reason: {output}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn real_disk() -> (TempDir, Endpoint, DiskStore) {
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let endpoint =
|
||||
|
||||
@@ -134,7 +134,7 @@ impl crate::storage_api_contracts::list::ListOperations for SetDisks {
|
||||
type WalkCancellation = CancellationToken;
|
||||
type WalkResultSender = Sender<ObjectInfoOrErr>;
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(self))]
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn list_objects_v2(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
|
||||
@@ -28,7 +28,7 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
||||
type Error = Error;
|
||||
type NamespaceLock = NamespaceLockWrapper;
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(self))]
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
|
||||
// Resolved from this set's own instance context (backlog#1052), not the
|
||||
// ambient facade: the facade tracks whichever context is currently
|
||||
|
||||
@@ -357,20 +357,6 @@ fn paginate_upload_page(remaining: &[MultipartInfo], max_uploads: usize) -> (Vec
|
||||
(page, is_truncated, next_upload_id_marker)
|
||||
}
|
||||
|
||||
/// Deterministic per-upload damage during a multipart listing: the metadata
|
||||
/// exists but is torn, undecodable, or not identifiable as an upload. The same
|
||||
/// corrupt family as `classify_metadata_response_error`'s corrupt group. This
|
||||
/// deliberately excludes transient fault shapes (`DiskNotFound`, `Timeout`,
|
||||
/// `Io`, `ErasureReadQuorum` from offline-disk quorum loss, ...): degrading on
|
||||
/// those would silently drop healthy uploads from a 200 listing while disks
|
||||
/// are merely unreachable (issue #5716 review).
|
||||
fn is_corrupt_upload_metadata_error(err: &DiskError) -> bool {
|
||||
matches!(
|
||||
err,
|
||||
DiskError::FileCorrupt | DiskError::CorruptedFormat | DiskError::CorruptedBackend | DiskError::OutdatedXLMeta
|
||||
)
|
||||
}
|
||||
|
||||
async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str) -> disk::error::Result<Vec<String>> {
|
||||
if !disk.is_online().await {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
@@ -651,78 +637,25 @@ impl SetDisks {
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
// Affirmative per-upload damage (issue #5716): the staging
|
||||
// namespace is one flat set of sha256(bucket/object)
|
||||
// directories shared by every bucket, so a directory whose
|
||||
// metadata is torn or undecodable must degrade to that
|
||||
// upload alone. Erroring out instead turns one damaged
|
||||
// directory into a permanent InternalError for every
|
||||
// ListMultipartUploads of the bucket. Only the corrupt
|
||||
// error family counts as damage — transient faults (offline
|
||||
// disks, timeouts, IO churn) keep failing the listing below
|
||||
// so clients retry instead of silently losing entries.
|
||||
let corrupt_metadata = errs
|
||||
.iter()
|
||||
.filter(|err| err.as_ref().is_some_and(is_corrupt_upload_metadata_error))
|
||||
.count();
|
||||
if corrupt_metadata > 0 && missing_metadata + corrupt_metadata >= discovery_quorum {
|
||||
debug!(
|
||||
bucket,
|
||||
upload_path = %upload_path,
|
||||
missing_metadata,
|
||||
corrupt_metadata,
|
||||
"skipping multipart upload directory with corrupt metadata during listing"
|
||||
);
|
||||
let (read_quorum, _) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)?;
|
||||
let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
let (_, mod_time, etag) = Self::list_online_disks(disks, &parts_metadata, &errs, read_quorum);
|
||||
let file_info = Self::pick_valid_fileinfo(&parts_metadata, mod_time, etag, read_quorum)?;
|
||||
if expected_incarnation_id
|
||||
.is_some_and(|expected| !multipart_bucket_incarnation_matches(&file_info.metadata, expected))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let decoded: disk::error::Result<Option<(FileInfo, String)>> = (|| {
|
||||
let (read_quorum, _) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)?;
|
||||
let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
let (_, mod_time, etag) = Self::list_online_disks(disks, &parts_metadata, &errs, read_quorum);
|
||||
let file_info = Self::pick_valid_fileinfo(&parts_metadata, mod_time, etag, read_quorum)?;
|
||||
if expected_incarnation_id
|
||||
.is_some_and(|expected| !multipart_bucket_incarnation_matches(&file_info.metadata, expected))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let object = match (
|
||||
file_info.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
|
||||
file_info.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY),
|
||||
) {
|
||||
(Some(stored_bucket), Some(object)) if stored_bucket == bucket && !object.is_empty() => {
|
||||
object.clone()
|
||||
}
|
||||
// A healthy upload that belongs to another bucket:
|
||||
// not ours to list, and not corruption.
|
||||
(Some(stored_bucket), Some(_)) if stored_bucket != bucket => return Ok(None),
|
||||
_ => return Err(DiskError::CorruptedFormat),
|
||||
};
|
||||
Ok(Some((file_info, object)))
|
||||
})();
|
||||
let (file_info, object) = match decoded {
|
||||
Ok(Some(decoded)) => decoded,
|
||||
Ok(None) => return Ok(None),
|
||||
// Deterministic damage (undecodable metadata that still
|
||||
// reached quorum, or metadata without its owner keys):
|
||||
// skip this upload only.
|
||||
Err(err) if is_corrupt_upload_metadata_error(&err) => {
|
||||
debug!(
|
||||
bucket,
|
||||
upload_path = %upload_path,
|
||||
error = %err,
|
||||
"skipping multipart upload directory with unidentifiable metadata during listing"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
// Everything else — quorum loss from offline disks,
|
||||
// timeouts, transport errors — can hide uploads that are
|
||||
// actually fine; keep failing the listing so clients
|
||||
// retry instead of silently losing entries.
|
||||
Err(err) => return Err(err),
|
||||
let object = match (
|
||||
file_info.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
|
||||
file_info.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY),
|
||||
) {
|
||||
(Some(stored_bucket), Some(object)) if stored_bucket == bucket && !object.is_empty() => object.clone(),
|
||||
_ => return Err(DiskError::CorruptedFormat),
|
||||
};
|
||||
if !object.starts_with(prefix) {
|
||||
return Ok(None);
|
||||
@@ -2055,26 +1988,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Detach admission before any post-commit await: client cancellation
|
||||
// must not couple durable convergence repair to cleanup work.
|
||||
if convergence.needs_heal() {
|
||||
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(self.pool_index),
|
||||
Some(self.set_index),
|
||||
);
|
||||
request.object_version_id = fi
|
||||
.version_id
|
||||
.or_else(|| opts.version_suspended.then(Uuid::nil))
|
||||
.map(|version_id| version_id.to_string());
|
||||
tokio::spawn(async move {
|
||||
let _ = rustfs_common::heal_channel::send_heal_request(request).await;
|
||||
});
|
||||
}
|
||||
|
||||
// Crash-consistency injection: hard power loss after the authoritative
|
||||
// rename_data commit succeeded but before the stale part.N.meta cleanup.
|
||||
// The new version is durably committed and visible, so a crash here must
|
||||
@@ -2148,6 +2061,48 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
drop(object_lock_guard); // drop object lock guard to release the lock
|
||||
|
||||
// backlog#1321: enqueue heal only when the committed replicas actually
|
||||
// need to converge — a partial commit (some disk failed/offline) or a
|
||||
// signature divergence between committed replicas. A fully healthy MPU
|
||||
// (identical signatures on every disk) is `AllSuccessIdentical` and
|
||||
// submits nothing, which is the fix: the old `Option::is_some()` gate
|
||||
// treated the mere existence of a version signature as "needs heal", so
|
||||
// every healthy <=10-version completion self-enqueued.
|
||||
//
|
||||
// The submit is detached (`tokio::spawn`) so it stays off the ACK
|
||||
// critical path AND survives cancellation of the completion future: the
|
||||
// write is already durable and ACK-worthy, so the heal admission must
|
||||
// not ride the client's request lifetime. The admission itself is
|
||||
// bounded / deduplicated / observable (`send_heal_request` ->
|
||||
// `HealAdmissionResult`), so this emits at most one submit per
|
||||
// completion and coalesces with any in-flight heal for the same object.
|
||||
//
|
||||
// Scanner backstop (backlog#1321 patch): a `PartialCommit` whose
|
||||
// completion is cancelled in the narrow window after the durable commit
|
||||
// but before this spawn runs is not lost — the divergence it would have
|
||||
// healed is exactly what the background scanner reconciles. `Unknown`
|
||||
// (>10 versions, no signature produced) likewise relies on the scanner
|
||||
// rather than self-enqueuing.
|
||||
if convergence.needs_heal() {
|
||||
let bucket = bucket.to_string();
|
||||
let object = object.to_string();
|
||||
let pool_index = self.pool_index;
|
||||
let set_index = self.set_index;
|
||||
tokio::spawn(async move {
|
||||
let _ = rustfs_common::heal_channel::send_heal_request(
|
||||
rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket,
|
||||
Some(object),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(pool_index),
|
||||
Some(set_index),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
|
||||
for (i, op_disk) in online_disks.iter().enumerate() {
|
||||
if let Some(disk) = op_disk
|
||||
&& disk.is_online().await
|
||||
@@ -4021,9 +3976,6 @@ mod tests {
|
||||
}
|
||||
|
||||
// Start more in-progress uploads on the same object than a single page holds.
|
||||
// Track only the decoded `<uuid>x<timestamp>` suffixes: the full upload id
|
||||
// embeds the process-global deployment id, which a concurrently running
|
||||
// test can swap between create and list time.
|
||||
let total = 5usize;
|
||||
let mut created = HashSet::new();
|
||||
for _ in 0..total {
|
||||
@@ -4031,10 +3983,7 @@ mod tests {
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
assert!(
|
||||
created.insert(runtime_sources::upload_uuid_suffix(&res.upload_id)),
|
||||
"each upload id must be unique"
|
||||
);
|
||||
assert!(created.insert(res.upload_id), "each upload id must be unique");
|
||||
}
|
||||
|
||||
// A single page must never return more than max_uploads entries.
|
||||
@@ -4089,7 +4038,7 @@ mod tests {
|
||||
assert!(page.uploads.len() <= 1, "max_uploads=1 must never return more than one upload");
|
||||
for upload in &page.uploads {
|
||||
assert!(
|
||||
seen.insert(runtime_sources::upload_uuid_suffix(&upload.upload_id)),
|
||||
seen.insert(upload.upload_id.clone()),
|
||||
"upload {} was returned more than once across pages",
|
||||
upload.upload_id
|
||||
);
|
||||
@@ -4118,16 +4067,13 @@ mod tests {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
// Compare only the decoded `<uuid>x<timestamp>` suffixes: the full
|
||||
// upload id embeds the process-global deployment id, which a
|
||||
// concurrently running test can swap between create and list time.
|
||||
let mut expected = Vec::new();
|
||||
for object in ["logs/a.bin", "logs/a.bin", "logs/b.bin", "other/c.bin"] {
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
expected.push((object.to_string(), runtime_sources::upload_uuid_suffix(&upload.upload_id)));
|
||||
expected.push((object.to_string(), upload.upload_id));
|
||||
}
|
||||
expected.sort();
|
||||
|
||||
@@ -4135,12 +4081,11 @@ mod tests {
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("bucket-wide multipart listing should succeed");
|
||||
let mut listed = all
|
||||
let listed = all
|
||||
.uploads
|
||||
.iter()
|
||||
.map(|upload| (upload.object.clone(), runtime_sources::upload_uuid_suffix(&upload.upload_id)))
|
||||
.map(|upload| (upload.object.clone(), upload.upload_id.clone()))
|
||||
.collect::<Vec<_>>();
|
||||
listed.sort();
|
||||
assert_eq!(listed, expected);
|
||||
assert!(!all.is_truncated);
|
||||
|
||||
@@ -4204,18 +4149,7 @@ mod tests {
|
||||
assert!(upload_id_marker.is_some());
|
||||
}
|
||||
|
||||
// Compare only the decoded `<uuid>x<timestamp>` suffixes: the full
|
||||
// upload id embeds the process-global deployment id, which a
|
||||
// concurrently running test can swap between create and list time.
|
||||
let normalize = |uploads: &[(String, String)]| {
|
||||
let mut normalized = uploads
|
||||
.iter()
|
||||
.map(|(object, upload_id)| (object.clone(), runtime_sources::upload_uuid_suffix(upload_id)))
|
||||
.collect::<Vec<_>>();
|
||||
normalized.sort();
|
||||
normalized
|
||||
};
|
||||
assert_eq!(normalize(&listed), normalize(&expected));
|
||||
assert_eq!(listed, expected);
|
||||
|
||||
let key_only = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "logs/", Some("logs/a.bin".to_string()), None, None, 1000, None)
|
||||
@@ -4334,203 +4268,7 @@ mod tests {
|
||||
.await
|
||||
.expect("incarnation-scoped multipart listing should succeed");
|
||||
assert_eq!(scoped.uploads.len(), 1);
|
||||
// Compare only the decoded `<uuid>x<timestamp>` suffixes: the full
|
||||
// upload id embeds the process-global deployment id, which a
|
||||
// concurrently running test can swap between create and list time.
|
||||
assert_eq!(
|
||||
runtime_sources::upload_uuid_suffix(&scoped.uploads[0].upload_id),
|
||||
runtime_sources::upload_uuid_suffix(¤t.upload_id)
|
||||
);
|
||||
}
|
||||
|
||||
/// The `<deployment-id>.` prefix inside an upload id is read from a
|
||||
/// process-global that concurrently-running tests reinitialize, so id
|
||||
/// assertions compare only the stable `<uuid>x<timestamp>` suffix.
|
||||
fn upload_uuid_suffix(upload_id: &str) -> String {
|
||||
let decoded = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(upload_id.as_bytes())
|
||||
.expect("upload id should be url-safe base64");
|
||||
let decoded = String::from_utf8(decoded).expect("upload id should decode to utf8");
|
||||
decoded
|
||||
.split_once('.')
|
||||
.map(|(_, suffix)| suffix.to_owned())
|
||||
.unwrap_or(decoded)
|
||||
}
|
||||
|
||||
/// Regression (issue #5716): the multipart staging namespace is flat —
|
||||
/// `sha256(bucket/object)` directories from every bucket share one volume —
|
||||
/// so a bucket-scoped listing reads metadata belonging to other buckets'
|
||||
/// in-flight uploads. Those entries must be filtered out, not treated as
|
||||
/// corruption: with the `_ => CorruptedFormat` arm, any concurrent upload
|
||||
/// in another bucket turned every ListMultipartUploads for this bucket
|
||||
/// into an InternalError.
|
||||
#[tokio::test]
|
||||
async fn list_multipart_uploads_ignores_other_buckets_uploads() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cross-bucket-a";
|
||||
let other_bucket = "multipart-cross-bucket-b";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
disk.make_volume(other_bucket)
|
||||
.await
|
||||
.expect("other bucket volume should be created");
|
||||
}
|
||||
|
||||
let mine = set_disks
|
||||
.new_multipart_upload(bucket, "blobs/data/layer.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
set_disks
|
||||
.new_multipart_upload(other_bucket, "cache/other-layer.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("other bucket multipart upload should be created");
|
||||
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "blobs/", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("a concurrent upload in another bucket must not poison this bucket's listing");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&mine.upload_id));
|
||||
|
||||
let bucket_wide = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("bucket-wide listing must also skip other buckets' uploads");
|
||||
assert_eq!(bucket_wide.uploads.len(), 1);
|
||||
assert_eq!(bucket_wide.uploads[0].object, "blobs/data/layer.bin");
|
||||
}
|
||||
|
||||
/// Regression (issue #5716): a single upload directory whose `xl.meta` was
|
||||
/// destroyed (crash mid-write, torn disk state) must degrade to that upload
|
||||
/// alone. Failing the whole ListMultipartUploads turns one piece of stale
|
||||
/// debris into a permanent outage for every multipart client of the bucket.
|
||||
#[tokio::test]
|
||||
async fn list_multipart_uploads_skips_undecodable_upload_dirs() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-corrupt-dir-bucket";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let healthy = set_disks
|
||||
.new_multipart_upload(bucket, "healthy/object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("healthy multipart upload should be created");
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "debris/object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("debris multipart upload should be created");
|
||||
|
||||
// Destroy the debris upload's metadata on every disk, as an unclean
|
||||
// shutdown mid-create can. The directory stays listable while its
|
||||
// xl.meta no longer decodes.
|
||||
let debris_sha = SetDisks::get_multipart_sha_dir(bucket, "debris/object.bin");
|
||||
let mut corrupted = 0usize;
|
||||
for temp_dir in &temp_dirs {
|
||||
let sha_dir = temp_dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&debris_sha);
|
||||
for meta in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
|
||||
if meta.starts_with(sha_dir.to_string_lossy().as_ref()) {
|
||||
tokio::fs::write(&meta, b"not an xl.meta")
|
||||
.await
|
||||
.expect("corrupting xl.meta should succeed");
|
||||
corrupted += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(corrupted > 0, "the debris upload must exist on disk before corruption");
|
||||
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "healthy/", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("one undecodable upload dir must not fail the whole listing");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&healthy.upload_id));
|
||||
|
||||
let bucket_wide = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("bucket-wide listing must skip the undecodable upload dir");
|
||||
assert_eq!(bucket_wide.uploads.len(), 1);
|
||||
assert_eq!(
|
||||
upload_uuid_suffix(&bucket_wide.uploads[0].upload_id),
|
||||
upload_uuid_suffix(&healthy.upload_id)
|
||||
);
|
||||
}
|
||||
|
||||
/// Companion boundary to `list_multipart_uploads_skips_undecodable_upload_dirs`:
|
||||
/// corruption BELOW the discovery quorum must not hide the upload — the
|
||||
/// surviving disks still identify it, so it stays listed.
|
||||
#[tokio::test]
|
||||
async fn list_multipart_uploads_survives_sub_quorum_corruption() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-partial-corrupt-bucket";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, "partial/object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
|
||||
// Corrupt the upload's metadata on exactly one disk (discovery quorum
|
||||
// on this 4-disk set is 2): the other replicas keep it identifiable.
|
||||
let sha = SetDisks::get_multipart_sha_dir(bucket, "partial/object.bin");
|
||||
let mut corrupted = 0usize;
|
||||
for temp_dir in &temp_dirs {
|
||||
let sha_dir = temp_dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&sha);
|
||||
for meta in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
|
||||
if meta.starts_with(sha_dir.to_string_lossy().as_ref()) {
|
||||
tokio::fs::write(&meta, b"not an xl.meta")
|
||||
.await
|
||||
.expect("corrupting xl.meta should succeed");
|
||||
corrupted += 1;
|
||||
}
|
||||
}
|
||||
if corrupted > 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert_eq!(corrupted, 1, "exactly one disk's metadata should be corrupted");
|
||||
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "partial/", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("sub-quorum corruption must not fail the listing");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&upload.upload_id));
|
||||
}
|
||||
|
||||
/// Pins the degrade-vs-propagate classification (issue #5716 review): only
|
||||
/// affirmative corruption may skip an upload during listing; every
|
||||
/// transient fault shape must keep failing the listing so clients retry
|
||||
/// instead of silently losing entries.
|
||||
#[test]
|
||||
fn corrupt_upload_metadata_classification_excludes_transient_faults() {
|
||||
for corrupt in [
|
||||
DiskError::FileCorrupt,
|
||||
DiskError::CorruptedFormat,
|
||||
DiskError::CorruptedBackend,
|
||||
DiskError::OutdatedXLMeta,
|
||||
] {
|
||||
assert!(is_corrupt_upload_metadata_error(&corrupt), "{corrupt:?} is deterministic damage");
|
||||
}
|
||||
for transient in [
|
||||
DiskError::DiskNotFound,
|
||||
DiskError::Timeout,
|
||||
DiskError::FaultyDisk,
|
||||
DiskError::FaultyRemoteDisk,
|
||||
DiskError::DiskAccessDenied,
|
||||
DiskError::VolumeAccessDenied,
|
||||
DiskError::ErasureReadQuorum,
|
||||
DiskError::FileNotFound,
|
||||
DiskError::Io(std::io::Error::other("connection refused")),
|
||||
] {
|
||||
assert!(
|
||||
!is_corrupt_upload_metadata_error(&transient),
|
||||
"{transient:?} must keep failing the listing"
|
||||
);
|
||||
}
|
||||
assert_eq!(scoped.uploads[0].upload_id, current.upload_id);
|
||||
}
|
||||
|
||||
/// Recursively collect every file named `file_name` under the multipart
|
||||
@@ -5017,13 +4755,7 @@ mod tests {
|
||||
.list_multipart_uploads_for_incarnation(bucket, object, None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("listing multipart uploads should succeed");
|
||||
// Compare only the decoded `<uuid>x<timestamp>` suffixes: the full
|
||||
// upload id embeds the process-global deployment id, which a
|
||||
// concurrently running test can swap between create and list time.
|
||||
let expected_suffix = runtime_sources::upload_uuid_suffix(upload_id);
|
||||
page.uploads
|
||||
.iter()
|
||||
.any(|u| runtime_sources::upload_uuid_suffix(&u.upload_id) == expected_suffix)
|
||||
page.uploads.iter().any(|u| u.upload_id == upload_id)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -972,9 +972,8 @@ impl SetDisks {
|
||||
|
||||
let mut object_lock_guard = None;
|
||||
let mut bucket_lifecycle_guard = None;
|
||||
let deferred_data_movement_precondition = opts.data_movement && opts.http_preconditions.is_some();
|
||||
|
||||
if opts.http_preconditions.is_some() && !deferred_data_movement_precondition {
|
||||
if opts.http_preconditions.is_some() {
|
||||
if !opts.no_lock {
|
||||
if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id
|
||||
&& opts.bucket_lifecycle_lock_fence.is_none()
|
||||
@@ -982,9 +981,10 @@ impl SetDisks {
|
||||
bucket_lifecycle_guard = Some(
|
||||
metadata_sys::object_store_in(&self.ctx)
|
||||
.await?
|
||||
.acquire_bucket_incarnation_fence(bucket, expected_incarnation_id)
|
||||
.acquire_bucket_lifecycle_read_lock(bucket)
|
||||
.await?,
|
||||
);
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
}
|
||||
object_lock_guard = Some(
|
||||
self.acquire_write_lock_diag("put_object_precondition", bucket, object)
|
||||
@@ -1320,19 +1320,16 @@ impl SetDisks {
|
||||
bucket_lifecycle_guard = Some(
|
||||
metadata_sys::object_store_in(&self.ctx)
|
||||
.await?
|
||||
.acquire_bucket_incarnation_fence(bucket, expected_incarnation_id)
|
||||
.acquire_bucket_lifecycle_read_lock(bucket)
|
||||
.await?,
|
||||
);
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
}
|
||||
object_lock_guard = Some(self.acquire_write_lock_diag("put_object_commit", bucket, object).await?);
|
||||
}
|
||||
#[cfg(test)]
|
||||
pause_put_object_commit(bucket, object, PutObjectCommitPause::AfterNamespace).await;
|
||||
|
||||
if deferred_data_movement_precondition && let Some(err) = self.check_write_precondition(bucket, object, opts).await {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
// Generate ordinary PUT timestamps under the commit lock so version
|
||||
// ordering follows durable commit ordering when writers queued on
|
||||
// the same object. Internal callers with an explicit timestamp keep
|
||||
@@ -1448,7 +1445,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let rename_stage_start = Instant::now();
|
||||
let (online_disks, convergence, op_old_dir, cleanup_disks, old_current_size) = Self::rename_data(
|
||||
let (online_disks, _, op_old_dir, cleanup_disks, old_current_size) = Self::rename_data(
|
||||
&shuffle_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
tmp_dir.as_str(),
|
||||
@@ -1458,23 +1455,6 @@ impl SetDisks {
|
||||
write_quorum,
|
||||
)
|
||||
.await?;
|
||||
// Do this before any post-commit await so request cancellation cannot
|
||||
// bypass best-effort admission. A process crash before admission
|
||||
// remains subject to the existing scanner reconciliation path.
|
||||
if convergence.needs_heal() {
|
||||
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(self.pool_index),
|
||||
Some(self.set_index),
|
||||
);
|
||||
request.object_version_id = fi.version_id.map(|version_id| version_id.to_string());
|
||||
tokio::spawn(async move {
|
||||
let _ = rustfs_common::heal_channel::send_heal_request(request).await;
|
||||
});
|
||||
}
|
||||
let rename_stage_ms = rename_stage_start.elapsed().as_millis() as u64;
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", rename_stage_ms as f64);
|
||||
if (rename_stage_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS {
|
||||
@@ -4363,7 +4343,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
Ok(obj_info)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(self))]
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
|
||||
crate::hp_guard!("SetDisks::get_object_info");
|
||||
// Acquire a shared read-lock to protect consistency during info fetch
|
||||
@@ -4388,16 +4368,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
|
||||
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(self.pool_index),
|
||||
Some(self.set_index),
|
||||
);
|
||||
request.object_version_id = (!version_id.is_empty()).then(|| version_id.to_string());
|
||||
if let Err(e) = rustfs_common::heal_channel::send_heal_request(request).await {
|
||||
if let Err(e) =
|
||||
rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(self.pool_index),
|
||||
Some(self.set_index),
|
||||
))
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
bucket,
|
||||
object,
|
||||
@@ -5783,7 +5764,7 @@ mod transition_commit_failure_tests {
|
||||
use http::HeaderMap;
|
||||
use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
|
||||
use s3s::dto::RestoreRequest;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
fn restore_operation_id_metadata(operation_id: Uuid) -> HashMap<String, String> {
|
||||
let mut metadata = HashMap::new();
|
||||
@@ -7581,58 +7562,6 @@ mod transition_upload_integrity_tests {
|
||||
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread", start_paused = true)]
|
||||
#[serial_test::serial]
|
||||
async fn data_movement_cleanup_aborts_after_outer_lock_loss() {
|
||||
let refresh_calls = Arc::new(AtomicUsize::new(0));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = (0..4)
|
||||
.map(|_| Arc::new(LockLostRefreshClient::new(Arc::clone(&refresh_calls))) as Arc<dyn LockClient>)
|
||||
.collect();
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "data-movement-cleanup-lock-lost";
|
||||
let object = "object.bin";
|
||||
let payload = b"lost data movement cleanup lock must preserve the source".repeat(1024);
|
||||
write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
||||
let expected = set_disks
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("source versions should be readable")
|
||||
.expect("source versions should exist");
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(bucket, object);
|
||||
|
||||
let cleanup_set = Arc::clone(&set_disks);
|
||||
let cleanup = tokio::spawn(async move {
|
||||
crate::data_movement::cleanup_source_entry_if_unchanged(
|
||||
cleanup_set,
|
||||
bucket,
|
||||
object,
|
||||
&expected,
|
||||
&[],
|
||||
"test_data_movement",
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
tokio::time::advance(Duration::from_secs(11)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(
|
||||
refresh_calls.load(Ordering::SeqCst) > 0,
|
||||
"test must drive the real distributed-lock heartbeat before cleanup commit"
|
||||
);
|
||||
barrier.release();
|
||||
|
||||
let error = cleanup
|
||||
.await
|
||||
.expect("cleanup task should not panic")
|
||||
.expect_err("cleanup must fail after its outer namespace lock loses refresh quorum");
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::data_movement::SourceCleanupError::Storage(StorageError::NamespaceLockQuorumUnavailable { .. })
|
||||
));
|
||||
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn partial_remote_acceptance_cleans_exact_candidate_and_preserves_source() {
|
||||
@@ -8727,87 +8656,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn data_movement_precondition_is_rechecked_at_commit() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "data-movement-commit-precondition";
|
||||
let object = "object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let migration_body = vec![b'm'; 64 * 1024];
|
||||
let split = migration_body.len() / 2;
|
||||
let (mut source, stream) = tokio::io::duplex(64);
|
||||
let hash_reader = HashReader::from_stream(
|
||||
stream,
|
||||
i64::try_from(migration_body.len()).expect("migration body length should fit i64"),
|
||||
i64::try_from(migration_body.len()).expect("migration body length should fit i64"),
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.expect("migration hash reader should be created");
|
||||
let migration_store = Arc::clone(&set_disks);
|
||||
let migration = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::new(hash_reader);
|
||||
migration_store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
source
|
||||
.write_all(&migration_body[..split])
|
||||
.await
|
||||
.expect("migration should consume the first half before commit");
|
||||
let mut client_reader = PutObjReader::from_vec(b"new client body".to_vec());
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
set_disks.put_object(bucket, object, &mut client_reader, &ObjectOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("client write must not wait for the migration body")
|
||||
.expect("client write should commit while migration waits for the remaining source body");
|
||||
let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
|
||||
source
|
||||
.write_all(&migration_body[split..])
|
||||
.await
|
||||
.expect("migration should consume the remaining source body");
|
||||
drop(source);
|
||||
barrier.wait_until_paused().await;
|
||||
barrier.release();
|
||||
|
||||
let err = migration
|
||||
.await
|
||||
.expect("migration task should join")
|
||||
.expect_err("migration must recheck the target after acquiring its commit lock");
|
||||
assert_eq!(err, StorageError::PreconditionFailed);
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("client object should remain readable");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("client object should drain");
|
||||
assert_eq!(body, b"new client body");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_copy_no_lock_aborts_after_outer_namespace_lock_loss() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
|
||||
@@ -176,52 +176,6 @@ impl ECStore {
|
||||
})
|
||||
}
|
||||
|
||||
/// Acquire the bucket lifecycle read lock and validate the bucket
|
||||
/// incarnation against `expected`, memoizing the validation while this
|
||||
/// node keeps continuous read-lock coverage (see [`super::bucket_fence`]).
|
||||
///
|
||||
/// Semantics are identical to the pre-existing per-PUT
|
||||
/// `acquire_bucket_lifecycle_read_lock` + from-disk
|
||||
/// `validate_bucket_incarnation` pair: the first PUT in a coverage window
|
||||
/// performs exactly that authoritative disk validation; overlapping PUTs
|
||||
/// reuse its result, which is sound because bucket deletion/recreation
|
||||
/// requires the lifecycle WRITE lock and therefore cannot have run while
|
||||
/// any read guard was continuously held.
|
||||
pub(crate) async fn acquire_bucket_incarnation_fence(
|
||||
&self,
|
||||
bucket: &str,
|
||||
expected: uuid::Uuid,
|
||||
) -> Result<super::bucket_fence::BucketIncarnationFenceGuard> {
|
||||
let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let pieces = super::bucket_fence::FencePieces {
|
||||
registry: self.bucket_fence_registry.clone(),
|
||||
inner,
|
||||
};
|
||||
let memoized = pieces.enter(bucket);
|
||||
let current = match memoized {
|
||||
Some(current) => current,
|
||||
None => match metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await {
|
||||
Ok(current) => {
|
||||
// Never memoize under lost coverage: a granted lifecycle
|
||||
// write lock could already have changed the incarnation.
|
||||
if !pieces.lock_lost() {
|
||||
pieces.memoize(bucket, current);
|
||||
}
|
||||
current
|
||||
}
|
||||
Err(err) => {
|
||||
pieces.abandon(bucket);
|
||||
return Err(err);
|
||||
}
|
||||
},
|
||||
};
|
||||
if current != expected {
|
||||
pieces.abandon(bucket);
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
Ok(pieces.into_guard(bucket))
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_bucket_lifecycle_write_lock(&self, bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
|
||||
let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?;
|
||||
lock.get_write_lock(get_lock_acquire_timeout())
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Memoized bucket-incarnation validation under continuous lifecycle read-lock
|
||||
//! coverage.
|
||||
//!
|
||||
//! The PUT commit fence introduced by #5648 validated the bucket incarnation
|
||||
//! with an uncached read (`get_bucket_incarnation_id_from_disk`: a distributed
|
||||
//! metadata-transaction read lock plus an EC quorum read of the bucket
|
||||
//! metadata) on every PUT commit. Under small-object write load that is two
|
||||
//! extra quorum round-trips per PUT, and the resulting lock-manager pressure
|
||||
//! produced sustained `Lock acquisition timeout` errors (~1,000 client-visible
|
||||
//! failures per 5-minute 64-concurrency window in benchmarks).
|
||||
//!
|
||||
//! The memo exploits the fence's own locking protocol: bucket deletion and
|
||||
//! recreation take the bucket lifecycle WRITE lock, while every fenced PUT
|
||||
//! holds a lifecycle READ lock for the whole commit. Therefore, while at least
|
||||
//! one lifecycle read guard on this node has been held continuously, no
|
||||
//! lifecycle write lock can have been granted anywhere in the cluster, so the
|
||||
//! bucket incarnation cannot have changed. The first fenced PUT in such a
|
||||
//! coverage window pays the authoritative disk validation exactly as before;
|
||||
//! subsequent PUTs whose guards overlap that window compare against the
|
||||
//! memoized value. When the node's last guard drops — or any guard observes
|
||||
//! `is_lock_lost` — the memo is cleared and the next PUT revalidates from
|
||||
//! disk.
|
||||
//!
|
||||
//! The memo is deliberately per-node process state (not a cross-node cache):
|
||||
//! its validity is derived purely from locks this process itself holds, so
|
||||
//! best-effort peer cache invalidation (which is why the fence read from disk
|
||||
//! in the first place) is irrelevant to its correctness.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Default)]
|
||||
struct FenceEntry {
|
||||
guards: usize,
|
||||
validated: Option<Uuid>,
|
||||
}
|
||||
|
||||
/// Per-store registry tracking, per bucket, how many lifecycle read guards are
|
||||
/// live on this node and the incarnation id validated under that coverage.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct BucketFenceRegistry {
|
||||
entries: Mutex<HashMap<String, FenceEntry>>,
|
||||
}
|
||||
|
||||
impl BucketFenceRegistry {
|
||||
/// Register a new live guard for `bucket` and return the memoized
|
||||
/// incarnation id if one is valid for the current coverage window.
|
||||
fn enter(&self, bucket: &str) -> Option<Uuid> {
|
||||
let mut entries = self.entries.lock().expect("bucket fence registry poisoned");
|
||||
let entry = entries.entry(bucket.to_string()).or_default();
|
||||
entry.guards += 1;
|
||||
entry.validated
|
||||
}
|
||||
|
||||
/// Memoize `incarnation` for `bucket`. Only meaningful while the caller
|
||||
/// still holds a registered guard (which it does by construction).
|
||||
fn memoize(&self, bucket: &str, incarnation: Uuid) {
|
||||
let mut entries = self.entries.lock().expect("bucket fence registry poisoned");
|
||||
if let Some(entry) = entries.get_mut(bucket)
|
||||
&& entry.guards > 0
|
||||
{
|
||||
entry.validated = Some(incarnation);
|
||||
}
|
||||
}
|
||||
|
||||
/// Deregister a guard. Clears the memo when the last guard leaves or when
|
||||
/// the leaving guard lost its lock (lost coverage means a lifecycle write
|
||||
/// lock may have been granted, so the memo can no longer be trusted).
|
||||
fn exit(&self, bucket: &str, lock_lost: bool) {
|
||||
let mut entries = self.entries.lock().expect("bucket fence registry poisoned");
|
||||
if let Some(entry) = entries.get_mut(bucket) {
|
||||
entry.guards = entry.guards.saturating_sub(1);
|
||||
if lock_lost {
|
||||
entry.validated = None;
|
||||
}
|
||||
if entry.guards == 0 {
|
||||
entries.remove(bucket);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A held bucket lifecycle read lock plus its registration in the fence
|
||||
/// registry. Dropping the guard deregisters it; the memo is cleared when the
|
||||
/// last guard for the bucket drops (or a lost lock is observed).
|
||||
pub(crate) struct BucketIncarnationFenceGuard {
|
||||
inner: Option<NamespaceLockGuard>,
|
||||
registry: Arc<BucketFenceRegistry>,
|
||||
bucket: String,
|
||||
}
|
||||
|
||||
impl BucketIncarnationFenceGuard {
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BucketIncarnationFenceGuard {
|
||||
fn drop(&mut self) {
|
||||
let lost = self.is_lock_lost();
|
||||
self.registry.exit(&self.bucket, lost);
|
||||
self.inner.take();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct FencePieces {
|
||||
pub(super) registry: Arc<BucketFenceRegistry>,
|
||||
pub(super) inner: NamespaceLockGuard,
|
||||
}
|
||||
|
||||
impl FencePieces {
|
||||
/// Register the freshly acquired read lock and return the memoized
|
||||
/// incarnation for the coverage window, if any.
|
||||
pub(super) fn enter(&self, bucket: &str) -> Option<Uuid> {
|
||||
self.registry.enter(bucket)
|
||||
}
|
||||
|
||||
pub(super) fn memoize(&self, bucket: &str, incarnation: Uuid) {
|
||||
self.registry.memoize(bucket, incarnation)
|
||||
}
|
||||
|
||||
pub(super) fn lock_lost(&self) -> bool {
|
||||
self.inner.is_lock_lost()
|
||||
}
|
||||
|
||||
pub(super) fn into_guard(self, bucket: &str) -> BucketIncarnationFenceGuard {
|
||||
BucketIncarnationFenceGuard {
|
||||
inner: Some(self.inner),
|
||||
registry: self.registry,
|
||||
bucket: bucket.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Abandon the acquisition (validation failed): deregister and release.
|
||||
pub(super) fn abandon(self, bucket: &str) {
|
||||
let lost = self.lock_lost();
|
||||
self.registry.exit(bucket, lost);
|
||||
drop(self.inner);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn uuid(n: u128) -> Uuid {
|
||||
Uuid::from_u128(n)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memo_valid_only_while_guards_overlap() {
|
||||
let reg = BucketFenceRegistry::default();
|
||||
|
||||
assert_eq!(reg.enter("b"), None, "first guard sees no memo");
|
||||
reg.memoize("b", uuid(1));
|
||||
assert_eq!(reg.enter("b"), Some(uuid(1)), "overlapping guard reuses memo");
|
||||
reg.exit("b", false);
|
||||
reg.exit("b", false);
|
||||
|
||||
// Coverage gap: all guards gone, memo must be dropped.
|
||||
assert_eq!(reg.enter("b"), None, "post-gap guard must revalidate");
|
||||
reg.exit("b", false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lost_lock_clears_memo_but_keeps_other_guards_registered() {
|
||||
let reg = BucketFenceRegistry::default();
|
||||
|
||||
assert_eq!(reg.enter("b"), None);
|
||||
reg.memoize("b", uuid(7));
|
||||
assert_eq!(reg.enter("b"), Some(uuid(7)));
|
||||
|
||||
// First guard exits reporting a lost lock: memo cleared even though
|
||||
// a second guard is still live.
|
||||
reg.exit("b", true);
|
||||
assert_eq!(reg.enter("b"), None, "memo not trusted after a lost lock");
|
||||
reg.exit("b", false);
|
||||
reg.exit("b", false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn buckets_are_isolated() {
|
||||
let reg = BucketFenceRegistry::default();
|
||||
assert_eq!(reg.enter("a"), None);
|
||||
reg.memoize("a", uuid(1));
|
||||
assert_eq!(reg.enter("b"), None, "memo does not leak across buckets");
|
||||
reg.exit("b", false);
|
||||
reg.exit("a", false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memoize_without_live_guard_is_ignored() {
|
||||
let reg = BucketFenceRegistry::default();
|
||||
reg.memoize("b", uuid(9));
|
||||
assert_eq!(reg.enter("b"), None);
|
||||
reg.exit("b", false);
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use tracing::trace;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_HEAL: &str = "heal";
|
||||
@@ -84,7 +83,7 @@ impl ECStore {
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -92,7 +91,7 @@ impl ECStore {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
trace!(
|
||||
info!(
|
||||
event = EVENT_HEAL_OBJECT_STARTED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
@@ -284,7 +283,6 @@ mod tests {
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
};
|
||||
|
||||
let (result, err) = store
|
||||
|
||||
@@ -419,7 +419,6 @@ impl ECStore {
|
||||
// legacy path) so startup writes (erasure type recorded before
|
||||
// this point) and later reads share one cell.
|
||||
ctx: instance_ctx.clone(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
});
|
||||
|
||||
// Only set it when this instance's deployment ID is not yet configured
|
||||
@@ -612,7 +611,6 @@ mod tests {
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_config::server_config::KVS;
|
||||
use rustfs_filemeta::ObjectPartInfo;
|
||||
#[cfg(feature = "test-util")]
|
||||
use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
|
||||
use std::{
|
||||
@@ -1155,145 +1153,6 @@ mod tests {
|
||||
(instance_ctx, store, shutdown)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn data_movement_conflicts_preserve_newer_target_and_abort_staging() {
|
||||
let temp_dir = tempfile::tempdir().expect("create data movement store dir");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "data-movement-conflict-convergence", &[4, 4]))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let bucket = format!("data-movement-conflict-{}", uuid::Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create data movement bucket");
|
||||
let source_mod_time = OffsetDateTime::UNIX_EPOCH;
|
||||
let target_mod_time = source_mod_time + time::Duration::SECOND;
|
||||
|
||||
let object = "single-object";
|
||||
let target_body = b"newer client body".to_vec();
|
||||
let mut target_reader = PutObjReader::from_vec(target_body.clone());
|
||||
store.pools[1]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut target_reader,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(target_mod_time),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write newer single-part target");
|
||||
|
||||
let source_body = b"stale migration body".to_vec();
|
||||
crate::data_movement::migrate_object(
|
||||
store.clone(),
|
||||
0,
|
||||
bucket.clone(),
|
||||
GetObjectReader {
|
||||
stream: Box::new(Cursor::new(source_body.clone())),
|
||||
object_info: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object.to_string(),
|
||||
size: i64::try_from(source_body.len()).expect("single source size should fit i64"),
|
||||
actual_size: i64::try_from(source_body.len()).expect("single source size should fit i64"),
|
||||
etag: Some("0123456789abcdef0123456789abcdef".to_string()),
|
||||
mod_time: Some(source_mod_time),
|
||||
..Default::default()
|
||||
},
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
},
|
||||
"test_data_movement",
|
||||
)
|
||||
.await
|
||||
.expect("newer single-part target should converge migration");
|
||||
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read converged single-part target");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("drain single-part target");
|
||||
assert_eq!(body, target_body);
|
||||
|
||||
let multipart_object = "multipart-object";
|
||||
let multipart_target_body = b"newer multipart client body".to_vec();
|
||||
let mut multipart_target_reader = PutObjReader::from_vec(multipart_target_body.clone());
|
||||
store.pools[1]
|
||||
.put_object(
|
||||
&bucket,
|
||||
multipart_object,
|
||||
&mut multipart_target_reader,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(target_mod_time),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write newer multipart target");
|
||||
|
||||
let first_part_size = 5 * 1024 * 1024;
|
||||
let mut multipart_source_body = vec![b'a'; first_part_size];
|
||||
multipart_source_body.push(b'b');
|
||||
let multipart_source_size = i64::try_from(multipart_source_body.len()).expect("multipart source size should fit i64");
|
||||
crate::data_movement::migrate_object(
|
||||
store.clone(),
|
||||
0,
|
||||
bucket.clone(),
|
||||
GetObjectReader {
|
||||
stream: Box::new(Cursor::new(multipart_source_body)),
|
||||
object_info: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: multipart_object.to_string(),
|
||||
size: multipart_source_size,
|
||||
actual_size: multipart_source_size,
|
||||
etag: Some("source-multipart-etag-2".to_string()),
|
||||
mod_time: Some(source_mod_time),
|
||||
parts: Arc::new(vec![
|
||||
ObjectPartInfo {
|
||||
number: 1,
|
||||
size: first_part_size,
|
||||
actual_size: i64::try_from(first_part_size).expect("first part size should fit i64"),
|
||||
etag: "source-part-1".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectPartInfo {
|
||||
number: 2,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
etag: "source-part-2".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
]),
|
||||
..Default::default()
|
||||
},
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
},
|
||||
"test_data_movement",
|
||||
)
|
||||
.await
|
||||
.expect("newer multipart target should converge migration");
|
||||
|
||||
let uploads = store.pools[1]
|
||||
.list_multipart_uploads(&bucket, multipart_object, None, None, None, 100)
|
||||
.await
|
||||
.expect("list target pool multipart uploads");
|
||||
assert!(uploads.uploads.is_empty(), "superseded migration staging must be aborted");
|
||||
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read converged multipart target");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("drain multipart target");
|
||||
assert_eq!(body, multipart_target_body);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn tier_delete_journal_count(store: Arc<crate::store::ECStore>) -> usize {
|
||||
store
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use super::*;
|
||||
|
||||
impl ECStore {
|
||||
#[instrument(level = "trace", skip(self))]
|
||||
#[instrument(skip(self))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn handle_list_objects_v2(
|
||||
self: Arc<Self>,
|
||||
|
||||
@@ -141,7 +141,6 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
|
||||
const MAX_UPLOADS_LIST: usize = 10000;
|
||||
|
||||
mod bucket;
|
||||
mod bucket_fence;
|
||||
pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
mod heal;
|
||||
mod heal_walk;
|
||||
@@ -194,9 +193,6 @@ pub struct ECStore {
|
||||
/// startup writes and post-construction reads share one cell — single
|
||||
/// instance behavior is unchanged.
|
||||
pub(crate) ctx: Arc<InstanceContext>,
|
||||
/// Memoizes bucket-incarnation validation under continuous lifecycle
|
||||
/// read-lock coverage (see [`bucket_fence`]).
|
||||
pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ECStore {
|
||||
@@ -586,7 +582,7 @@ impl crate::storage_api_contracts::list::ListOperations for ECStore {
|
||||
// @start_after as marker when continuation_token empty
|
||||
// @delimiter default="/", empty when recursive
|
||||
// @max_keys limit
|
||||
#[instrument(level = "trace", skip(self))]
|
||||
#[instrument(skip(self))]
|
||||
async fn list_objects_v2(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
@@ -791,7 +787,7 @@ impl crate::storage_api_contracts::heal::HealOperations for ECStore {
|
||||
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
|
||||
self.handle_heal_bucket(bucket, opts).await
|
||||
}
|
||||
#[instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
|
||||
#[instrument(skip(self))]
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -894,7 +890,6 @@ mod tests {
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(()),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -761,7 +761,6 @@ mod tests {
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1253,7 +1253,7 @@ impl ECStore {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip(self))]
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
|
||||
check_object_args(bucket, object)?;
|
||||
|
||||
@@ -3012,7 +3012,6 @@ mod tests {
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3053,7 +3052,6 @@ mod tests {
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(()),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -661,7 +661,7 @@ impl ECStore {
|
||||
unique_disks.into_values().collect()
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip(self))]
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
|
||||
self.pools[0].new_ns_lock(bucket, object).await
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ use rustfs_utils::HashAlgorithm;
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_FREE_VERSION, SUFFIX_HEALING, SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID,
|
||||
SUFFIX_TIER_FV_MARKER, SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str, has_internal_suffix, insert_str,
|
||||
is_encryption_metadata_key, starts_with_ignore_ascii_case,
|
||||
};
|
||||
use s3s::dto::{RestoreStatus, Timestamp};
|
||||
use s3s::header::X_AMZ_RESTORE;
|
||||
@@ -232,7 +231,7 @@ pub enum TransitionVersionState {
|
||||
Exact,
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Clone, Default)]
|
||||
#[derive(Debug, PartialEq, Clone, Default)]
|
||||
pub struct FileInfo {
|
||||
pub volume: String,
|
||||
pub name: String,
|
||||
@@ -272,117 +271,6 @@ pub struct FileInfo {
|
||||
pub uses_legacy_checksum: bool,
|
||||
}
|
||||
|
||||
/// Metadata keys whose values carry sealed encryption material (KEK-wrapped DEK,
|
||||
/// IV) under either the `x-rustfs-internal-` or `x-minio-internal-` prefix.
|
||||
/// Values of these keys must never reach logs at any level.
|
||||
fn is_sensitive_metadata_key(key: &str) -> bool {
|
||||
// `is_encryption_metadata_key` covers the x-minio-internal- SSE prefix but not
|
||||
// its x-rustfs-internal- twin, which the dual-key invariant writes alongside it.
|
||||
is_encryption_metadata_key(key) || starts_with_ignore_ascii_case(key, "x-rustfs-internal-server-side-encryption-")
|
||||
}
|
||||
|
||||
struct RedactedMetadata<'a>(&'a HashMap<String, String>);
|
||||
|
||||
impl std::fmt::Debug for RedactedMetadata<'_> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let mut map = f.debug_map();
|
||||
for (key, value) in self.0 {
|
||||
if is_sensitive_metadata_key(key) {
|
||||
map.entry(key, &format_args!("<redacted {} bytes>", value.len()));
|
||||
} else {
|
||||
map.entry(key, value);
|
||||
}
|
||||
}
|
||||
map.finish()
|
||||
}
|
||||
}
|
||||
|
||||
struct ElidedBytes<'a>(&'a Option<Bytes>);
|
||||
|
||||
impl std::fmt::Debug for ElidedBytes<'_> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self.0 {
|
||||
Some(bytes) => write!(f, "Some(<{} bytes elided>)", bytes.len()),
|
||||
None => f.write_str("None"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Manual Debug: `data` holds full inline object bytes (plaintext user content for
|
||||
// non-SSE objects) and `metadata` holds sealed key material — both must stay out
|
||||
// of Debug output so whole-struct log dumps cannot leak them. `checksum` is elided
|
||||
// too: for non-SSE objects it fingerprints plaintext content, and its raw bytes
|
||||
// carry no diagnostic value. The exhaustive destructuring (no `..`) forces every
|
||||
// future field through an explicit show/redact decision here.
|
||||
impl std::fmt::Debug for FileInfo {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let Self {
|
||||
volume,
|
||||
name,
|
||||
version_id,
|
||||
is_latest,
|
||||
deleted,
|
||||
transition_status,
|
||||
transitioned_objname,
|
||||
transition_tier,
|
||||
transition_version_id,
|
||||
transition_version,
|
||||
transition_version_state,
|
||||
expire_restored,
|
||||
data_dir,
|
||||
mod_time,
|
||||
size,
|
||||
mode,
|
||||
written_by_version,
|
||||
metadata,
|
||||
parts,
|
||||
erasure,
|
||||
mark_deleted,
|
||||
replication_state_internal,
|
||||
data,
|
||||
num_versions,
|
||||
successor_mod_time,
|
||||
fresh,
|
||||
idx,
|
||||
checksum,
|
||||
versioned,
|
||||
uses_legacy_checksum,
|
||||
} = self;
|
||||
f.debug_struct("FileInfo")
|
||||
.field("volume", volume)
|
||||
.field("name", name)
|
||||
.field("version_id", version_id)
|
||||
.field("is_latest", is_latest)
|
||||
.field("deleted", deleted)
|
||||
.field("transition_status", transition_status)
|
||||
.field("transitioned_objname", transitioned_objname)
|
||||
.field("transition_tier", transition_tier)
|
||||
.field("transition_version_id", transition_version_id)
|
||||
.field("transition_version", transition_version)
|
||||
.field("transition_version_state", transition_version_state)
|
||||
.field("expire_restored", expire_restored)
|
||||
.field("data_dir", data_dir)
|
||||
.field("mod_time", mod_time)
|
||||
.field("size", size)
|
||||
.field("mode", mode)
|
||||
.field("written_by_version", written_by_version)
|
||||
.field("metadata", &RedactedMetadata(metadata))
|
||||
.field("parts", parts)
|
||||
.field("erasure", erasure)
|
||||
.field("mark_deleted", mark_deleted)
|
||||
.field("replication_state_internal", replication_state_internal)
|
||||
.field("data", &ElidedBytes(data))
|
||||
.field("num_versions", num_versions)
|
||||
.field("successor_mod_time", successor_mod_time)
|
||||
.field("fresh", fresh)
|
||||
.field("idx", idx)
|
||||
.field("checksum", &ElidedBytes(checksum))
|
||||
.field("versioned", versioned)
|
||||
.field("uses_legacy_checksum", uses_legacy_checksum)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(remote = "FileInfo")]
|
||||
struct FileInfoMapDef {
|
||||
@@ -1111,12 +999,6 @@ impl FileInfo {
|
||||
insert_str(&mut self.metadata, SUFFIX_HEALING, "true".to_string());
|
||||
}
|
||||
|
||||
/// Reader for the marker [`Self::set_healing`] writes: true when this
|
||||
/// FileInfo is being committed by the heal path.
|
||||
pub fn is_healing(&self) -> bool {
|
||||
contains_key_str(&self.metadata, SUFFIX_HEALING)
|
||||
}
|
||||
|
||||
pub fn set_tier_free_version_id(&mut self, version_id: &str) {
|
||||
insert_str(&mut self.metadata, SUFFIX_TIER_FV_ID, version_id.to_string());
|
||||
}
|
||||
@@ -2528,54 +2410,4 @@ mod tests {
|
||||
};
|
||||
assert!(with_state.replication_info_equals(&with_state_clone));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_redacts_sealed_encryption_metadata_values() {
|
||||
let sealed_key = "IAAfANqt7wIJfVSgFAG3f5S6HuC2eyM5DdJlx7RSJKw2ZakSb3d5";
|
||||
let sealed_iv = "0Vr8QLGvQThk8gIWFCUnBOTUwZgs7TTBteRnAK9avD0=";
|
||||
let mut fi = FileInfo::default();
|
||||
for key in [
|
||||
"X-Rustfs-Internal-Server-Side-Encryption-Sealed-Key",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key",
|
||||
] {
|
||||
fi.metadata.insert(key.to_string(), sealed_key.to_string());
|
||||
}
|
||||
for key in [
|
||||
"X-Rustfs-Internal-Server-Side-Encryption-Iv",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv",
|
||||
"x-rustfs-encryption-iv",
|
||||
] {
|
||||
fi.metadata.insert(key.to_string(), sealed_iv.to_string());
|
||||
}
|
||||
fi.metadata.insert("content-type".to_string(), "text/plain".to_string());
|
||||
|
||||
let dump = format!("{fi:?}");
|
||||
assert!(!dump.contains(sealed_key), "sealed key leaked into Debug output: {dump}");
|
||||
assert!(!dump.contains(sealed_iv), "sealed IV leaked into Debug output: {dump}");
|
||||
// Keys stay visible so operators can still see which metadata is present.
|
||||
assert!(dump.contains("X-Rustfs-Internal-Server-Side-Encryption-Sealed-Key"));
|
||||
assert!(dump.contains(&format!("<redacted {} bytes>", sealed_key.len())));
|
||||
// Non-sensitive metadata values keep their diagnostic value.
|
||||
assert!(dump.contains("text/plain"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_elides_inline_data_bytes() {
|
||||
let fi = FileInfo {
|
||||
data: Some(Bytes::from_static(b"plaintext user object content")),
|
||||
checksum: Some(Bytes::from_static(b"\x01\x02checksumblob")),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let dump = format!("{fi:?}");
|
||||
assert!(
|
||||
!dump.contains("plaintext user object content"),
|
||||
"inline data leaked into Debug output: {dump}"
|
||||
);
|
||||
assert!(!dump.contains("checksumblob"), "checksum bytes leaked into Debug output: {dump}");
|
||||
assert!(dump.contains("data: Some(<29 bytes elided>)"), "missing data length summary: {dump}");
|
||||
|
||||
let empty = FileInfo::default();
|
||||
assert!(format!("{empty:?}").contains("data: None"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,17 +44,12 @@ impl FileMeta {
|
||||
}
|
||||
|
||||
pub fn check_xl2_v1(buf: &[u8]) -> Result<(&[u8], u16, u16)> {
|
||||
// A file too short to hold the XL2 magic, or one that carries the
|
||||
// wrong magic, is not merely unreadable — it is affirmative evidence
|
||||
// of a torn or foreign write. Classify it as FileCorrupt so quorum
|
||||
// and listing code can distinguish deterministic damage from
|
||||
// transient IO faults (issue #5716).
|
||||
if buf.len() < 8 {
|
||||
return Err(Error::FileCorrupt);
|
||||
return Err(Error::other("xl file header not exists"));
|
||||
}
|
||||
|
||||
if buf[0..4] != XL_FILE_HEADER {
|
||||
return Err(Error::FileCorrupt);
|
||||
return Err(Error::other("xl file header err"));
|
||||
}
|
||||
|
||||
let major = byteorder::LittleEndian::read_u16(&buf[4..6]);
|
||||
|
||||
@@ -48,14 +48,6 @@ pub const REPLICATE_HEAL: &str = "replicate:heal";
|
||||
pub const REPLICATE_HEAL_DELETE: &str = "replicate:heal:delete";
|
||||
|
||||
/// StatusType of Replication for x-amz-replication-status header
|
||||
///
|
||||
/// NOTE: `rustfs-replication` owns a sibling copy of this enum (plus
|
||||
/// `VersionPurgeStatusType` and `ReplicationState`) bound to the MRF/resync
|
||||
/// persistence format, while this copy is bound to the xl.meta disk format.
|
||||
/// When adding or renaming a variant here, reconcile the sibling and the
|
||||
/// conversion layer — the reconciliation tests in
|
||||
/// `crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs`
|
||||
/// fail to compile until both sides agree.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Hash)]
|
||||
pub enum ReplicationStatusType {
|
||||
/// Pending - replication is pending.
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::heal::{
|
||||
progress::HealProgress,
|
||||
resume::{CheckpointManager, ResumeManager, ResumeUtils, compose_key},
|
||||
storage::{HealStorageAPI, next_heal_listing_token},
|
||||
task::{demote_to_debug_when, is_missing_object_dir_heal_result, take_failure_log_sample},
|
||||
task::is_missing_object_dir_heal_result,
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use futures::{StreamExt, stream::FuturesUnordered};
|
||||
@@ -612,12 +612,6 @@ impl ErasureSetHealer {
|
||||
let page_concurrency_limit =
|
||||
Self::effective_heal_page_object_concurrency_for_source(self.source, self.heal_opts.scan_mode);
|
||||
let in_flight = Arc::new(AtomicUsize::new(0));
|
||||
// Per-bucket sample caps for per-object warn! lines: a flapping rebuild
|
||||
// disk can fail/skip hundreds of thousands of versions in one sweep, so
|
||||
// only the first few occurrences warn and the rest demote to debug!.
|
||||
// The end-of-pass summary reports the full failed/skipped counts.
|
||||
let mut transient_skip_samples_logged = 0_u64;
|
||||
let mut failure_samples_logged = 0_u64;
|
||||
|
||||
// backlog#920: select the per-erasure-set DISK-WALK union enumerator when
|
||||
// the scan is Deep OR the request came from AutoHeal — these are the paths
|
||||
@@ -754,7 +748,8 @@ impl ErasureSetHealer {
|
||||
Err(Error::TransientSkip { message }) => {
|
||||
*skipped_objects += 1;
|
||||
checkpoint_manager.add_skipped_object(key).await?;
|
||||
demote_to_debug_when!(!take_failure_log_sample(&mut transient_skip_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
|
||||
warn!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
@@ -765,12 +760,13 @@ impl ErasureSetHealer {
|
||||
state = "transient_skip",
|
||||
error = %message,
|
||||
"Erasure set object heal skipped due to transient error"
|
||||
});
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
*failed_objects += 1;
|
||||
checkpoint_manager.add_failed_object(key).await?;
|
||||
demote_to_debug_when!(!take_failure_log_sample(&mut failure_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
|
||||
warn!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
@@ -781,7 +777,7 @@ impl ErasureSetHealer {
|
||||
state = "failed",
|
||||
error = %err,
|
||||
"Erasure set object heal failed"
|
||||
});
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use crate::heal::{
|
||||
progress::{HealProgress, HealStatistics},
|
||||
storage::HealStorageAPI,
|
||||
task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType, demote_to_debug_when},
|
||||
task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType},
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use metrics::{counter, gauge};
|
||||
@@ -54,13 +54,6 @@ const EVENT_HEAL_UNCLEAN_SHUTDOWN: &str = "heal_unclean_shutdown";
|
||||
const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
|
||||
const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
|
||||
// Admission/scheduler outcomes for per-object requests (Object/Metadata/MRF/
|
||||
// ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner
|
||||
// recovery loops submit those per object, so a full queue or a retry storm
|
||||
// would otherwise emit one warn! per object (rustfs/rustfs#5716). The
|
||||
// `rustfs_heal_admission_total` metric and the `heal_queue_state` backlog
|
||||
// event keep the aggregate signal at operator-visible levels.
|
||||
|
||||
#[cfg(test)]
|
||||
struct RetryOwnershipTestHook {
|
||||
task_id: String,
|
||||
@@ -983,7 +976,6 @@ impl HealManager {
|
||||
let queue_len = queue.len();
|
||||
publish_heal_queue_length(queue);
|
||||
let queue_capacity = config.queue_size;
|
||||
let per_object_request = request.heal_type.is_per_object();
|
||||
|
||||
if queue_len >= queue_capacity && !request.force_start {
|
||||
if Self::can_displace_queued_work(&request) && queue.can_displace_lower_priority(request.priority) {
|
||||
@@ -993,7 +985,8 @@ impl HealManager {
|
||||
if let Some(displaced) = queue.push_displacing_lower_priority(request) {
|
||||
publish_heal_queue_length(queue);
|
||||
Self::record_admission_metric(source, HealAdmissionResult::Accepted, context);
|
||||
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
@@ -1007,11 +1000,12 @@ impl HealManager {
|
||||
queue_capacity,
|
||||
result = "accepted_by_displacement",
|
||||
"Heal queue request accepted by displacement"
|
||||
});
|
||||
);
|
||||
return HealAdmissionResult::Accepted;
|
||||
}
|
||||
|
||||
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
@@ -1023,7 +1017,7 @@ impl HealManager {
|
||||
queue_capacity,
|
||||
result = "full_no_displacement_candidate",
|
||||
"Heal queue request rejected without displacement"
|
||||
});
|
||||
);
|
||||
Self::record_admission_metric(source, HealAdmissionResult::Full, context);
|
||||
return HealAdmissionResult::Full;
|
||||
}
|
||||
@@ -1032,7 +1026,8 @@ impl HealManager {
|
||||
Self::record_admission_metric(request.source, admission, context);
|
||||
match admission {
|
||||
HealAdmissionResult::Dropped(reason) => {
|
||||
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
@@ -1045,10 +1040,11 @@ impl HealManager {
|
||||
reason = reason.as_str(),
|
||||
result = "dropped_full",
|
||||
"Heal queue request dropped"
|
||||
});
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Full => {
|
||||
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
@@ -1060,7 +1056,7 @@ impl HealManager {
|
||||
queue_capacity,
|
||||
result = "rejected_full",
|
||||
"Heal queue request rejected"
|
||||
});
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {}
|
||||
}
|
||||
@@ -1485,7 +1481,6 @@ impl HealManager {
|
||||
drop(retrying_heals);
|
||||
drop(queue);
|
||||
drop(active_heals);
|
||||
Self::record_admission_metric(request.source, admission, "duplicate");
|
||||
|
||||
match admission {
|
||||
HealAdmissionResult::Merged => {
|
||||
@@ -1506,7 +1501,8 @@ impl HealManager {
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Dropped(reason) => {
|
||||
demote_to_debug_when!(request.heal_type.is_per_object(), warn, target: "rustfs::heal::manager", {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
@@ -1516,7 +1512,7 @@ impl HealManager {
|
||||
duplicate_state,
|
||||
result = "dropped_duplicate",
|
||||
"Heal queue admission decided"
|
||||
});
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
|
||||
}
|
||||
@@ -2558,7 +2554,8 @@ impl HealManager {
|
||||
Err(e) => {
|
||||
let will_retry = retry_request.is_some();
|
||||
if will_retry {
|
||||
demote_to_debug_when!(task.heal_type.is_per_object(), warn, target: "rustfs::heal::manager", {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_SCHEDULER_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
@@ -2569,7 +2566,7 @@ impl HealManager {
|
||||
retry_attempt = task.retry_attempts.saturating_add(1),
|
||||
error = %e,
|
||||
"Heal scheduler task retrying"
|
||||
});
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
target: "rustfs::heal::manager",
|
||||
@@ -2672,7 +2669,7 @@ impl HealManager {
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = retry_cancel_token.cancelled() => {
|
||||
debug!(
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -2705,7 +2702,7 @@ impl HealManager {
|
||||
};
|
||||
if active_duplicate {
|
||||
retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
|
||||
debug!(
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -2733,7 +2730,7 @@ impl HealManager {
|
||||
retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
|
||||
drop(queue);
|
||||
retry_completed_heals.lock().await.remove(&retry_request_id);
|
||||
debug!(
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -2754,7 +2751,7 @@ impl HealManager {
|
||||
HealAdmissionResult::Merged => {
|
||||
retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
|
||||
drop(queue);
|
||||
debug!(
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -2768,11 +2765,7 @@ impl HealManager {
|
||||
return;
|
||||
}
|
||||
HealAdmissionResult::Full => {
|
||||
// admit_request_to_queue already logged the
|
||||
// rejection (context = "retry"); this repeats
|
||||
// every backoff cycle while the queue stays
|
||||
// full, so keep it at debug!.
|
||||
debug!(
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
@@ -3767,23 +3760,6 @@ mod tests {
|
||||
assert!(retry_error.contains("Lock acquisition timeout"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_retry_request_for_incomplete_heal_rename() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage);
|
||||
let result = Err(Error::TaskExecutionFailed {
|
||||
message: "Failed to heal object bucket/object: heal rename incomplete: 1 of 2 targets committed".to_string(),
|
||||
});
|
||||
|
||||
let (retry_request, retry_delay, retry_error) =
|
||||
retry_request_for_result(&task, &result).expect("incomplete target rename should be retryable");
|
||||
|
||||
assert_eq!(retry_request.id, task.id);
|
||||
assert_eq!(retry_request.retry_attempts, 1);
|
||||
assert!(retry_delay > Duration::ZERO);
|
||||
assert!(retry_error.contains("heal rename incomplete"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_retry_request_for_typed_read_quorum_error() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
|
||||
+17
-125
@@ -47,25 +47,6 @@ const MAX_RETAINED_HEAL_RESULT_ITEMS: usize = 1024;
|
||||
const EVENT_HEAL_OBJECT_RESULT: &str = "heal_object_result";
|
||||
const MAX_BUCKET_OBJECT_HEAL_RETRIES: u32 = 3;
|
||||
const MAX_BUCKET_FAILURE_LOG_SAMPLES: u64 = 5;
|
||||
|
||||
/// Emits at `$level`, demoted to `debug!` when `$demote` is true. Keeps
|
||||
/// per-object heal work — Object/Metadata/MRF/ECDecode tasks queued per
|
||||
/// object by MRF/autoheal/scanner loops, and per-object sweep failures past
|
||||
/// a sample cap — from amplifying into one info!/warn!/error! line per
|
||||
/// object during mass recovery (rustfs/rustfs#5716). Aggregate task kinds
|
||||
/// and foreground (admin/internal) requests keep operator-visible levels;
|
||||
/// metrics and end-of-sweep summaries carry the aggregate signal for the
|
||||
/// demoted paths.
|
||||
macro_rules! demote_to_debug_when {
|
||||
($demote:expr, $level:ident, target: $target:expr, { $($fields:tt)* }) => {
|
||||
if $demote {
|
||||
tracing::debug!(target: $target, $($fields)*);
|
||||
} else {
|
||||
tracing::$level!(target: $target, $($fields)*);
|
||||
}
|
||||
};
|
||||
}
|
||||
pub(crate) use demote_to_debug_when;
|
||||
const EVENT_HEAL_BUCKET_STAGE: &str = "heal_bucket_stage";
|
||||
const EVENT_HEAL_BUCKET_RESULT: &str = "heal_bucket_result";
|
||||
const EVENT_HEAL_METADATA_STAGE: &str = "heal_metadata_stage";
|
||||
@@ -119,18 +100,6 @@ impl HealType {
|
||||
Self::ECDecode { .. } => "ec_decode",
|
||||
}
|
||||
}
|
||||
|
||||
/// Task kinds enqueued at per-object granularity (MRF, autoheal, scanner,
|
||||
/// read-repair loops). Their lifecycle and admission logs stay at `debug!`
|
||||
/// so a recovery loop queuing hundreds of thousands of object heal tasks
|
||||
/// cannot amplify into per-object `info!`/`warn!` lines; aggregate kinds
|
||||
/// (cluster/bucket/prefix/erasure-set) keep operator-visible levels.
|
||||
pub(crate) fn is_per_object(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::Object { .. } | Self::Metadata { .. } | Self::MRF { .. } | Self::ECDecode { .. }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn is_object_level_not_found_error(err: &Error) -> bool {
|
||||
@@ -146,20 +115,6 @@ pub(crate) fn is_missing_object_dir_heal_result(object: &str, err: &Error) -> bo
|
||||
object.ends_with(SLASH_SEPARATOR) && is_object_level_not_found_error(err)
|
||||
}
|
||||
|
||||
/// Sample cap for per-object failure logs during a sweep: returns true (and
|
||||
/// consumes a sample slot) for the first [`MAX_BUCKET_FAILURE_LOG_SAMPLES`]
|
||||
/// calls, false afterwards so callers demote the remaining occurrences to
|
||||
/// `debug!`. Aggregate failed/skipped counts still surface in end-of-sweep
|
||||
/// summaries.
|
||||
pub(crate) fn take_failure_log_sample(samples_logged: &mut u64) -> bool {
|
||||
if *samples_logged < MAX_BUCKET_FAILURE_LOG_SAMPLES {
|
||||
*samples_logged = samples_logged.saturating_add(1);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Heal priority
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
pub enum HealPriority {
|
||||
@@ -663,7 +618,8 @@ impl HealTask {
|
||||
)
|
||||
.increment(1);
|
||||
|
||||
demote_to_debug_when!(self.heal_type.is_per_object(), info, target: "rustfs::heal::task", {
|
||||
info!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_TASK_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
@@ -672,7 +628,7 @@ impl HealTask {
|
||||
state = "started",
|
||||
queue_delay = ?queue_delay,
|
||||
"Heal task started"
|
||||
});
|
||||
);
|
||||
|
||||
let result = match &self.heal_type {
|
||||
HealType::Cluster => self.heal_cluster().await,
|
||||
@@ -704,7 +660,8 @@ impl HealTask {
|
||||
Ok(_) => {
|
||||
let mut status = self.status.write().await;
|
||||
*status = HealTaskStatus::Completed;
|
||||
demote_to_debug_when!(self.heal_type.is_per_object(), info, target: "rustfs::heal::task", {
|
||||
info!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_TASK_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
@@ -712,7 +669,7 @@ impl HealTask {
|
||||
heal_type = self.heal_type.log_kind(),
|
||||
state = "completed",
|
||||
"Heal task completed"
|
||||
});
|
||||
);
|
||||
}
|
||||
Err(Error::TaskCancelled) => {
|
||||
let mut status = self.status.write().await;
|
||||
@@ -731,7 +688,8 @@ impl HealTask {
|
||||
Err(Error::TaskTimeout) => {
|
||||
let mut status = self.status.write().await;
|
||||
*status = HealTaskStatus::Timeout;
|
||||
demote_to_debug_when!(self.heal_type.is_per_object(), warn, target: "rustfs::heal::task", {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_TASK_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
@@ -739,16 +697,13 @@ impl HealTask {
|
||||
heal_type = self.heal_type.log_kind(),
|
||||
state = "timed_out",
|
||||
"Heal task timed out"
|
||||
});
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
let mut status = self.status.write().await;
|
||||
*status = HealTaskStatus::Failed { error: e.to_string() };
|
||||
// Per-object failures are already logged with full object
|
||||
// context by the heal_* implementations and terminally by the
|
||||
// scheduler's task_failed error!; this generic duplicate would
|
||||
// multiply every failed object by the retry count.
|
||||
demote_to_debug_when!(self.heal_type.is_per_object(), error, target: "rustfs::heal::task", {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_TASK_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
@@ -757,7 +712,7 @@ impl HealTask {
|
||||
state = "failed",
|
||||
error = %e,
|
||||
"Heal task failed"
|
||||
});
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -875,21 +830,17 @@ impl HealTask {
|
||||
};
|
||||
|
||||
if !object_exists {
|
||||
// Background loops (scanner/MRF/autoheal/read-repair) routinely
|
||||
// race object deletion, so a missing target is per-object noise
|
||||
// for them; only foreground admin/internal requests keep the warn.
|
||||
let background_source = !matches!(self.source, HealRequestSource::Admin | HealRequestSource::Internal);
|
||||
demote_to_debug_when!(background_source, warn, target: "rustfs::heal::task", {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_MISSING,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
source = self.source.as_str(),
|
||||
recreate_missing = self.options.recreate_missing,
|
||||
"Heal target object is missing"
|
||||
});
|
||||
);
|
||||
if self.options.recreate_missing {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
@@ -1585,7 +1536,8 @@ impl HealTask {
|
||||
}
|
||||
first_failed_object.get_or_insert_with(|| object.to_string());
|
||||
first_error.get_or_insert_with(|| err.to_string());
|
||||
if take_failure_log_sample(&mut failure_samples_logged) {
|
||||
if failure_samples_logged < MAX_BUCKET_FAILURE_LOG_SAMPLES {
|
||||
failure_samples_logged = failure_samples_logged.saturating_add(1);
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
@@ -2444,66 +2396,6 @@ mod tests {
|
||||
resume_disk: Mutex<Option<DiskStore>>,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn per_object_heal_types_are_classified_for_log_demotion() {
|
||||
assert!(
|
||||
HealType::Object {
|
||||
bucket: "b".to_string(),
|
||||
object: "o".to_string(),
|
||||
version_id: None,
|
||||
}
|
||||
.is_per_object()
|
||||
);
|
||||
assert!(
|
||||
HealType::Metadata {
|
||||
bucket: "b".to_string(),
|
||||
object: "o".to_string(),
|
||||
}
|
||||
.is_per_object()
|
||||
);
|
||||
assert!(
|
||||
HealType::MRF {
|
||||
meta_path: "p".to_string(),
|
||||
}
|
||||
.is_per_object()
|
||||
);
|
||||
assert!(
|
||||
HealType::ECDecode {
|
||||
bucket: "b".to_string(),
|
||||
object: "o".to_string(),
|
||||
version_id: None,
|
||||
}
|
||||
.is_per_object()
|
||||
);
|
||||
assert!(!HealType::Cluster.is_per_object());
|
||||
assert!(!HealType::Bucket { bucket: "b".to_string() }.is_per_object());
|
||||
assert!(
|
||||
!HealType::Prefix {
|
||||
bucket: "b".to_string(),
|
||||
prefix: "p".to_string(),
|
||||
}
|
||||
.is_per_object()
|
||||
);
|
||||
assert!(
|
||||
!HealType::ErasureSet {
|
||||
buckets: Vec::new(),
|
||||
set_disk_id: "s".to_string(),
|
||||
}
|
||||
.is_per_object()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_log_sampling_caps_at_max_samples() {
|
||||
let mut samples_logged = 0_u64;
|
||||
for _ in 0..MAX_BUCKET_FAILURE_LOG_SAMPLES {
|
||||
assert!(take_failure_log_sample(&mut samples_logged));
|
||||
}
|
||||
assert!(!take_failure_log_sample(&mut samples_logged));
|
||||
assert!(!take_failure_log_sample(&mut samples_logged));
|
||||
assert_eq!(samples_logged, MAX_BUCKET_FAILURE_LOG_SAMPLES);
|
||||
}
|
||||
|
||||
/// Build a latest, non-delete-marker heal list item with no version id.
|
||||
fn heal_item(name: &str) -> HealListItem {
|
||||
HealListItem {
|
||||
|
||||
+1
-74
@@ -16,7 +16,7 @@ use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
ops::{Deref, DerefMut},
|
||||
ptr,
|
||||
sync::{Arc, Mutex, Weak},
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use arc_swap::{ArcSwap, Guard};
|
||||
@@ -65,29 +65,6 @@ pub struct Cache {
|
||||
state: ArcSwap<CacheState>,
|
||||
write_lock: Mutex<()>,
|
||||
service_account_mutation_lock: AsyncMutex<()>,
|
||||
sts_account_mutation_locks: Arc<StsMutationLockRegistry>,
|
||||
}
|
||||
|
||||
struct StsMutationLockRegistry {
|
||||
locks: Mutex<HashMap<String, Weak<AsyncMutex<StsMutationLockState>>>>,
|
||||
}
|
||||
|
||||
pub(crate) struct StsMutationLockState {
|
||||
access_key: String,
|
||||
registry: Weak<StsMutationLockRegistry>,
|
||||
lock: Weak<AsyncMutex<StsMutationLockState>>,
|
||||
}
|
||||
|
||||
impl Drop for StsMutationLockState {
|
||||
fn drop(&mut self) {
|
||||
let Some(registry) = self.registry.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let mut locks = registry.locks.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if locks.get(&self.access_key).is_some_and(|current| current.ptr_eq(&self.lock)) {
|
||||
locks.remove(&self.access_key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Cache {
|
||||
@@ -96,9 +73,6 @@ impl Default for Cache {
|
||||
state: ArcSwap::new(Arc::new(CacheState::default())),
|
||||
write_lock: Mutex::new(()),
|
||||
service_account_mutation_lock: AsyncMutex::new(()),
|
||||
sts_account_mutation_locks: Arc::new(StsMutationLockRegistry {
|
||||
locks: Mutex::new(HashMap::new()),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -110,29 +84,6 @@ impl Cache {
|
||||
&self.service_account_mutation_lock
|
||||
}
|
||||
|
||||
pub(crate) fn sts_account_mutation_lock(&self, access_key: &str) -> Arc<AsyncMutex<StsMutationLockState>> {
|
||||
let mut locks = self
|
||||
.sts_account_mutation_locks
|
||||
.locks
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(lock) = locks.get(access_key).and_then(Weak::upgrade) {
|
||||
return lock;
|
||||
}
|
||||
|
||||
let registry = Arc::downgrade(&self.sts_account_mutation_locks);
|
||||
let access_key_owned = access_key.to_string();
|
||||
let lock = Arc::new_cyclic(|lock| {
|
||||
AsyncMutex::new(StsMutationLockState {
|
||||
access_key: access_key_owned,
|
||||
registry,
|
||||
lock: lock.clone(),
|
||||
})
|
||||
});
|
||||
locks.insert(access_key.to_string(), Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
|
||||
pub(crate) fn snapshot(&self) -> CacheSnapshot {
|
||||
self.state.load()
|
||||
}
|
||||
@@ -494,30 +445,6 @@ mod tests {
|
||||
use crate::cache::Cache;
|
||||
use crate::store::MappedPolicy;
|
||||
|
||||
#[test]
|
||||
fn sts_mutation_locks_are_keyed_and_prune_unused_entries() {
|
||||
let cache = Cache::default();
|
||||
let first = cache.sts_account_mutation_lock("first");
|
||||
let same = cache.sts_account_mutation_lock("first");
|
||||
let different = cache.sts_account_mutation_lock("different");
|
||||
|
||||
assert!(Arc::ptr_eq(&first, &same));
|
||||
assert!(!Arc::ptr_eq(&first, &different));
|
||||
drop(first);
|
||||
drop(same);
|
||||
drop(different);
|
||||
|
||||
let _next = cache.sts_account_mutation_lock("next");
|
||||
let locks = cache
|
||||
.sts_account_mutation_locks
|
||||
.locks
|
||||
.lock()
|
||||
.expect("STS mutation lock registry mutex poisoned");
|
||||
assert!(!locks.contains_key("first"));
|
||||
assert!(!locks.contains_key("different"));
|
||||
assert!(locks.contains_key("next"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cache_entity_add() {
|
||||
let owner = Arc::new(Cache::default());
|
||||
|
||||
@@ -102,49 +102,7 @@ pub(crate) async fn notify_iam_delete_user(access_key: &str) -> Vec<IamNotificat
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct LoadUserNotificationProbe {
|
||||
pub(crate) observed: std::sync::Mutex<Option<(String, bool)>>,
|
||||
pub(crate) remaining_failures: std::sync::atomic::AtomicUsize,
|
||||
pub(crate) attempts: std::sync::atomic::AtomicUsize,
|
||||
pub(crate) panic: bool,
|
||||
pub(crate) started: tokio::sync::Notify,
|
||||
pub(crate) release: Option<tokio::sync::Notify>,
|
||||
pub(crate) completed: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
pub(crate) static LOAD_USER_NOTIFICATION_PROBE: std::sync::Arc<LoadUserNotificationProbe>;
|
||||
}
|
||||
|
||||
pub(crate) async fn notify_iam_load_user(access_key: &str, temp: bool) -> Vec<IamNotificationPeerErr> {
|
||||
#[cfg(test)]
|
||||
if let Ok(probe) = LOAD_USER_NOTIFICATION_PROBE.try_with(std::sync::Arc::clone) {
|
||||
probe.attempts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
*probe.observed.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some((access_key.to_string(), temp));
|
||||
probe.started.notify_one();
|
||||
if let Some(release) = &probe.release {
|
||||
release.notified().await;
|
||||
}
|
||||
assert!(!probe.panic, "notification probe panic");
|
||||
let should_fail = probe
|
||||
.remaining_failures
|
||||
.fetch_update(std::sync::atomic::Ordering::SeqCst, std::sync::atomic::Ordering::SeqCst, |remaining| {
|
||||
remaining.checked_sub(1)
|
||||
})
|
||||
.is_ok();
|
||||
let result = if should_fail {
|
||||
vec![IamNotificationPeerErr {
|
||||
err: Some(IamEcstoreError::other("peer notification failed")),
|
||||
}]
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
probe.completed.notify_one();
|
||||
return result;
|
||||
}
|
||||
|
||||
match runtime_sources::notification_sys() {
|
||||
Some(notification_sys) => notification_sys
|
||||
.load_user(access_key, temp)
|
||||
|
||||
+33
-198
@@ -293,8 +293,6 @@ where
|
||||
}
|
||||
|
||||
pub async fn load_user(&self, access_key: &str) -> Result<()> {
|
||||
let sts_mutation_lock = self.cache.sts_account_mutation_lock(access_key);
|
||||
let _sts_mutation_guard = sts_mutation_lock.lock().await;
|
||||
let mut users_map: HashMap<String, UserIdentity> = HashMap::new();
|
||||
let mut user_policy_map = HashMap::new();
|
||||
let mut sts_users_map = HashMap::new();
|
||||
@@ -1209,9 +1207,6 @@ where
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
|
||||
let mutation_lock = self.cache.sts_account_mutation_lock(access_key);
|
||||
let _mutation_guard = mutation_lock.lock().await;
|
||||
|
||||
let sts_policy_update = if let Some(policy) = policy_name {
|
||||
let mp = MappedPolicy::new(policy);
|
||||
let (_, combined_policy_stmt) = filter_policies(&self.cache, &mp.policies, "temp");
|
||||
@@ -1438,11 +1433,6 @@ where
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
|
||||
let sts_mutation_lock = (utype == UserType::Sts).then(|| self.cache.sts_account_mutation_lock(access_key));
|
||||
let _sts_mutation_guard = match &sts_mutation_lock {
|
||||
Some(lock) => Some(lock.lock().await),
|
||||
None => None,
|
||||
};
|
||||
let _service_account_guard = if utype == UserType::Svc {
|
||||
Some(self.cache.service_account_mutation_lock().lock().await)
|
||||
} else {
|
||||
@@ -1503,13 +1493,9 @@ where
|
||||
});
|
||||
}
|
||||
|
||||
if utype != UserType::Sts {
|
||||
let _ = self.api.delete_mapped_policy(access_key, utype, false).await;
|
||||
}
|
||||
let _ = self.api.delete_mapped_policy(access_key, utype, false).await;
|
||||
|
||||
if utype != UserType::Sts {
|
||||
self.cache.delete_user_policy(access_key, OffsetDateTime::now_utc());
|
||||
}
|
||||
self.cache.delete_user_policy(access_key, OffsetDateTime::now_utc());
|
||||
|
||||
if let Err(err) = self.api.delete_user_identity(access_key, utype).await
|
||||
&& !is_err_no_such_user(&err)
|
||||
@@ -1521,17 +1507,8 @@ where
|
||||
self.cache.with_write_lock(|cache| {
|
||||
if utype == UserType::Sts {
|
||||
cache.delete_sts_account(access_key, deleted_at);
|
||||
if cache
|
||||
.state()
|
||||
.users
|
||||
.get(access_key)
|
||||
.is_some_and(|identity| identity.credentials.is_temp())
|
||||
{
|
||||
cache.delete_user(access_key, deleted_at);
|
||||
}
|
||||
} else {
|
||||
cache.delete_user(access_key, deleted_at);
|
||||
}
|
||||
cache.delete_user(access_key, deleted_at);
|
||||
});
|
||||
|
||||
Ok(deleted_at)
|
||||
@@ -2055,11 +2032,6 @@ where
|
||||
Ok(())
|
||||
}
|
||||
pub async fn user_notification_handler(&self, name: &str, user_type: UserType) -> Result<()> {
|
||||
let sts_mutation_lock = (user_type == UserType::Sts).then(|| self.cache.sts_account_mutation_lock(name));
|
||||
let _sts_mutation_guard = match &sts_mutation_lock {
|
||||
Some(lock) => Some(lock.lock().await),
|
||||
None => None,
|
||||
};
|
||||
let _service_account_guard = if user_type == UserType::Svc {
|
||||
Some(self.cache.service_account_mutation_lock().lock().await)
|
||||
} else {
|
||||
@@ -2105,9 +2077,7 @@ where
|
||||
UserType::Reg | UserType::Svc => cache.delete_user(name, now),
|
||||
UserType::None => {}
|
||||
}
|
||||
if user_type != UserType::Sts {
|
||||
self.remove_user_from_cached_groups(cache, name, now);
|
||||
}
|
||||
self.remove_user_from_cached_groups(cache, name, now);
|
||||
if user_type == UserType::Reg {
|
||||
for access_key in service_accounts_to_delete.iter() {
|
||||
cache.delete_user(access_key, now);
|
||||
@@ -2117,9 +2087,7 @@ where
|
||||
cache.delete_user(access_key, now);
|
||||
}
|
||||
}
|
||||
if user_type != UserType::Sts {
|
||||
cache.delete_user_policy(name, now);
|
||||
}
|
||||
cache.delete_user_policy(name, now);
|
||||
});
|
||||
|
||||
return Ok(());
|
||||
@@ -2478,12 +2446,12 @@ mod tests {
|
||||
saved_user: Arc<Mutex<Option<UserIdentity>>>,
|
||||
load_attempts: Arc<AtomicUsize>,
|
||||
visible_after_attempt: usize,
|
||||
block_account_save: Arc<AtomicBool>,
|
||||
account_save_started: Arc<Notify>,
|
||||
release_account_save: Arc<Notify>,
|
||||
block_account_load: Arc<AtomicBool>,
|
||||
account_load_started: Arc<Notify>,
|
||||
release_account_load: Arc<Notify>,
|
||||
block_service_save: Arc<AtomicBool>,
|
||||
service_save_started: Arc<Notify>,
|
||||
release_service_save: Arc<Notify>,
|
||||
block_service_load: Arc<AtomicBool>,
|
||||
service_load_started: Arc<Notify>,
|
||||
release_service_load: Arc<Notify>,
|
||||
}
|
||||
|
||||
impl DelayedTempUserVisibilityStore {
|
||||
@@ -2492,12 +2460,12 @@ mod tests {
|
||||
saved_user: Arc::new(Mutex::new(None)),
|
||||
load_attempts: Arc::new(AtomicUsize::new(0)),
|
||||
visible_after_attempt,
|
||||
block_account_save: Arc::new(AtomicBool::new(false)),
|
||||
account_save_started: Arc::new(Notify::new()),
|
||||
release_account_save: Arc::new(Notify::new()),
|
||||
block_account_load: Arc::new(AtomicBool::new(false)),
|
||||
account_load_started: Arc::new(Notify::new()),
|
||||
release_account_load: Arc::new(Notify::new()),
|
||||
block_service_save: Arc::new(AtomicBool::new(false)),
|
||||
service_save_started: Arc::new(Notify::new()),
|
||||
release_service_save: Arc::new(Notify::new()),
|
||||
block_service_load: Arc::new(AtomicBool::new(false)),
|
||||
service_load_started: Arc::new(Notify::new()),
|
||||
release_service_load: Arc::new(Notify::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2527,9 +2495,9 @@ mod tests {
|
||||
item: UserIdentity,
|
||||
_ttl: Option<usize>,
|
||||
) -> Result<()> {
|
||||
if matches!(user_type, UserType::Svc | UserType::Sts) && self.block_account_save.load(Ordering::SeqCst) {
|
||||
self.account_save_started.notify_one();
|
||||
self.release_account_save.notified().await;
|
||||
if user_type == UserType::Svc && self.block_service_save.load(Ordering::SeqCst) {
|
||||
self.service_save_started.notify_one();
|
||||
self.release_service_save.notified().await;
|
||||
}
|
||||
*self.saved_user.lock().expect("saved_user mutex poisoned") = Some(item);
|
||||
Ok(())
|
||||
@@ -2560,18 +2528,9 @@ mod tests {
|
||||
.expect("saved_user mutex poisoned")
|
||||
.clone()
|
||||
.ok_or_else(|| Error::NoSuchUser(name.to_string()))?;
|
||||
let matches_user_type = match user_type {
|
||||
UserType::Sts => loaded.credentials.is_temp(),
|
||||
UserType::Svc => loaded.credentials.is_service_account(),
|
||||
UserType::Reg => !loaded.credentials.is_temp() && !loaded.credentials.is_service_account(),
|
||||
UserType::None => false,
|
||||
};
|
||||
if !matches_user_type {
|
||||
return Err(Error::NoSuchUser(name.to_string()));
|
||||
}
|
||||
if self.block_account_load.load(Ordering::SeqCst) {
|
||||
self.account_load_started.notify_one();
|
||||
self.release_account_load.notified().await;
|
||||
if user_type == UserType::Svc && self.block_service_load.load(Ordering::SeqCst) {
|
||||
self.service_load_started.notify_one();
|
||||
self.release_service_load.notified().await;
|
||||
}
|
||||
m.insert(name.to_string(), loaded);
|
||||
Ok(())
|
||||
@@ -2787,12 +2746,12 @@ mod tests {
|
||||
};
|
||||
cache.add_service_account(credentials).await.expect("seed service account");
|
||||
|
||||
store.block_account_load.store(true, Ordering::SeqCst);
|
||||
store.block_service_load.store(true, Ordering::SeqCst);
|
||||
let notification = {
|
||||
let cache = Arc::clone(&cache);
|
||||
tokio::spawn(async move { cache.user_notification_handler(access_key, UserType::Svc).await })
|
||||
};
|
||||
store.account_load_started.notified().await;
|
||||
store.service_load_started.notified().await;
|
||||
|
||||
let update = {
|
||||
let cache = Arc::clone(&cache);
|
||||
@@ -2817,7 +2776,7 @@ mod tests {
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!update.is_finished(), "update must wait for the in-flight cache refresh");
|
||||
|
||||
store.release_account_load.notify_one();
|
||||
store.release_service_load.notify_one();
|
||||
notification.await.expect("notification task").expect("notification refresh");
|
||||
update.await.expect("update task").expect("service account update");
|
||||
|
||||
@@ -2834,7 +2793,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn concurrent_service_account_create_cannot_overwrite_first_writer() {
|
||||
let store = DelayedTempUserVisibilityStore::new(0);
|
||||
store.block_account_save.store(true, Ordering::SeqCst);
|
||||
store.block_service_save.store(true, Ordering::SeqCst);
|
||||
let cache = Arc::new(build_test_iam_cache(store.clone()));
|
||||
let access_key = "SERIALIZEDSERVICE00";
|
||||
let credentials = |secret_key: &str| Credentials {
|
||||
@@ -2849,7 +2808,7 @@ mod tests {
|
||||
let cache = Arc::clone(&cache);
|
||||
tokio::spawn(async move { cache.add_service_account(credentials("firstServiceSecret123")).await })
|
||||
};
|
||||
store.account_save_started.notified().await;
|
||||
store.service_save_started.notified().await;
|
||||
|
||||
let second = {
|
||||
let cache = Arc::clone(&cache);
|
||||
@@ -2858,8 +2817,8 @@ mod tests {
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!second.is_finished(), "second create must wait for the first writer");
|
||||
|
||||
store.block_account_save.store(false, Ordering::SeqCst);
|
||||
store.release_account_save.notify_waiters();
|
||||
store.block_service_save.store(false, Ordering::SeqCst);
|
||||
store.release_service_save.notify_waiters();
|
||||
first.await.expect("first create task").expect("first create");
|
||||
let err = second
|
||||
.await
|
||||
@@ -2903,12 +2862,12 @@ mod tests {
|
||||
};
|
||||
cache.add_service_account(credentials).await.expect("seed service account");
|
||||
|
||||
store.block_account_load.store(true, Ordering::SeqCst);
|
||||
store.block_service_load.store(true, Ordering::SeqCst);
|
||||
let notification = {
|
||||
let cache = Arc::clone(&cache);
|
||||
tokio::spawn(async move { cache.user_notification_handler(access_key, UserType::Svc).await })
|
||||
};
|
||||
store.account_load_started.notified().await;
|
||||
store.service_load_started.notified().await;
|
||||
|
||||
let delete = {
|
||||
let cache = Arc::clone(&cache);
|
||||
@@ -2917,7 +2876,7 @@ mod tests {
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!delete.is_finished(), "delete must wait for the in-flight cache refresh");
|
||||
|
||||
store.release_account_load.notify_one();
|
||||
store.release_service_load.notify_one();
|
||||
notification.await.expect("notification task").expect("notification refresh");
|
||||
delete.await.expect("delete task").expect("service account delete");
|
||||
|
||||
@@ -2925,130 +2884,6 @@ mod tests {
|
||||
assert!(store.saved_user.lock().expect("saved_user mutex poisoned").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sts_notification_cannot_restore_concurrent_delete() {
|
||||
let store = DelayedTempUserVisibilityStore::new(0);
|
||||
let cache = Arc::new(build_test_iam_cache(store.clone()));
|
||||
let credentials = build_test_temp_credentials();
|
||||
let access_key = credentials.access_key.clone();
|
||||
cache
|
||||
.set_temp_user(&access_key, &credentials, None)
|
||||
.await
|
||||
.expect("seed temporary account");
|
||||
|
||||
store.block_account_load.store(true, Ordering::SeqCst);
|
||||
let notification = {
|
||||
let cache = Arc::clone(&cache);
|
||||
let access_key = access_key.clone();
|
||||
tokio::spawn(async move { cache.user_notification_handler(&access_key, UserType::Sts).await })
|
||||
};
|
||||
store.account_load_started.notified().await;
|
||||
|
||||
let delete_started = Arc::new(Notify::new());
|
||||
let delete = {
|
||||
let cache = Arc::clone(&cache);
|
||||
let access_key = access_key.clone();
|
||||
let delete_started = Arc::clone(&delete_started);
|
||||
tokio::spawn(async move {
|
||||
delete_started.notify_one();
|
||||
cache.delete_user(&access_key, UserType::Sts).await
|
||||
})
|
||||
};
|
||||
delete_started.notified().await;
|
||||
tokio::task::yield_now().await;
|
||||
let delete_waited_for_notification = !delete.is_finished();
|
||||
|
||||
store.release_account_load.notify_one();
|
||||
notification.await.expect("notification task").expect("notification refresh");
|
||||
delete.await.expect("delete task").expect("temporary account delete");
|
||||
|
||||
assert!(delete_waited_for_notification, "delete must wait for the in-flight STS cache refresh");
|
||||
assert!(!cache.cache.snapshot().sts_accounts.contains_key(&access_key));
|
||||
assert!(store.saved_user.lock().expect("saved_user mutex poisoned").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sts_auth_reload_cannot_restore_concurrent_delete() {
|
||||
let store = DelayedTempUserVisibilityStore::new(0);
|
||||
let cache = Arc::new(build_test_iam_cache(store.clone()));
|
||||
let credentials = build_test_temp_credentials();
|
||||
let access_key = credentials.access_key.clone();
|
||||
cache
|
||||
.set_temp_user(&access_key, &credentials, None)
|
||||
.await
|
||||
.expect("seed temporary account");
|
||||
cache.cache.delete_sts_account(&access_key, OffsetDateTime::now_utc());
|
||||
|
||||
store.block_account_load.store(true, Ordering::SeqCst);
|
||||
let reload = {
|
||||
let cache = Arc::clone(&cache);
|
||||
let access_key = access_key.clone();
|
||||
tokio::spawn(async move { cache.load_user(&access_key).await })
|
||||
};
|
||||
store.account_load_started.notified().await;
|
||||
|
||||
let delete_started = Arc::new(Notify::new());
|
||||
let delete = {
|
||||
let cache = Arc::clone(&cache);
|
||||
let access_key = access_key.clone();
|
||||
let delete_started = Arc::clone(&delete_started);
|
||||
tokio::spawn(async move {
|
||||
delete_started.notify_one();
|
||||
cache.delete_user(&access_key, UserType::Sts).await
|
||||
})
|
||||
};
|
||||
delete_started.notified().await;
|
||||
tokio::task::yield_now().await;
|
||||
let delete_waited_for_reload = !delete.is_finished();
|
||||
|
||||
store.release_account_load.notify_one();
|
||||
reload.await.expect("reload task").expect("authentication cache reload");
|
||||
delete.await.expect("delete task").expect("temporary account delete");
|
||||
|
||||
assert!(delete_waited_for_reload, "delete must wait for the in-flight authentication reload");
|
||||
assert!(!cache.cache.snapshot().sts_accounts.contains_key(&access_key));
|
||||
assert!(store.saved_user.lock().expect("saved_user mutex poisoned").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sts_create_cannot_restore_concurrent_delete() {
|
||||
let store = DelayedTempUserVisibilityStore::new(0);
|
||||
store.block_account_save.store(true, Ordering::SeqCst);
|
||||
let cache = Arc::new(build_test_iam_cache(store.clone()));
|
||||
let credentials = build_test_temp_credentials();
|
||||
let access_key = credentials.access_key.clone();
|
||||
|
||||
let create = {
|
||||
let cache = Arc::clone(&cache);
|
||||
let access_key = access_key.clone();
|
||||
tokio::spawn(async move { cache.set_temp_user(&access_key, &credentials, None).await })
|
||||
};
|
||||
store.account_save_started.notified().await;
|
||||
|
||||
let delete_started = Arc::new(Notify::new());
|
||||
let delete = {
|
||||
let cache = Arc::clone(&cache);
|
||||
let access_key = access_key.clone();
|
||||
let delete_started = Arc::clone(&delete_started);
|
||||
tokio::spawn(async move {
|
||||
delete_started.notify_one();
|
||||
cache.delete_user(&access_key, UserType::Sts).await
|
||||
})
|
||||
};
|
||||
delete_started.notified().await;
|
||||
tokio::task::yield_now().await;
|
||||
let delete_waited_for_create = !delete.is_finished();
|
||||
|
||||
store.block_account_save.store(false, Ordering::SeqCst);
|
||||
store.release_account_save.notify_one();
|
||||
create.await.expect("create task").expect("temporary account create");
|
||||
delete.await.expect("delete task").expect("temporary account delete");
|
||||
|
||||
assert!(delete_waited_for_create, "delete must wait for the in-flight STS create");
|
||||
assert!(!cache.cache.snapshot().sts_accounts.contains_key(&access_key));
|
||||
assert!(store.saved_user.lock().expect("saved_user mutex poisoned").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_init_keeps_error_state_when_initial_load_fails() {
|
||||
let (sender, receiver) = mpsc::channel::<i64>(1);
|
||||
|
||||
+1
-119
@@ -139,59 +139,6 @@ impl UserType {
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode a [`UserType`] as the site-replication wire value for
|
||||
/// `SRPolicyMapping.userType` / `SRCredInfo.iamUserType`.
|
||||
///
|
||||
/// The wire uses MinIO's `IAMUserType` table (cmd/iam.go):
|
||||
///
|
||||
/// | wire | MinIO meaning |
|
||||
/// |------|---------------|
|
||||
/// | -1 | unknown |
|
||||
/// | 0 | regUser |
|
||||
/// | 1 | stsUser |
|
||||
/// | 2 | svcUser |
|
||||
///
|
||||
/// This is deliberately distinct from the internal encoding
|
||||
/// [`UserType::to_u64`]/[`UserType::from_u64`] (None=0, Svc=1, Sts=2, Reg=3),
|
||||
/// which is used by intra-cluster node RPC and must never change (a rolling
|
||||
/// restart mixes old and new nodes on that RPC). Do not "unify" the two
|
||||
/// tables: internal values on the SR wire mislabel users on MinIO peers.
|
||||
///
|
||||
/// Group mappings always encode as 0: MinIO routes group mappings by the
|
||||
/// `isGroup` flag (userType is effectively ignored), and pre-fix RustFS peers
|
||||
/// sent 0 for groups, so 0 is the one value every peer generation accepts.
|
||||
pub fn sr_wire_user_type(user_type: UserType, is_group: bool) -> i64 {
|
||||
if is_group {
|
||||
return 0;
|
||||
}
|
||||
match user_type {
|
||||
UserType::Reg | UserType::None => 0,
|
||||
UserType::Sts => 1,
|
||||
UserType::Svc => 2,
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode a site-replication wire `userType` value (see [`sr_wire_user_type`]
|
||||
/// for the table) into a [`UserType`].
|
||||
///
|
||||
/// - `-1` (MinIO unknown, sent for group mappings) maps to [`UserType::None`];
|
||||
/// `policy_db_set` routes group items by `is_group`, and for non-group items
|
||||
/// `None` shares the users prefix with `Reg`.
|
||||
/// - `3` is a permanent alias for [`UserType::Reg`]: pre-fix RustFS peers sent
|
||||
/// the internal encoding (`Reg.to_u64() == 3`) on the wire. Keep it forever
|
||||
/// for mixed-version site replication; do not remove.
|
||||
/// - Anything else is unknown and rejected (`None`), so callers fail closed.
|
||||
pub fn user_type_from_sr_wire(v: i64) -> Option<UserType> {
|
||||
match v {
|
||||
-1 => Some(UserType::None),
|
||||
0 => Some(UserType::Reg),
|
||||
1 => Some(UserType::Sts),
|
||||
2 => Some(UserType::Svc),
|
||||
3 => Some(UserType::Reg),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
pub struct MappedPolicy {
|
||||
pub version: i64,
|
||||
@@ -267,72 +214,7 @@ impl GroupInfo {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{GroupInfo, MappedPolicy, UserType, sr_wire_user_type, user_type_from_sr_wire};
|
||||
|
||||
/// Site-replication inbound decode of `SRPolicyMapping.userType` must
|
||||
/// follow MinIO IAMUserType wire semantics (cmd/iam.go): stsUser = 1.
|
||||
/// The internal `UserType::from_u64` table maps 1 to Svc — reusing it at
|
||||
/// the SR boundary lands federated STS mappings under the wrong prefix
|
||||
/// and silently drops their effect.
|
||||
#[test]
|
||||
fn sr_inbound_decodes_minio_sts_wire_value_as_sts() {
|
||||
assert_eq!(user_type_from_sr_wire(1), Some(UserType::Sts));
|
||||
}
|
||||
|
||||
/// Wire-constant contract: literal MinIO IAMUserType values (cmd/iam.go).
|
||||
/// WARNING: these literals are the cross-vendor wire format. Never "tidy"
|
||||
/// them to match `UserType::to_u64`/`from_u64` — that internal table
|
||||
/// (None=0, Svc=1, Sts=2, Reg=3) belongs to intra-cluster node RPC only.
|
||||
#[test]
|
||||
fn sr_wire_decode_matches_minio_iam_user_type_table() {
|
||||
assert_eq!(user_type_from_sr_wire(-1), Some(UserType::None)); // MinIO unknown (group mappings)
|
||||
assert_eq!(user_type_from_sr_wire(0), Some(UserType::Reg)); // MinIO regUser
|
||||
assert_eq!(user_type_from_sr_wire(1), Some(UserType::Sts)); // MinIO stsUser
|
||||
assert_eq!(user_type_from_sr_wire(2), Some(UserType::Svc)); // MinIO svcUser
|
||||
// Permanent alias: pre-fix RustFS peers sent internal Reg=3 on the wire.
|
||||
assert_eq!(user_type_from_sr_wire(3), Some(UserType::Reg));
|
||||
// Unknown values fail closed.
|
||||
assert_eq!(user_type_from_sr_wire(4), None);
|
||||
assert_eq!(user_type_from_sr_wire(-2), None);
|
||||
}
|
||||
|
||||
/// Wire-constant contract for the outbound direction.
|
||||
#[test]
|
||||
fn sr_wire_encode_matches_minio_iam_user_type_table() {
|
||||
assert_eq!(sr_wire_user_type(UserType::Reg, false), 0); // MinIO regUser
|
||||
assert_eq!(sr_wire_user_type(UserType::Sts, false), 1); // MinIO stsUser
|
||||
assert_eq!(sr_wire_user_type(UserType::Svc, false), 2); // MinIO svcUser
|
||||
assert_eq!(sr_wire_user_type(UserType::None, false), 0);
|
||||
// Group mappings always go out as 0 — the value both MinIO (routes by
|
||||
// isGroup) and pre-fix RustFS peers accept.
|
||||
for ut in [UserType::Reg, UserType::Sts, UserType::Svc, UserType::None] {
|
||||
assert_eq!(sr_wire_user_type(ut, true), 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// Mixed-version matrix: every value a peer generation can emit decodes to
|
||||
/// a `UserType` the receiver stores correctly.
|
||||
#[test]
|
||||
fn sr_wire_round_trip_covers_old_rustfs_and_minio_peers() {
|
||||
// Old RustFS outbound: user mappings as internal Reg=3, groups as 0.
|
||||
assert_eq!(user_type_from_sr_wire(3), Some(UserType::Reg));
|
||||
assert_eq!(user_type_from_sr_wire(0), Some(UserType::Reg));
|
||||
// New RustFS outbound decodes on its own kind (self round-trip).
|
||||
for (ut, is_group) in [
|
||||
(UserType::Reg, false),
|
||||
(UserType::Sts, false),
|
||||
(UserType::Svc, false),
|
||||
(UserType::None, true),
|
||||
] {
|
||||
assert!(user_type_from_sr_wire(sr_wire_user_type(ut, is_group)).is_some());
|
||||
}
|
||||
// Internal RPC encoding is untouched (rolling-restart contract).
|
||||
assert_eq!(UserType::None.to_u64(), 0);
|
||||
assert_eq!(UserType::Svc.to_u64(), 1);
|
||||
assert_eq!(UserType::Sts.to_u64(), 2);
|
||||
assert_eq!(UserType::Reg.to_u64(), 3);
|
||||
assert_eq!(UserType::from_u64(1), Some(UserType::Svc));
|
||||
}
|
||||
use super::{GroupInfo, MappedPolicy};
|
||||
|
||||
/// uses RFC3339 for updatedAt. MappedPolicy must serialize as RFC3339.
|
||||
#[test]
|
||||
|
||||
@@ -28,10 +28,7 @@ use crate::{
|
||||
use futures::future::join_all;
|
||||
use rustfs_io_metrics::record_system_path_failure;
|
||||
use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
|
||||
use rustfs_utils::{
|
||||
MaskedAccessKey,
|
||||
path::{SLASH_SEPARATOR, path_join_buf},
|
||||
};
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::sync::{LazyLock, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -603,10 +600,10 @@ impl ObjectStore {
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if is_err_config_not_found(&err) {
|
||||
debug!(name = %MaskedAccessKey(name), user_type = ?user_type, "IAM user identity missing");
|
||||
warn!(name, user_type = ?user_type, "IAM user identity missing");
|
||||
Error::NoSuchUser(name.to_owned())
|
||||
} else {
|
||||
warn!(name = %MaskedAccessKey(name), user_type = ?user_type, error = ?err, "IAM user identity load failed");
|
||||
warn!(name, user_type = ?user_type, error = ?err, "IAM user identity load failed");
|
||||
err
|
||||
}
|
||||
})?;
|
||||
@@ -614,7 +611,7 @@ impl ObjectStore {
|
||||
if u.credentials.is_expired() {
|
||||
let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
|
||||
let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
|
||||
warn!(name = %MaskedAccessKey(name), user_type = ?user_type, "IAM user identity expired and was removed");
|
||||
warn!(name, user_type = ?user_type, "IAM user identity expired and was removed");
|
||||
return Err(Error::NoSuchUser(name.to_owned()));
|
||||
}
|
||||
|
||||
@@ -638,7 +635,7 @@ impl ObjectStore {
|
||||
let _ = self.delete_iam_config(get_user_identity_path(name, user_type)).await;
|
||||
let _ = self.delete_iam_config(get_mapped_policy_path(name, user_type, false)).await;
|
||||
}
|
||||
warn!(name = %MaskedAccessKey(name), user_type = ?user_type, error = ?err, "IAM JWT claim extraction failed");
|
||||
warn!(name, user_type = ?user_type, error = ?err, "IAM JWT claim extraction failed");
|
||||
return Err(Error::NoSuchUser(name.to_owned()));
|
||||
}
|
||||
}
|
||||
@@ -876,7 +873,13 @@ impl Store for ObjectStore {
|
||||
async fn delete_user_identity(&self, name: &str, user_type: UserType) -> Result<()> {
|
||||
self.delete_iam_config(get_user_identity_path(name, user_type))
|
||||
.await
|
||||
.map_err(|err| map_delete_user_identity_error(name, err))?;
|
||||
.map_err(|err| {
|
||||
if is_err_config_not_found(&err) {
|
||||
Error::NoSuchPolicy
|
||||
} else {
|
||||
err
|
||||
}
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
async fn load_user_identity(&self, name: &str, user_type: UserType) -> Result<UserIdentity> {
|
||||
@@ -1324,18 +1327,9 @@ impl Store for ObjectStore {
|
||||
}
|
||||
}
|
||||
|
||||
fn map_delete_user_identity_error(name: &str, err: Error) -> Error {
|
||||
if is_err_config_not_found(&err) {
|
||||
Error::NoSuchUser(name.to_owned())
|
||||
} else {
|
||||
err
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{DecryptSource, LoadMode, ObjectStore, map_delete_user_identity_error};
|
||||
use crate::error::Error;
|
||||
use super::{DecryptSource, LoadMode, ObjectStore};
|
||||
use crate::keyring;
|
||||
use rustfs_credentials::{Credentials, init_global_action_credentials};
|
||||
use serial_test::serial;
|
||||
@@ -1358,12 +1352,6 @@ mod tests {
|
||||
assert!(!LoadMode::Locked.read_opts().no_lock);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_user_identity_delete_maps_to_no_such_user() {
|
||||
let err = map_delete_user_identity_error("missing-sts", Error::ConfigNotFound);
|
||||
assert!(matches!(err, Error::NoSuchUser(name) if name == "missing-sts"));
|
||||
}
|
||||
|
||||
fn test_cred() -> Credentials {
|
||||
if let Some(cred) = crate::root_credentials::credentials() {
|
||||
return cred;
|
||||
|
||||
+40
-563
@@ -48,16 +48,6 @@ use time::OffsetDateTime;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
#[cfg(not(test))]
|
||||
const STS_INVALIDATION_RETRY_INITIAL_DELAY: std::time::Duration = std::time::Duration::from_secs(1);
|
||||
#[cfg(test)]
|
||||
const STS_INVALIDATION_RETRY_INITIAL_DELAY: std::time::Duration = std::time::Duration::from_millis(1);
|
||||
const STS_INVALIDATION_MAX_ATTEMPTS: usize = 3;
|
||||
#[cfg(not(test))]
|
||||
const STS_INVALIDATION_ATTEMPT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
#[cfg(test)]
|
||||
const STS_INVALIDATION_ATTEMPT_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(20);
|
||||
|
||||
pub const MAX_SVCSESSION_POLICY_SIZE: usize = 4096;
|
||||
pub const SITE_REPLICATOR_SERVICE_ACCOUNT: &str = "site-replicator-0";
|
||||
|
||||
@@ -79,42 +69,6 @@ enum PolicyPluginState {
|
||||
Failed,
|
||||
}
|
||||
|
||||
impl PolicyPluginState {
|
||||
fn prepared_iam_auth(&self) -> Option<PreparedIamAuth> {
|
||||
match self {
|
||||
Self::Ready(_) => Some(PreparedIamAuth {
|
||||
needs_existing_object_tag: true,
|
||||
mode: PreparedIamMode::Opa,
|
||||
}),
|
||||
Self::Initializing | Self::Failed => Some(PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
}),
|
||||
Self::Disabled => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn resolve_policy_plugin_state() -> PolicyPluginState {
|
||||
match opa::lookup_config().await {
|
||||
Ok(conf) if conf.enable() => {
|
||||
info!("OPA plugin enabled");
|
||||
PolicyPluginState::Ready(opa::AuthZPlugin::new(conf))
|
||||
}
|
||||
Ok(_) => PolicyPluginState::Failed,
|
||||
Err(e) => {
|
||||
error!(
|
||||
component = "iam",
|
||||
subsystem = "policy_plugin",
|
||||
result = "configuration_load_failed",
|
||||
error_kind = e.kind(),
|
||||
"OPA plugin configuration load failed"
|
||||
);
|
||||
PolicyPluginState::Failed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static POLICY_PLUGIN_STATE: OnceLock<Arc<RwLock<PolicyPluginState>>> = OnceLock::new();
|
||||
|
||||
fn get_policy_plugin_state() -> Arc<RwLock<PolicyPluginState>> {
|
||||
@@ -129,7 +83,23 @@ fn get_policy_plugin_state() -> Arc<RwLock<PolicyPluginState>> {
|
||||
if configured {
|
||||
let state = Arc::clone(&state);
|
||||
tokio::spawn(async move {
|
||||
let next_state = resolve_policy_plugin_state().await;
|
||||
let next_state = match opa::lookup_config().await {
|
||||
Ok(conf) if conf.enable() => {
|
||||
info!("OPA plugin enabled");
|
||||
PolicyPluginState::Ready(opa::AuthZPlugin::new(conf))
|
||||
}
|
||||
Ok(_) => PolicyPluginState::Failed,
|
||||
Err(e) => {
|
||||
error!(
|
||||
component = "iam",
|
||||
subsystem = "policy_plugin",
|
||||
result = "configuration_load_failed",
|
||||
error_kind = e.kind(),
|
||||
"OPA plugin configuration load failed"
|
||||
);
|
||||
PolicyPluginState::Failed
|
||||
}
|
||||
};
|
||||
*state.write().await = next_state;
|
||||
});
|
||||
}
|
||||
@@ -423,74 +393,17 @@ impl<T: Store> IamSys<T> {
|
||||
/// associated session token. This is the primitive used by the admin
|
||||
/// `revoke-tokens` endpoint to revoke STS credentials for a parent user.
|
||||
pub async fn delete_temp_account(&self, access_key: &str, notify: bool) -> Result<()> {
|
||||
if !notify || self.has_watcher() {
|
||||
return self.store.delete_user(access_key, UserType::Sts).await;
|
||||
}
|
||||
self.store.delete_user(access_key, UserType::Sts).await?;
|
||||
|
||||
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
|
||||
#[cfg(test)]
|
||||
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
|
||||
#[cfg(test)]
|
||||
let notification_available = notification_probe.is_some() || crate::runtime_sources::notification_sys().is_some();
|
||||
#[cfg(not(test))]
|
||||
let notification_available = crate::runtime_sources::notification_sys().is_some();
|
||||
if !notification_available {
|
||||
return Err(Error::other("IAM peer notification system is unavailable"));
|
||||
}
|
||||
|
||||
let store = Arc::clone(&self.store);
|
||||
let access_key = access_key.to_string();
|
||||
|
||||
let operation = async move {
|
||||
store.delete_user(&access_key, UserType::Sts).await?;
|
||||
|
||||
let mut delay = STS_INVALIDATION_RETRY_INITIAL_DELAY;
|
||||
for attempt in 1..=STS_INVALIDATION_MAX_ATTEMPTS {
|
||||
let attempt_error =
|
||||
match tokio::time::timeout(STS_INVALIDATION_ATTEMPT_TIMEOUT, notify_iam_load_user(&access_key, true)).await {
|
||||
Ok(results) => results.into_iter().find_map(|result| result.err).map(Error::other),
|
||||
Err(_) => Some(Error::other("peer STS invalidation timed out")),
|
||||
};
|
||||
let Some(err) = attempt_error else {
|
||||
return Ok(());
|
||||
};
|
||||
if attempt == STS_INVALIDATION_MAX_ATTEMPTS {
|
||||
return Err(Error::other(err));
|
||||
if notify && !self.has_watcher() {
|
||||
for r in notify_iam_delete_user(access_key).await {
|
||||
if let Some(err) = r.err {
|
||||
warn!("notify delete_temp_account failed: {}", err);
|
||||
}
|
||||
tokio::time::sleep(delay).await;
|
||||
delay = delay.saturating_mul(2);
|
||||
}
|
||||
unreachable!("STS invalidation retry loop always returns")
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
let task = runtime.spawn(async move {
|
||||
if let Some(probe) = notification_probe {
|
||||
return crate::LOAD_USER_NOTIFICATION_PROBE.scope(probe, operation).await;
|
||||
}
|
||||
operation.await
|
||||
});
|
||||
#[cfg(not(test))]
|
||||
let task = runtime.spawn(operation);
|
||||
|
||||
task.await.map_err(Error::other)?
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn load_user_notification_probe(
|
||||
failures_before_success: usize,
|
||||
block: bool,
|
||||
panic: bool,
|
||||
) -> Arc<crate::LoadUserNotificationProbe> {
|
||||
Arc::new(crate::LoadUserNotificationProbe {
|
||||
observed: std::sync::Mutex::new(None),
|
||||
remaining_failures: std::sync::atomic::AtomicUsize::new(failures_before_success),
|
||||
attempts: std::sync::atomic::AtomicUsize::new(0),
|
||||
panic,
|
||||
started: tokio::sync::Notify::new(),
|
||||
release: block.then(tokio::sync::Notify::new),
|
||||
completed: tokio::sync::Notify::new(),
|
||||
})
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn notify_for_user(&self, name: &str, is_temp: bool) {
|
||||
@@ -1184,8 +1097,20 @@ impl<T: Store> IamSys<T> {
|
||||
};
|
||||
}
|
||||
|
||||
if let Some(prepared) = Self::policy_plugin_state().await.prepared_iam_auth() {
|
||||
return prepared;
|
||||
match Self::policy_plugin_state().await {
|
||||
PolicyPluginState::Ready(_) => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: true,
|
||||
mode: PreparedIamMode::Opa,
|
||||
};
|
||||
}
|
||||
PolicyPluginState::Initializing | PolicyPluginState::Failed => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
PolicyPluginState::Disabled => {}
|
||||
}
|
||||
|
||||
let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
|
||||
@@ -1848,8 +1773,6 @@ mod tests {
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
#[test]
|
||||
fn test_combined_policy_for_view_returns_regular_policy() {
|
||||
@@ -1912,74 +1835,6 @@ mod tests {
|
||||
assert!(needs_secondary_tags, "OPA mode must request existing object tags for secondary actions");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_prepare_auth_denies_while_policy_plugin_is_unavailable() {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let iam_sys = IamSys::new(IamCache::new(store).await.expect("initialize IAM cache"));
|
||||
let claims = HashMap::new();
|
||||
let groups = None;
|
||||
let conditions = HashMap::new();
|
||||
let args = Args {
|
||||
account: "opa-unavailable-test-user",
|
||||
groups: &groups,
|
||||
action: Action::S3Action(S3Action::ListAllMyBucketsAction),
|
||||
bucket: "",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
object: "",
|
||||
claims: &claims,
|
||||
deny_only: false,
|
||||
};
|
||||
|
||||
let mut outcomes = Vec::new();
|
||||
for state in [PolicyPluginState::Initializing, PolicyPluginState::Failed] {
|
||||
let prepared = state
|
||||
.prepared_iam_auth()
|
||||
.expect("unavailable policy plugin must prepare fail-closed IAM auth");
|
||||
outcomes.push((
|
||||
matches!(&prepared.mode, PreparedIamMode::Deny),
|
||||
iam_sys.eval_prepared(&prepared, &args).await,
|
||||
));
|
||||
}
|
||||
|
||||
assert_eq!(outcomes, [(true, false), (true, false)]);
|
||||
assert!(PolicyPluginState::Disabled.prepared_iam_auth().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_policy_plugin_state_fails_after_opa_validation_returns_503() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind OPA validation test listener");
|
||||
let url = format!(
|
||||
"http://{}/v1/data/rustfs/authz/allow",
|
||||
listener.local_addr().expect("read listener address")
|
||||
);
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("accept OPA validation connection");
|
||||
let mut request = [0_u8; 1024];
|
||||
let bytes = stream.read(&mut request).await.expect("read OPA validation request");
|
||||
assert!(bytes > 0, "OPA validation should send an HTTP request");
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("write OPA unavailable response");
|
||||
});
|
||||
|
||||
let state = temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", Some(url.as_str())),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", None),
|
||||
],
|
||||
resolve_policy_plugin_state(),
|
||||
)
|
||||
.await;
|
||||
server.await.expect("join OPA validation test server");
|
||||
|
||||
assert!(matches!(state, PolicyPluginState::Failed));
|
||||
}
|
||||
|
||||
const CUSTOM_STS_CLAIM_POLICY: &str = "custom-sts-claim-getobject";
|
||||
const CUSTOM_STS_CLAIM_BUCKET: &str = "claim-bucket";
|
||||
const CUSTOM_STS_CLAIM_POLICY_JSON: &str = r#"{
|
||||
@@ -2000,11 +1855,6 @@ mod tests {
|
||||
empty_policies: bool,
|
||||
saved_sts_users: Arc<Mutex<HashMap<String, UserIdentity>>>,
|
||||
saved_service_account_count: Arc<Mutex<usize>>,
|
||||
fail_delete: Arc<std::sync::atomic::AtomicBool>,
|
||||
deleted_mapped_policies: Arc<Mutex<Vec<(String, UserType)>>>,
|
||||
block_delete: Arc<std::sync::atomic::AtomicBool>,
|
||||
delete_started: Arc<tokio::sync::Notify>,
|
||||
release_delete: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
impl StsTestMockStore {
|
||||
@@ -2013,11 +1863,6 @@ mod tests {
|
||||
empty_policies,
|
||||
saved_sts_users: Arc::new(Mutex::new(HashMap::new())),
|
||||
saved_service_account_count: Arc::new(Mutex::new(0)),
|
||||
fail_delete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
deleted_mapped_policies: Arc::new(Mutex::new(Vec::new())),
|
||||
block_delete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
delete_started: Arc::new(tokio::sync::Notify::new()),
|
||||
release_delete: Arc::new(tokio::sync::Notify::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2068,13 +1913,6 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn delete_user_identity(&self, name: &str, _user_type: UserType) -> Result<()> {
|
||||
if self.block_delete.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
self.delete_started.notify_one();
|
||||
self.release_delete.notified().await;
|
||||
}
|
||||
if self.fail_delete.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
return Err(Error::Io(std::io::Error::other("delete temporary account failed")));
|
||||
}
|
||||
self.saved_sts_users
|
||||
.lock()
|
||||
.expect("saved_sts_users mutex poisoned")
|
||||
@@ -2092,7 +1930,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn load_user(&self, name: &str, user_type: UserType, m: &mut HashMap<String, UserIdentity>) -> Result<()> {
|
||||
if matches!(name, "deleted-notify-user" | "deleted-notify-sts") {
|
||||
if name == "deleted-notify-user" {
|
||||
return Err(Error::NoSuchUser(name.to_string()));
|
||||
}
|
||||
|
||||
@@ -2170,11 +2008,7 @@ mod tests {
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
async fn delete_mapped_policy(&self, name: &str, user_type: UserType, _is_group: bool) -> Result<()> {
|
||||
self.deleted_mapped_policies
|
||||
.lock()
|
||||
.expect("deleted_mapped_policies mutex poisoned")
|
||||
.push((name.to_string(), user_type));
|
||||
async fn delete_mapped_policy(&self, _name: &str, _user_type: UserType, _is_group: bool) -> Result<()> {
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
@@ -4033,363 +3867,6 @@ mod tests {
|
||||
assert!(iam_sys.store.cache.snapshot().sts_policies.contains_key("notify-sts-parent"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_temp_account_notifies_peers_as_sts_user() {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
|
||||
let iam_sys = IamSys::new(cache_manager);
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(Arc::clone(&probe), async {
|
||||
iam_sys
|
||||
.delete_temp_account("deleted-notify-sts", true)
|
||||
.await
|
||||
.expect("delete temporary account");
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
probe
|
||||
.observed
|
||||
.lock()
|
||||
.expect("notification probe mutex poisoned")
|
||||
.as_ref()
|
||||
.map(|(access_key, temp)| (access_key.as_str(), *temp)),
|
||||
Some(("deleted-notify-sts", true)),
|
||||
"peer notification must retain the STS access key and user type"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_temp_account_reports_peer_invalidation_failure() {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
|
||||
let iam_sys = IamSys::new(cache_manager);
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(STS_INVALIDATION_MAX_ATTEMPTS, false, false);
|
||||
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(Arc::clone(&probe), async {
|
||||
let result = iam_sys.delete_temp_account("deleted-notify-sts", true).await;
|
||||
let err = result.expect_err("failed peer invalidation must fail STS revocation");
|
||||
assert!(err.to_string().contains("peer notification failed"));
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
probe
|
||||
.observed
|
||||
.lock()
|
||||
.expect("notification probe mutex poisoned")
|
||||
.as_ref()
|
||||
.map(|(access_key, temp)| (access_key.as_str(), *temp)),
|
||||
Some(("deleted-notify-sts", true)),
|
||||
"failed notification must retain the STS access key and user type"
|
||||
);
|
||||
assert_eq!(probe.attempts.load(std::sync::atomic::Ordering::SeqCst), STS_INVALIDATION_MAX_ATTEMPTS);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transient_peer_invalidation_retries_after_local_delete() {
|
||||
const ACCESS_KEY: &str = "retryable-revoked-sts";
|
||||
|
||||
let store = StsTestMockStore::new(false);
|
||||
let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
|
||||
let iam_sys = IamSys::new(cache_manager);
|
||||
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(2, false, false);
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(Arc::clone(&probe), iam_sys.delete_temp_account(ACCESS_KEY, true))
|
||||
.await
|
||||
.expect("transient peer invalidation should converge within the retry budget");
|
||||
|
||||
assert_eq!(
|
||||
probe.attempts.load(std::sync::atomic::Ordering::SeqCst),
|
||||
STS_INVALIDATION_MAX_ATTEMPTS,
|
||||
"peer invalidation must retry until it succeeds"
|
||||
);
|
||||
assert_eq!(
|
||||
probe
|
||||
.observed
|
||||
.lock()
|
||||
.expect("notification probe mutex poisoned")
|
||||
.as_ref()
|
||||
.map(|(access_key, temp)| (access_key.as_str(), *temp)),
|
||||
Some((ACCESS_KEY, true))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stalled_peer_invalidation_is_bounded_by_attempt_timeouts() {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
|
||||
let iam_sys = IamSys::new(cache_manager);
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, true, false);
|
||||
|
||||
let err = crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(Arc::clone(&probe), async {
|
||||
iam_sys
|
||||
.delete_temp_account("stalled-peer-sts", true)
|
||||
.await
|
||||
.expect_err("stalled peer invalidation must fail after bounded attempts")
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(err.to_string().contains("peer STS invalidation timed out"));
|
||||
assert_eq!(probe.attempts.load(std::sync::atomic::Ordering::SeqCst), STS_INVALIDATION_MAX_ATTEMPTS);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_temp_account_reports_local_deletion_failure_without_notifying() {
|
||||
let store = StsTestMockStore::new(false);
|
||||
store.fail_delete.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
|
||||
let iam_sys = IamSys::new(cache_manager);
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
|
||||
let err = crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(Arc::clone(&probe), async {
|
||||
iam_sys
|
||||
.delete_temp_account("deleted-notify-sts", true)
|
||||
.await
|
||||
.expect_err("local deletion failure must fail STS revocation")
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(err.to_string().contains("delete temporary account failed"));
|
||||
assert!(
|
||||
probe.observed.lock().expect("notification probe mutex poisoned").is_none(),
|
||||
"peer invalidation must not run after local deletion fails"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_temp_account_reports_notification_task_panic() {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
|
||||
let iam_sys = IamSys::new(cache_manager);
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, true);
|
||||
|
||||
let err = crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(Arc::clone(&probe), async {
|
||||
iam_sys
|
||||
.delete_temp_account("deleted-notify-sts", true)
|
||||
.await
|
||||
.expect_err("notification task panic must fail STS revocation")
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(err.to_string().contains("panicked"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_temp_account_notification_survives_caller_cancellation() {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
|
||||
let iam_sys = Arc::new(IamSys::new(cache_manager));
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, true, false);
|
||||
let call = {
|
||||
let iam_sys = Arc::clone(&iam_sys);
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE.scope(Arc::clone(&probe), async move {
|
||||
iam_sys.delete_temp_account("deleted-notify-sts", true).await
|
||||
})
|
||||
};
|
||||
let call = tokio::spawn(call);
|
||||
probe.started.notified().await;
|
||||
|
||||
call.abort();
|
||||
assert!(call.await.expect_err("caller task should be cancelled").is_cancelled());
|
||||
probe
|
||||
.release
|
||||
.as_ref()
|
||||
.expect("blocking probe must have a release signal")
|
||||
.notify_one();
|
||||
probe.completed.notified().await;
|
||||
|
||||
assert_eq!(
|
||||
probe
|
||||
.observed
|
||||
.lock()
|
||||
.expect("notification probe mutex poisoned")
|
||||
.as_ref()
|
||||
.map(|(access_key, temp)| (access_key.as_str(), *temp)),
|
||||
Some(("deleted-notify-sts", true)),
|
||||
"background peer invalidation must complete with the STS access key and user type"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_temp_account_local_delete_survives_caller_cancellation() {
|
||||
const ACCESS_KEY: &str = "cancelled-during-local-delete";
|
||||
|
||||
let store = StsTestMockStore::new(false);
|
||||
store.saved_sts_users.lock().expect("saved_sts_users mutex poisoned").insert(
|
||||
ACCESS_KEY.to_string(),
|
||||
UserIdentity::from(Credentials {
|
||||
access_key: ACCESS_KEY.to_string(),
|
||||
secret_key: "temporary-user-secret".to_string(),
|
||||
session_token: "session-token".to_string(),
|
||||
status: ACCOUNT_ON.to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
);
|
||||
store.block_delete.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let store_probe = store.clone();
|
||||
let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
|
||||
let iam_sys = Arc::new(IamSys::new(cache_manager));
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
let call = {
|
||||
let iam_sys = Arc::clone(&iam_sys);
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(Arc::clone(&probe), async move { iam_sys.delete_temp_account(ACCESS_KEY, true).await })
|
||||
};
|
||||
let call = tokio::spawn(call);
|
||||
store_probe.delete_started.notified().await;
|
||||
|
||||
call.abort();
|
||||
assert!(call.await.expect_err("caller task should be cancelled").is_cancelled());
|
||||
store_probe.release_delete.notify_one();
|
||||
probe.completed.notified().await;
|
||||
|
||||
assert!(
|
||||
!store_probe
|
||||
.saved_sts_users
|
||||
.lock()
|
||||
.expect("saved_sts_users mutex poisoned")
|
||||
.contains_key(ACCESS_KEY)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notified_delete_without_tokio_runtime_returns_error() {
|
||||
let runtime = tokio::runtime::Runtime::new().expect("create test runtime");
|
||||
let iam_sys = runtime.block_on(async {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
|
||||
IamSys::new(cache_manager)
|
||||
});
|
||||
drop(runtime);
|
||||
|
||||
let result = futures::executor::block_on(iam_sys.delete_temp_account("deleted-notify-sts", true));
|
||||
let err = result.expect_err("notified deletion without a Tokio runtime must return an error");
|
||||
assert!(err.to_string().contains("Tokio"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notified_delete_without_notification_system_returns_error() {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
|
||||
let iam_sys = IamSys::new(cache_manager);
|
||||
|
||||
let err = iam_sys
|
||||
.delete_temp_account("deleted-notify-sts", true)
|
||||
.await
|
||||
.expect_err("missing peer notification system must fail STS revocation");
|
||||
assert!(err.to_string().contains("peer notification system is unavailable"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_sts_notification_evicts_only_sts_cache_entry() {
|
||||
const ACCESS_KEY: &str = "deleted-notify-sts";
|
||||
const GROUP: &str = "deleted-notify-sts-group";
|
||||
|
||||
let store = StsTestMockStore::new(false);
|
||||
let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
|
||||
let iam_sys = IamSys::new(cache_manager);
|
||||
let regular_user = UserIdentity::from(Credentials {
|
||||
access_key: ACCESS_KEY.to_string(),
|
||||
secret_key: "regular-user-secret".to_string(),
|
||||
status: ACCOUNT_ON.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
let sts_user = UserIdentity::from(Credentials {
|
||||
access_key: ACCESS_KEY.to_string(),
|
||||
secret_key: "temporary-user-secret".to_string(),
|
||||
session_token: "session-token".to_string(),
|
||||
status: ACCOUNT_ON.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
let mapped_policy = MappedPolicy::new("readwrite");
|
||||
let membership = HashSet::from([GROUP.to_string()]);
|
||||
let group = GroupInfo::new(vec![ACCESS_KEY.to_string()]);
|
||||
iam_sys.store.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_user(ACCESS_KEY, ®ular_user, now);
|
||||
cache.add_or_update_user_policy(ACCESS_KEY, &mapped_policy, now);
|
||||
cache.add_or_update_group(GROUP, &group, now);
|
||||
cache.add_or_update_user_group_membership(ACCESS_KEY, &membership, now);
|
||||
cache.add_or_update_sts_account(ACCESS_KEY, &sts_user, now);
|
||||
});
|
||||
|
||||
iam_sys
|
||||
.load_user(ACCESS_KEY, UserType::Sts)
|
||||
.await
|
||||
.expect("process missing STS user notification");
|
||||
|
||||
let cache = iam_sys.store.cache.snapshot();
|
||||
assert!(!cache.sts_accounts.contains_key(ACCESS_KEY));
|
||||
assert!(
|
||||
cache.users.contains_key(ACCESS_KEY),
|
||||
"STS invalidation must not evict a same-name regular user"
|
||||
);
|
||||
assert!(cache.user_policies.contains_key(ACCESS_KEY));
|
||||
assert!(cache.user_group_memberships.contains_key(ACCESS_KEY));
|
||||
assert!(
|
||||
cache
|
||||
.groups
|
||||
.get(GROUP)
|
||||
.is_some_and(|group| group.members.contains(&ACCESS_KEY.to_string())),
|
||||
"STS invalidation must preserve same-name regular-user group membership"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_temp_account_preserves_same_name_regular_cache_state() {
|
||||
const ACCESS_KEY: &str = "deleted-notify-sts";
|
||||
|
||||
let store = StsTestMockStore::new(false);
|
||||
let cache_manager = IamCache::new(store).await.expect("initialize IAM cache");
|
||||
let iam_sys = IamSys::new(cache_manager);
|
||||
let regular_user = UserIdentity::from(Credentials {
|
||||
access_key: ACCESS_KEY.to_string(),
|
||||
secret_key: "regular-user-secret".to_string(),
|
||||
status: ACCOUNT_ON.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
let sts_user = UserIdentity::from(Credentials {
|
||||
access_key: ACCESS_KEY.to_string(),
|
||||
secret_key: "temporary-user-secret".to_string(),
|
||||
session_token: "session-token".to_string(),
|
||||
status: ACCOUNT_ON.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
let mapped_policy = MappedPolicy::new("readwrite");
|
||||
iam_sys.store.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_user(ACCESS_KEY, ®ular_user, now);
|
||||
cache.add_or_update_user_policy(ACCESS_KEY, &mapped_policy, now);
|
||||
cache.add_or_update_sts_account(ACCESS_KEY, &sts_user, now);
|
||||
});
|
||||
|
||||
iam_sys
|
||||
.delete_temp_account(ACCESS_KEY, false)
|
||||
.await
|
||||
.expect("delete temporary account without peer notification");
|
||||
|
||||
let cache = iam_sys.store.cache.snapshot();
|
||||
assert!(!cache.sts_accounts.contains_key(ACCESS_KEY));
|
||||
assert!(cache.users.contains_key(ACCESS_KEY));
|
||||
assert!(cache.user_policies.contains_key(ACCESS_KEY));
|
||||
assert!(
|
||||
iam_sys
|
||||
.store
|
||||
.api
|
||||
.deleted_mapped_policies
|
||||
.lock()
|
||||
.expect("deleted_mapped_policies mutex poisoned")
|
||||
.is_empty(),
|
||||
"deleting one STS identity must not delete a parent-scoped STS policy mapping"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_missing_user_notification_cleans_related_cache_state() {
|
||||
let store = StsTestMockStore::new(false);
|
||||
|
||||
@@ -12,8 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
//! Regression test for rustfs#4304: IAM bootstrap must not depend on the
|
||||
//! distributed namespace-lock quorum.
|
||||
//!
|
||||
|
||||
@@ -27,7 +27,6 @@ pub const INTERNODE_OPERATION_NS_SCANNER: &str = "ns_scanner";
|
||||
pub const INTERNODE_OPERATION_GRPC_READ_ALL: &str = "grpc_read_all";
|
||||
pub const INTERNODE_OPERATION_GRPC_WRITE_ALL: &str = "grpc_write_all";
|
||||
pub const INTERNODE_OPERATION_GRPC_READ_MULTIPLE: &str = "grpc_read_multiple";
|
||||
pub const INTERNODE_OPERATION_GRPC_OTHER: &str = "grpc_other";
|
||||
pub const INTERNODE_TRANSPORT_BACKEND_TCP_HTTP: &str = "tcp-http";
|
||||
pub const INTERNODE_TRANSPORT_BACKEND_GRPC: &str = "grpc";
|
||||
pub const INTERNODE_TRANSPORT_BACKEND_UNKNOWN: &str = "unknown";
|
||||
@@ -46,9 +45,6 @@ const CLASSIFICATION_LABEL: &str = "classification";
|
||||
const STAGE_LABEL: &str = "stage";
|
||||
const DOMINANT_ERROR_LABEL: &str = "dominant_error";
|
||||
const HTTP_VERSION_LABEL: &str = "http_version";
|
||||
const FAILURE_REASON_LABEL: &str = "failure_reason";
|
||||
const RPC_PATH_LABEL: &str = "rpc_path";
|
||||
const REASON_LABEL: &str = "reason";
|
||||
const DIRECTION_LABEL: &str = "direction";
|
||||
const MESSAGE_LABEL: &str = "message";
|
||||
const CODEC_LABEL: &str = "codec";
|
||||
@@ -65,7 +61,6 @@ const INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL: &str = "rustfs_system_network_int
|
||||
const INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL: &str = "rustfs_system_network_internode_operation_stall_timeouts_total";
|
||||
const INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL: &str =
|
||||
"rustfs_system_network_internode_operation_write_shutdown_errors_total";
|
||||
const INTERNODE_RPC_AUTH_FAILURES_TOTAL: &str = "rustfs_system_network_internode_rpc_auth_failures_total";
|
||||
const INTERNODE_OPERATION_PAYLOAD_BYTES: &str = "rustfs_system_network_internode_operation_payload_bytes";
|
||||
const INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL: &str = "rustfs_system_network_internode_operation_large_payloads_total";
|
||||
const INTERNODE_MSGPACK_JSON_DECODE_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_total";
|
||||
@@ -75,11 +70,6 @@ const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_inter
|
||||
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
|
||||
const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_replay_scope_fallback_total";
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_BY_OPERATION_TOTAL: &str =
|
||||
"rustfs_system_network_internode_replay_cache_overflow_by_operation_total";
|
||||
const INTERNODE_REPLAY_CACHE_ENTRIES: &str = "rustfs_system_network_internode_replay_cache_entries";
|
||||
const INTERNODE_REPLAY_CACHE_CAPACITY: &str = "rustfs_system_network_internode_replay_cache_capacity";
|
||||
const INTERNODE_REPLAY_CACHE_EVICTIONS_TOTAL: &str = "rustfs_system_network_internode_replay_cache_evictions_total";
|
||||
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -92,11 +82,6 @@ const SERVER_OPERATION_BACKEND_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL
|
||||
const SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] =
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL];
|
||||
const SERVER_OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL];
|
||||
const SERVER_OPERATION_BACKEND_FAILURE_REASON_LABELS: &[&str] =
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL];
|
||||
const SERVER_OPERATION_BACKEND_RPC_PATH_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL];
|
||||
const SERVER_LABELS: &[&str] = &[SERVER_LABEL];
|
||||
const SERVER_REASON_LABELS: &[&str] = &[SERVER_LABEL, REASON_LABEL];
|
||||
const SERVER_QUORUM_FAILURE_LABELS: &[&str] = &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL];
|
||||
|
||||
pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &[
|
||||
@@ -148,26 +133,6 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
|
||||
name: INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL,
|
||||
labels: SERVER_OPERATION_BACKEND_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_RPC_AUTH_FAILURES_TOTAL,
|
||||
labels: SERVER_OPERATION_BACKEND_FAILURE_REASON_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_REPLAY_CACHE_OVERFLOW_BY_OPERATION_TOTAL,
|
||||
labels: SERVER_OPERATION_BACKEND_RPC_PATH_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_REPLAY_CACHE_ENTRIES,
|
||||
labels: SERVER_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_REPLAY_CACHE_CAPACITY,
|
||||
labels: SERVER_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_REPLAY_CACHE_EVICTIONS_TOTAL,
|
||||
labels: SERVER_REASON_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
|
||||
labels: SERVER_QUORUM_FAILURE_LABELS,
|
||||
@@ -213,14 +178,10 @@ pub struct InternodeMetricsSnapshot {
|
||||
pub operation_http_versions_total: u64,
|
||||
pub operation_stall_timeouts_total: u64,
|
||||
pub operation_write_shutdown_errors_total: u64,
|
||||
pub rpc_auth_failures_total: u64,
|
||||
pub signature_v1_fallback_total: u64,
|
||||
pub body_digest_fallback_total: u64,
|
||||
pub replay_scope_fallback_total: u64,
|
||||
pub replay_cache_overflow_total: u64,
|
||||
pub replay_cache_entries: u64,
|
||||
pub replay_cache_capacity: u64,
|
||||
pub replay_cache_evictions_total: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -237,20 +198,12 @@ pub struct InternodeMetrics {
|
||||
operation_http_versions_total: AtomicU64,
|
||||
operation_stall_timeouts_total: AtomicU64,
|
||||
operation_write_shutdown_errors_total: AtomicU64,
|
||||
rpc_auth_failures_total: AtomicU64,
|
||||
msgpack_json_decode_total: AtomicU64,
|
||||
msgpack_json_decode_error_total: AtomicU64,
|
||||
signature_v1_fallback_total: AtomicU64,
|
||||
body_digest_fallback_total: AtomicU64,
|
||||
replay_scope_fallback_total: AtomicU64,
|
||||
replay_cache_overflow_total: AtomicU64,
|
||||
replay_cache_entries: AtomicU64,
|
||||
replay_cache_capacity: AtomicU64,
|
||||
replay_cache_evictions_total: AtomicU64,
|
||||
}
|
||||
|
||||
fn usize_to_u64_saturating(value: usize) -> u64 {
|
||||
u64::try_from(value).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
impl InternodeMetrics {
|
||||
@@ -470,23 +423,6 @@ impl InternodeMetrics {
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub fn record_rpc_auth_failure_for_operation_and_backend(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
failure_reason: &'static str,
|
||||
) {
|
||||
self.rpc_auth_failures_total.fetch_add(1, Ordering::Relaxed);
|
||||
counter!(
|
||||
INTERNODE_RPC_AUTH_FAILURES_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
OPERATION_LABEL => operation,
|
||||
BACKEND_LABEL => backend,
|
||||
FAILURE_REASON_LABEL => failure_reason
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Record the payload size (bytes) of a completed internode operation into a histogram
|
||||
/// keyed by operation+backend. Used to size which unary `bytes`-carrying RPCs
|
||||
/// (`ReadAll`/`ReadMultiple`/`WriteAll`) would benefit from being moved off the shared
|
||||
@@ -601,46 +537,6 @@ impl InternodeMetrics {
|
||||
counter!(INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL, SERVER_LABEL => current_server_label()).increment(1);
|
||||
}
|
||||
|
||||
pub fn record_replay_cache_overflow_for_operation_and_backend_path(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
rpc_path: &str,
|
||||
) {
|
||||
self.record_replay_cache_overflow();
|
||||
counter!(
|
||||
INTERNODE_REPLAY_CACHE_OVERFLOW_BY_OPERATION_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
OPERATION_LABEL => operation,
|
||||
BACKEND_LABEL => backend,
|
||||
RPC_PATH_LABEL => rpc_path.to_owned()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub fn record_replay_cache_state(&self, entries: usize, capacity: usize) {
|
||||
let entries = usize_to_u64_saturating(entries);
|
||||
let capacity = usize_to_u64_saturating(capacity);
|
||||
self.replay_cache_entries.store(entries, Ordering::Relaxed);
|
||||
self.replay_cache_capacity.store(capacity, Ordering::Relaxed);
|
||||
gauge!(INTERNODE_REPLAY_CACHE_ENTRIES, SERVER_LABEL => current_server_label()).set(entries as f64);
|
||||
gauge!(INTERNODE_REPLAY_CACHE_CAPACITY, SERVER_LABEL => current_server_label()).set(capacity as f64);
|
||||
}
|
||||
|
||||
pub fn record_replay_cache_evictions(&self, reason: &'static str, count: usize) {
|
||||
if count == 0 {
|
||||
return;
|
||||
}
|
||||
let count = usize_to_u64_saturating(count);
|
||||
self.replay_cache_evictions_total.fetch_add(count, Ordering::Relaxed);
|
||||
counter!(
|
||||
INTERNODE_REPLAY_CACHE_EVICTIONS_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
REASON_LABEL => reason
|
||||
)
|
||||
.increment(count);
|
||||
}
|
||||
|
||||
pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) {
|
||||
counter!(
|
||||
ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
|
||||
@@ -689,14 +585,10 @@ impl InternodeMetrics {
|
||||
operation_http_versions_total: self.operation_http_versions_total.load(Ordering::Relaxed),
|
||||
operation_stall_timeouts_total: self.operation_stall_timeouts_total.load(Ordering::Relaxed),
|
||||
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
|
||||
rpc_auth_failures_total: self.rpc_auth_failures_total.load(Ordering::Relaxed),
|
||||
signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
|
||||
body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
|
||||
replay_scope_fallback_total: self.replay_scope_fallback_total.load(Ordering::Relaxed),
|
||||
replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
|
||||
replay_cache_entries: self.replay_cache_entries.load(Ordering::Relaxed),
|
||||
replay_cache_capacity: self.replay_cache_capacity.load(Ordering::Relaxed),
|
||||
replay_cache_evictions_total: self.replay_cache_evictions_total.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -714,16 +606,12 @@ impl InternodeMetrics {
|
||||
self.operation_http_versions_total.store(0, Ordering::Relaxed);
|
||||
self.operation_stall_timeouts_total.store(0, Ordering::Relaxed);
|
||||
self.operation_write_shutdown_errors_total.store(0, Ordering::Relaxed);
|
||||
self.rpc_auth_failures_total.store(0, Ordering::Relaxed);
|
||||
self.msgpack_json_decode_total.store(0, Ordering::Relaxed);
|
||||
self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
|
||||
self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.body_digest_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.replay_scope_fallback_total.store(0, Ordering::Relaxed);
|
||||
self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
|
||||
self.replay_cache_entries.store(0, Ordering::Relaxed);
|
||||
self.replay_cache_capacity.store(0, Ordering::Relaxed);
|
||||
self.replay_cache_evictions_total.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -890,7 +778,7 @@ mod tests {
|
||||
use super::*;
|
||||
use metrics::with_local_recorder;
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[test]
|
||||
fn snapshot_reports_recorded_values() {
|
||||
@@ -941,13 +829,6 @@ mod tests {
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
);
|
||||
metrics.record_error_for_operation_and_backend(INTERNODE_OPERATION_WALK_DIR, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP);
|
||||
metrics.record_rpc_auth_failure_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_OTHER,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
"missing_v2_signature",
|
||||
);
|
||||
metrics.record_replay_cache_state(64, 1024);
|
||||
metrics.record_replay_cache_evictions("expired", 3);
|
||||
|
||||
let snapshot = metrics.snapshot();
|
||||
assert_eq!(snapshot.sent_bytes_total, 128);
|
||||
@@ -955,15 +836,11 @@ mod tests {
|
||||
assert_eq!(snapshot.outgoing_requests_total, 1);
|
||||
assert_eq!(snapshot.incoming_requests_total, 1);
|
||||
assert_eq!(snapshot.errors_total, 1);
|
||||
assert_eq!(snapshot.rpc_auth_failures_total, 1);
|
||||
assert_eq!(snapshot.replay_cache_entries, 64);
|
||||
assert_eq!(snapshot.replay_cache_capacity, 1024);
|
||||
assert_eq!(snapshot.replay_cache_evictions_total, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operation_metric_descriptors_include_backend_and_operation_labels() {
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 20);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 15);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[..6] {
|
||||
assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
@@ -977,22 +854,10 @@ mod tests {
|
||||
for metric in &INTERNODE_OPERATION_METRICS[10..12] {
|
||||
assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[12].labels,
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL]
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[13].labels,
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL]
|
||||
);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[14..16] {
|
||||
assert_eq!(metric.labels, &[SERVER_LABEL]);
|
||||
}
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[16].labels, &[SERVER_LABEL, REASON_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[17].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[12].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]);
|
||||
// Payload histogram + large-payload counter carry operation+backend labels.
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[18].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[19].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[13].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[14].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1002,7 +867,6 @@ mod tests {
|
||||
assert_eq!(INTERNODE_OPERATION_WALK_DIR, "walk_dir");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_READ_ALL, "grpc_read_all");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_WRITE_ALL, "grpc_write_all");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_OTHER, "grpc_other");
|
||||
|
||||
assert_eq!(INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, "tcp-http");
|
||||
assert_eq!(INTERNODE_TRANSPORT_BACKEND_GRPC, "grpc");
|
||||
@@ -1038,34 +902,14 @@ mod tests {
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[12].name,
|
||||
"rustfs_system_network_internode_rpc_auth_failures_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[13].name,
|
||||
"rustfs_system_network_internode_replay_cache_overflow_by_operation_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[14].name,
|
||||
"rustfs_system_network_internode_replay_cache_entries"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[15].name,
|
||||
"rustfs_system_network_internode_replay_cache_capacity"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[16].name,
|
||||
"rustfs_system_network_internode_replay_cache_evictions_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[17].name,
|
||||
"rustfs_system_storage_erasure_write_quorum_failures_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[18].name,
|
||||
INTERNODE_OPERATION_METRICS[13].name,
|
||||
"rustfs_system_network_internode_operation_payload_bytes"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[19].name,
|
||||
INTERNODE_OPERATION_METRICS[14].name,
|
||||
"rustfs_system_network_internode_operation_large_payloads_total"
|
||||
);
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, "grpc_read_multiple");
|
||||
@@ -1089,94 +933,6 @@ mod tests {
|
||||
INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL,
|
||||
"rustfs_system_network_internode_signature_v1_fallback_total"
|
||||
);
|
||||
assert_eq!(FAILURE_REASON_LABEL, "failure_reason");
|
||||
assert_eq!(RPC_PATH_LABEL, "rpc_path");
|
||||
assert_eq!(REASON_LABEL, "reason");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rpc_auth_failure_counter_records_low_cardinality_labels() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let metrics = InternodeMetrics::default();
|
||||
|
||||
with_local_recorder(&recorder, || {
|
||||
metrics.record_rpc_auth_failure_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
"invalid_v2_signature",
|
||||
);
|
||||
});
|
||||
|
||||
assert_eq!(metrics.snapshot().rpc_auth_failures_total, 1);
|
||||
let entries: Vec<_> = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter(|(composite, _, _, _)| composite.key().name() == INTERNODE_RPC_AUTH_FAILURES_TOTAL)
|
||||
.collect();
|
||||
assert_eq!(entries.len(), 1);
|
||||
let labels: HashMap<_, _> = entries[0]
|
||||
.0
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect();
|
||||
assert_eq!(labels.get(OPERATION_LABEL).map(String::as_str), Some(INTERNODE_OPERATION_GRPC_READ_ALL));
|
||||
assert_eq!(labels.get(BACKEND_LABEL).map(String::as_str), Some(INTERNODE_TRANSPORT_BACKEND_GRPC));
|
||||
assert_eq!(labels.get(FAILURE_REASON_LABEL).map(String::as_str), Some("invalid_v2_signature"));
|
||||
assert!(labels.get(SERVER_LABEL).is_some_and(|value| !value.is_empty()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_metrics_record_state_eviction_and_overflow_scope() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let metrics = InternodeMetrics::default();
|
||||
|
||||
with_local_recorder(&recorder, || {
|
||||
metrics.record_replay_cache_state(7, 11);
|
||||
metrics.record_replay_cache_evictions("expired", 5);
|
||||
metrics.record_replay_cache_overflow_for_operation_and_backend_path(
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
"/node_service.NodeService/ReadAll",
|
||||
);
|
||||
});
|
||||
|
||||
let snapshot = metrics.snapshot();
|
||||
assert_eq!(snapshot.replay_cache_entries, 7);
|
||||
assert_eq!(snapshot.replay_cache_capacity, 11);
|
||||
assert_eq!(snapshot.replay_cache_evictions_total, 5);
|
||||
assert_eq!(snapshot.replay_cache_overflow_total, 1);
|
||||
|
||||
let entries: Vec<_> = snapshotter.snapshot().into_vec();
|
||||
assert!(
|
||||
entries
|
||||
.iter()
|
||||
.any(|(composite, _, _, _)| composite.key().name() == INTERNODE_REPLAY_CACHE_ENTRIES)
|
||||
);
|
||||
assert!(
|
||||
entries
|
||||
.iter()
|
||||
.any(|(composite, _, _, _)| composite.key().name() == INTERNODE_REPLAY_CACHE_CAPACITY)
|
||||
);
|
||||
|
||||
let overflow: Vec<_> = entries
|
||||
.iter()
|
||||
.filter(|(composite, _, _, _)| composite.key().name() == INTERNODE_REPLAY_CACHE_OVERFLOW_BY_OPERATION_TOTAL)
|
||||
.collect();
|
||||
assert_eq!(overflow.len(), 1);
|
||||
let labels: HashMap<_, _> = overflow[0]
|
||||
.0
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect();
|
||||
assert_eq!(labels.get(OPERATION_LABEL).map(String::as_str), Some(INTERNODE_OPERATION_GRPC_READ_ALL));
|
||||
assert_eq!(labels.get(BACKEND_LABEL).map(String::as_str), Some(INTERNODE_TRANSPORT_BACKEND_GRPC));
|
||||
assert_eq!(labels.get(RPC_PATH_LABEL).map(String::as_str), Some("/node_service.NodeService/ReadAll"));
|
||||
assert!(labels.get(SERVER_LABEL).is_some_and(|value| !value.is_empty()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -585,16 +585,13 @@ impl KmsBackend for AwsKmsBackend {
|
||||
// AWS rejects a `Limit` of zero, and clamping it up to one would return
|
||||
// a key to a caller that asked for none; the empty page is answered
|
||||
// here instead.
|
||||
let Some(page_size) = list_keys_page_size(request.limit) else {
|
||||
if list_keys_page_size(request.limit).is_none() {
|
||||
return Ok(empty_key_page());
|
||||
};
|
||||
}
|
||||
|
||||
// Taking the remote page size from the shared resolver keeps the AWS
|
||||
// request under the same ceiling every other backend obeys; the AWS API
|
||||
// maximum is the same 1000, so this never widens the remote page.
|
||||
let limit = request
|
||||
.limit
|
||||
.map(|_| i32::try_from(page_size).unwrap_or(i32::MAX).clamp(1, 1000));
|
||||
.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX).clamp(1, 1000));
|
||||
let marker = request.marker.clone();
|
||||
|
||||
let output = self
|
||||
@@ -620,17 +617,7 @@ impl KmsBackend for AwsKmsBackend {
|
||||
let Some(key_id) = entry.key_id() else {
|
||||
continue;
|
||||
};
|
||||
let metadata = match self.describe(key_id).await {
|
||||
Ok(metadata) => metadata,
|
||||
// AWS `ListKeys` is eventually consistent, so a key destroyed
|
||||
// between the listing and the describe is routine: it is
|
||||
// dropped and the remote cursor still advances past it. There
|
||||
// is no local record to be damaged here — key state lives in
|
||||
// AWS — so `unreadable_key_ids` stays empty on this backend and
|
||||
// every other failure fails the listing.
|
||||
Err(KmsError::KeyNotFound { .. }) => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let metadata = self.describe(key_id).await?;
|
||||
if request
|
||||
.usage_filter
|
||||
.as_ref()
|
||||
@@ -655,8 +642,6 @@ impl KmsBackend for AwsKmsBackend {
|
||||
created_at: metadata.creation_date,
|
||||
rotated_at: None,
|
||||
created_by: None,
|
||||
rotation_due: false,
|
||||
rotation_due_reason: None,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -664,8 +649,6 @@ impl KmsBackend for AwsKmsBackend {
|
||||
keys,
|
||||
next_marker: output.next_marker.clone(),
|
||||
truncated: output.truncated,
|
||||
// AWS owns key state; nothing here can be present-but-unreadable.
|
||||
unreadable_key_ids: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -183,7 +183,6 @@ async fn assert_state_machine_contract(backend: &dyn KmsBackend, key_id: &str) {
|
||||
.await
|
||||
.expect("decrypt with a disabled key must keep working");
|
||||
assert_eq!(decrypted.plaintext, data_key.plaintext_key, "decrypt must recover the original data key");
|
||||
assert_eq!(decrypted.key_id, key_id, "decrypt must report the master key that opened the envelope");
|
||||
// ...disable stays idempotent, cancel has nothing to cancel, and enable recovers.
|
||||
backend.disable_key(key_id).await.expect("disable must be idempotent");
|
||||
expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "not pending deletion");
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user