mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 16:46:55 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 77b712dcb6 |
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: rustfs-logging-governance
|
||||
description: Standardize and review RustFS logging with structured `tracing` events, lower noise on hot paths, preserve security-sensitive diagnostics, and extend guardrails to prevent legacy logging patterns from returning. Use whenever a change adds or edits any `tracing` macro call (`error!`/`warn!`/`info!`/`debug!`/`trace!`/`#[instrument]`) — including a single log line added in passing while fixing unrelated logic, which is how most new log sites enter the repo — and when reviewing RustFS logs, startup/config diagnostics, cloud metadata logs, request validation logs, or `scripts/check_logging_guardrails.sh`.
|
||||
description: Standardize and review RustFS logging with structured `tracing` events, lower noise on hot paths, preserve security-sensitive diagnostics, and extend guardrails to prevent legacy logging patterns from returning. Use when editing or reviewing RustFS logs, startup/config diagnostics, cloud metadata logs, request validation logs, or `scripts/check_logging_guardrails.sh`.
|
||||
---
|
||||
|
||||
# RustFS Logging Governance
|
||||
|
||||
@@ -60,16 +60,6 @@ body-cache-whitelist-check: ## Check the body-cache eligibility gate stays a fai
|
||||
@echo "🧱 Checking body-cache whitelist guard..."
|
||||
./scripts/check_body_cache_whitelist.sh
|
||||
|
||||
.PHONY: s3s-footprint-check
|
||||
s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
@echo "📦 Checking s3s footprint ratchet..."
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: fips-wording-check
|
||||
fips-wording-check: ## Check outward docs do not make unsupported FIPS claims
|
||||
@echo "📣 Checking FIPS wording guard..."
|
||||
./scripts/check_fips_wording.sh
|
||||
|
||||
.PHONY: log-analyzer-rules-check
|
||||
log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
|
||||
@echo "🩺 Checking log-analyzer rule anchors..."
|
||||
|
||||
@@ -19,13 +19,13 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
@echo "✅ Fast development checks passed!"
|
||||
|
||||
+12
-77
@@ -9,8 +9,6 @@
|
||||
# * bucket_lifecycle_ops::tests::concurrent_resend_same_part_commits_one_generation
|
||||
# uses the shared multipart fixture and a deterministic uploadId-lock
|
||||
# handoff, so it must not overlap another process mutating that fixture.
|
||||
# * bucket::metadata_sys::tests::concurrent_config_writes_from_separate_nodes_do_not_lose_writes
|
||||
# uses the shared transaction lock and must not overlap other ecstore tests.
|
||||
#
|
||||
# serial_test's #[serial] attribute does NOT serialize these across runs:
|
||||
# nextest executes each test in its own process, where the in-process
|
||||
@@ -29,8 +27,6 @@
|
||||
|
||||
[test-groups]
|
||||
ecstore-serial-flaky = { max-threads = 1 }
|
||||
embedded-test-ports = { max-threads = 1 }
|
||||
e2e-vault = { max-threads = 1 }
|
||||
|
||||
# Reliability / fault-injection e2e tests each spawn a single-node 4-disk RustFS
|
||||
# server and manipulate its disk directories at runtime (crates/e2e_test:
|
||||
@@ -44,7 +40,7 @@ e2e-inline-boundaries = { max-threads = 1 }
|
||||
|
||||
# --- default profile (local): serialize the flaky groups, never retry --------
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(concurrent_config_writes_from_separate_nodes_do_not_lose_writes) | test(/^store::bucket::tests::bucket_delete_(mark_delete|purge_removes|default_s3_delete)/))'
|
||||
filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(/^store::bucket::tests::bucket_delete_(mark_delete|purge_removes|default_s3_delete)/))'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the multipart crash-consistency scenarios (dist-2, backlog#1150):
|
||||
@@ -56,36 +52,12 @@ test-group = 'ecstore-serial-flaky'
|
||||
filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# The production-handler relocation regression builds an isolated 8-disk,
|
||||
# 2-pool store and commits a 72 MiB multipart object. Keep that cross-disk IO
|
||||
# from overlapping the ecstore commit fixtures above.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & test(execute_get_object_resumes_from_relocated_pool_without_splicing_body)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Embedded integration-test binaries discover an ephemeral port and release
|
||||
# the probe listener before RustFS binds it. Serialize that cross-process
|
||||
# TOCTOU window; retries would only hide real startup failures.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
# Serialize the durable manual-transition checkpoint test across nextest's
|
||||
# process boundary; it mutates bucket lifecycle metadata and is not quarantined.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the bucket-incarnation / lifecycle-fence tests. They drive
|
||||
# init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
|
||||
# OnceLock state that serial_test's #[serial] cannot protect across nextest's
|
||||
# process boundary, and they delete+recreate buckets — the same shape that
|
||||
# raced into InsufficientWriteQuorum in backlog#937. Preventive only, no
|
||||
# retries. The matching ci-profile override is after [profile.ci].
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & (test(bucket_delete_waits_for_config_mutation_fence) | test(stale_config_request_cannot_mutate_a_recreated_bucket) | test(disk_incarnation_read_detects_stale_cache_until_peer_reload) | test(lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata) | test(expiry_configs_are_resolved_from_the_owning_store))'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the 4-disk reliability / degraded-read e2e tests (see the
|
||||
# e2e-reliability test-group note above). The matching ci-profile override is at
|
||||
# the end of the file, after [profile.ci] is declared.
|
||||
@@ -97,12 +69,6 @@ test-group = 'e2e-reliability'
|
||||
filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
|
||||
test-group = 'e2e-inline-boundaries'
|
||||
|
||||
# Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
|
||||
# does not cross nextest process boundaries, so keep these tests in one group.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -138,9 +104,9 @@ filter = 'package(rustfs-ecstore) & test(/^store::bucket::tests::bucket_delete_(
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
retries = 2
|
||||
|
||||
# Keep deterministic ECStore write handoffs isolated across nextest processes.
|
||||
# Keep the deterministic multipart handoff isolated across nextest processes.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(concurrent_config_writes_from_separate_nodes_do_not_lose_writes))'
|
||||
filter = 'package(rustfs-ecstore) & test(concurrent_resend_same_part_commits_one_generation)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# QUARANTINE: OPEN rustfs#4690 — walk_dir stall-budget accounting test depends
|
||||
@@ -165,28 +131,12 @@ test-group = 'e2e-reliability'
|
||||
filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & test(execute_get_object_resumes_from_relocated_pool_without_splicing_body)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Match the default-profile embedded test isolation without quarantining or
|
||||
# retrying failures in CI.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
# Serialize the durable manual-transition checkpoint test under the ci profile
|
||||
# too. No retries: failures stay visible.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
|
||||
# too (see the matching default-profile override near the top). No retries.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & (test(bucket_delete_waits_for_config_mutation_fence) | test(stale_config_request_cannot_mutate_a_recreated_bucket) | test(disk_incarnation_read_detects_stale_cache_until_peer_reload) | test(lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata) | test(expiry_configs_are_resolved_from_the_owning_store))'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-smoke profile — PR smoke subset of the e2e_test crate (backlog#1149 ci-4)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -218,7 +168,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
# the nightly profile derives its set as "the replication module MINUS this
|
||||
# allowlist", so any new replication test lands in nightly by default (never
|
||||
# silently unrun) until it is explicitly blessed as fast here. Keep the two
|
||||
# regexes byte-identical. Count invariant: 20 here + 36 nightly = 56 total
|
||||
# regexes byte-identical. Count invariant: 20 here + 28 nightly = 48 total
|
||||
# (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
|
||||
# HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
|
||||
# (#4724) because they set a loopback (127.0.0.1) replication target that the
|
||||
@@ -254,7 +204,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^reliant::tiering::/)
|
||||
@@ -262,17 +212,6 @@ default-filter = """
|
||||
"""
|
||||
fail-fast = false
|
||||
|
||||
[profile.e2e-smoke.junit]
|
||||
path = "junit.xml"
|
||||
|
||||
# The pagination boundary cases can stall when a server/listing regression
|
||||
# prevents the continuation request from completing. Keep the timeout scoped
|
||||
# to those known failure modes so legitimate lifecycle/tiering waits retain
|
||||
# their test-level timing budget.
|
||||
[[profile.e2e-smoke.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/)'
|
||||
slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-repl-nightly profile — scheduled full replication e2e lane (repl-1)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -280,10 +219,10 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
|
||||
# tests that are unfit for the per-PR e2e-smoke gate:
|
||||
#
|
||||
# * 2 remote-target TLS validation tests.
|
||||
# * 13 bucket-replication data-plane/helper tests — they PUT/delete objects
|
||||
# and poll until source and target converge; two replicate over HTTPS,
|
||||
# four pin active SSE fail-closed contracts (SSE-C, SSE-S3, SSE-KMS, and
|
||||
# the SSE-S3 resync path), and one guards event/history observers.
|
||||
# * 12 bucket-replication data-plane/helper tests — they PUT/delete objects
|
||||
# and poll until source and target converge; two replicate over HTTPS, two
|
||||
# pin active SSE failure contracts, and one guards event/history observers.
|
||||
# The SSE-S3 contract remains ignored under backlog#1291.
|
||||
# * 12 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
|
||||
# servers and drives the cross-process site-replication control plane.
|
||||
# * 1 `_real_three_node` site-replication test.
|
||||
@@ -349,9 +288,9 @@ path = "junit.xml"
|
||||
#
|
||||
# Each e2e test spawns its own single-node rustfs server on a random port with
|
||||
# an isolated temp dir (crates/e2e_test/src/common.rs), so the set is
|
||||
# parallel-safe — the same property e2e-smoke relies on. The exceptions are the
|
||||
# 4-disk reliability / degraded-read fault-injection tests and the fixed-port
|
||||
# Vault tests, both serialized below.
|
||||
# parallel-safe — the same property e2e-smoke relies on. The exception is the
|
||||
# 4-disk reliability / degraded-read fault-injection tests, serialized below
|
||||
# (identical to the ci profile) so several 4-disk servers never run at once.
|
||||
# KNOWN-FAILURE EXCLUSIONS (characterization run 29381309848, 2026-07-15:
|
||||
# 341 ran / 32 failed on the suites' first automated run ever). Deterministic
|
||||
# product failures cannot be quarantined away with retries, so each family is
|
||||
@@ -387,7 +326,3 @@ test-group = 'e2e-reliability'
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
|
||||
test-group = 'e2e-inline-boundaries'
|
||||
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^kms::kms_vault_test::/)'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
@@ -170,10 +170,6 @@ Important behavior notes:
|
||||
|
||||
- Logs and metrics usually appear during startup, so seeing those two signals
|
||||
first is expected.
|
||||
- The OpenTelemetry bridge sends `tracing` fields as log attributes. Loki stores
|
||||
those attributes as structured metadata, and the Collector also mirrors the
|
||||
common troubleshooting fields into the log line so simple line filters can
|
||||
find them.
|
||||
- Visible trace data usually requires real HTTP/S3/gRPC request traffic after
|
||||
startup, because request-path spans are created on demand.
|
||||
- `RUSTFS_OBS_LOGGER_LEVEL=info` keeps the top-level request span but filters
|
||||
@@ -199,17 +195,6 @@ curl -I http://127.0.0.1:9000/health/ready
|
||||
# Jaeger: http://localhost:16686
|
||||
```
|
||||
|
||||
For a structured RustFS log such as an inter-node RPC authentication failure,
|
||||
the Loki line now includes fields such as `event`, `component`, `subsystem`,
|
||||
`failure_reason`, `rpc_service`, `rpc_method`, and `expected_audience`. Useful
|
||||
LogQL checks:
|
||||
|
||||
```logql
|
||||
{service_name="RustFS"} |= "RPC signature verification failed"
|
||||
{service_name="RustFS"} |= "failure_reason="
|
||||
{service_name="RustFS"} | failure_reason != ""
|
||||
```
|
||||
|
||||
If logs and metrics are present but traces are sparse, the most common cause is
|
||||
"no real request traffic yet" or "`info` level filtered nested spans", not an
|
||||
OTLP routing failure.
|
||||
|
||||
@@ -169,7 +169,6 @@ RustFS 会自动在该基础 URL 后补全:
|
||||
需要注意:
|
||||
|
||||
- 启动阶段通常会先看到日志和指标,因此“先有日志/指标、后有 trace”是正常现象。
|
||||
- OpenTelemetry bridge 会把 `tracing` 字段作为日志 attributes 发送。Loki 会将这些 attributes 存为 structured metadata,同时 Collector 会把常用排障字段镜像进日志行,方便用简单的行内容过滤直接查到。
|
||||
- 可见的 trace 数据通常依赖启动后的真实 HTTP/S3/gRPC 请求流量,因为请求路径上的 span 是按需创建的。
|
||||
- `RUSTFS_OBS_LOGGER_LEVEL=info` 会保留顶层请求 span,但会过滤掉很多 `debug` 级别的嵌套 span。
|
||||
如果 Tempo 或 Jaeger 中的 trace 看起来很稀疏,建议先改成 `RUSTFS_OBS_LOGGER_LEVEL=debug`,再判断是否是 collector 或 Tempo 问题。
|
||||
@@ -193,14 +192,6 @@ curl -I http://127.0.0.1:9000/health/ready
|
||||
# Jaeger: http://localhost:16686
|
||||
```
|
||||
|
||||
对于 RustFS 结构化日志,例如节点间 RPC 鉴权失败,Loki 日志行现在会包含 `event`、`component`、`subsystem`、`failure_reason`、`rpc_service`、`rpc_method`、`expected_audience` 等字段。常用 LogQL 检查:
|
||||
|
||||
```logql
|
||||
{service_name="RustFS"} |= "RPC signature verification failed"
|
||||
{service_name="RustFS"} |= "failure_reason="
|
||||
{service_name="RustFS"} | failure_reason != ""
|
||||
```
|
||||
|
||||
如果日志和指标已经正常,但 trace 仍然稀疏,最常见的原因通常是
|
||||
“还没有真实请求流量”或“`info` 级别过滤了嵌套 span”,而不是 OTLP 路由失败。
|
||||
|
||||
|
||||
@@ -11500,831 +11500,6 @@
|
||||
],
|
||||
"title": "Compression Operations Rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 332
|
||||
},
|
||||
"id": 531,
|
||||
"panels": [],
|
||||
"title": "Metrics Dimensions Drilldown",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 333
|
||||
},
|
||||
"id": 532,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "sum by (server, name, type) (rate(rustfs_api_requests_requests_total_by_server{job=~\"$job\",server=~\"$server\",name=~\"$api\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server}} | {{name}} | {{type}}"
|
||||
}
|
||||
],
|
||||
"title": "API Requests by Server and API",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byFrameRefID",
|
||||
"options": "A"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "none"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 333
|
||||
},
|
||||
"id": 533,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "max by (server, drive, pool_index, set_index, drive_index, state) (rustfs_system_drive_runtime_state{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
|
||||
"legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{state}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"expr": "max by (server, drive, pool_index, set_index, drive_index) (rustfs_system_drive_offline_duration_seconds{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
|
||||
"legendFormat": "{{server}} | {{drive}} | offline seconds"
|
||||
}
|
||||
],
|
||||
"title": "Drive Runtime State and Offline Duration",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 341
|
||||
},
|
||||
"id": 534,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "sum by (server, drive, pool_index, set_index, drive_index, api) (rate(rustfs_system_drive_api_calls_total{job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{api}}"
|
||||
}
|
||||
],
|
||||
"title": "Drive API Calls by Operation",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byFrameRefID",
|
||||
"options": "B"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "short"
|
||||
},
|
||||
{
|
||||
"id": "custom.axisPlacement",
|
||||
"value": "right"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 341
|
||||
},
|
||||
"id": 535,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "sum by (server, source, state) (rate(rustfs_scanner_source_work_total{job=~\"$job\",server=~\"$server\",source=~\"$scanner_source\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server}} | {{source}} | {{state}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"expr": "sum by (server, cycle_scope, source, state) (rustfs_scanner_cycle_source_work{job=~\"$job\",server=~\"$server\",source=~\"$scanner_source\"})",
|
||||
"legendFormat": "{{server}} | {{cycle_scope}} | {{source}} | {{state}}"
|
||||
}
|
||||
],
|
||||
"title": "Scanner Source Work by Server",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byFrameRefID",
|
||||
"options": "B"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "short"
|
||||
},
|
||||
{
|
||||
"id": "custom.axisPlacement",
|
||||
"value": "right"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 349
|
||||
},
|
||||
"id": 536,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "sum by (server, bucket, drive, result) (rate(rustfs_scanner_bucket_drive_result_total{job=~\"$job\",server=~\"$server\",bucket=~\"$bucket\",drive=~\"$drive\",result=~\"$scanner_result\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server}} | {{bucket}} | {{drive}} | {{result}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"expr": "sum by (server, cycle_scope, bucket, drive, result) (rustfs_scanner_cycle_bucket_drive_result{job=~\"$job\",server=~\"$server\",bucket=~\"$bucket\",drive=~\"$drive\",result=~\"$scanner_result\"})",
|
||||
"legendFormat": "{{server}} | {{cycle_scope}} | {{bucket}} | {{drive}} | {{result}}"
|
||||
}
|
||||
],
|
||||
"title": "Scanner Bucket Drive Results",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byFrameRefID",
|
||||
"options": "B"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "Bps"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 349
|
||||
},
|
||||
"id": 537,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "sum by (bucket, target_arn) (rate(rustfs_bucket_replication_target_sent_count{job=~\"$job\",bucket=~\"$bucket\",target_arn=~\"$target_arn\"}[$__rate_interval]))",
|
||||
"legendFormat": "sent objects | {{bucket}} | {{target_arn}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"expr": "sum by (bucket, target_arn) (rate(rustfs_bucket_replication_target_sent_bytes{job=~\"$job\",bucket=~\"$bucket\",target_arn=~\"$target_arn\"}[$__rate_interval]))",
|
||||
"legendFormat": "sent bytes | {{bucket}} | {{target_arn}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "C",
|
||||
"expr": "sum by (bucket, target_arn) (rate(rustfs_bucket_replication_target_total_failed_count{job=~\"$job\",bucket=~\"$bucket\",target_arn=~\"$target_arn\"}[$__rate_interval]))",
|
||||
"legendFormat": "failed objects | {{bucket}} | {{target_arn}}"
|
||||
}
|
||||
],
|
||||
"title": "Bucket Replication Target Flow",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 357
|
||||
},
|
||||
"id": 538,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "max by (server, target_id) (rustfs_audit_target_queue_length_by_server{job=~\"$job\",server=~\"$server\"})",
|
||||
"legendFormat": "audit queue | {{server}} | {{target_id}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"expr": "max by (server, action, state) (rustfs_ilm_action_tasks{job=~\"$job\",server=~\"$server\"})",
|
||||
"legendFormat": "ilm | {{server}} | {{action}} | {{state}}"
|
||||
}
|
||||
],
|
||||
"title": "Audit and ILM by Server",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"preload": false,
|
||||
@@ -12376,32 +11551,6 @@
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(rustfs_system_drive_api_calls_total,api)",
|
||||
"includeAll": true,
|
||||
"label": "Drive API",
|
||||
"multi": true,
|
||||
"name": "drive_api",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs_system_drive_api_calls_total,api)",
|
||||
"refId": "PrometheusVariableQueryEditor-drive_api"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
@@ -12521,136 +11670,6 @@
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(rustfs_api_requests_requests_total_by_server,server)",
|
||||
"includeAll": true,
|
||||
"label": "Server",
|
||||
"multi": true,
|
||||
"name": "server",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs_api_requests_requests_total_by_server,server)",
|
||||
"refId": "PrometheusVariableQueryEditor-server"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(rustfs_api_requests_requests_total_by_server,name)",
|
||||
"includeAll": true,
|
||||
"label": "API",
|
||||
"multi": true,
|
||||
"name": "api",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs_api_requests_requests_total_by_server,name)",
|
||||
"refId": "PrometheusVariableQueryEditor-api"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values({__name__=\"rustfs_bucket_replication_target_sent_count\",bucket=~\"$bucket\"},target_arn)",
|
||||
"includeAll": true,
|
||||
"label": "Target ARN",
|
||||
"multi": true,
|
||||
"name": "target_arn",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values({__name__=\"rustfs_bucket_replication_target_sent_count\",bucket=~\"$bucket\"},target_arn)",
|
||||
"refId": "PrometheusVariableQueryEditor-target_arn"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(rustfs_scanner_source_work_total,source)",
|
||||
"includeAll": true,
|
||||
"label": "Scanner Source",
|
||||
"multi": true,
|
||||
"name": "scanner_source",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs_scanner_source_work_total,source)",
|
||||
"refId": "PrometheusVariableQueryEditor-scanner_source"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(rustfs_scanner_bucket_drive_result_total,result)",
|
||||
"includeAll": true,
|
||||
"label": "Scanner Result",
|
||||
"multi": true,
|
||||
"name": "scanner_result",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs_scanner_bucket_drive_result_total,result)",
|
||||
"refId": "PrometheusVariableQueryEditor-scanner_result"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -29,27 +29,11 @@ processors:
|
||||
limit_mib: 1024
|
||||
spike_limit_mib: 256
|
||||
transform/logs:
|
||||
error_mode: ignore
|
||||
log_statements:
|
||||
- context: log
|
||||
statements:
|
||||
- set(attributes["message"], body.string) where IsString(body)
|
||||
- set(attributes["log.body"], body.string) where IsString(body)
|
||||
- set(body, Concat([body, " event=", attributes["event"]], "")) where IsString(body) and attributes["event"] != nil
|
||||
- set(body, Concat([body, " component=", attributes["component"]], "")) where IsString(body) and attributes["component"] != nil
|
||||
- set(body, Concat([body, " subsystem=", attributes["subsystem"]], "")) where IsString(body) and attributes["subsystem"] != nil
|
||||
- set(body, Concat([body, " state=", attributes["state"]], "")) where IsString(body) and attributes["state"] != nil
|
||||
- set(body, Concat([body, " result=", attributes["result"]], "")) where IsString(body) and attributes["result"] != nil
|
||||
- set(body, Concat([body, " reason=", attributes["reason"]], "")) where IsString(body) and attributes["reason"] != nil
|
||||
- set(body, Concat([body, " failure_reason=", attributes["failure_reason"]], "")) where IsString(body) and attributes["failure_reason"] != nil
|
||||
- set(body, Concat([body, " rpc_path=", attributes["rpc_path"]], "")) where IsString(body) and attributes["rpc_path"] != nil
|
||||
- set(body, Concat([body, " rpc_service=", attributes["rpc_service"]], "")) where IsString(body) and attributes["rpc_service"] != nil
|
||||
- set(body, Concat([body, " rpc_method=", attributes["rpc_method"]], "")) where IsString(body) and attributes["rpc_method"] != nil
|
||||
- set(body, Concat([body, " expected_audience=", attributes["expected_audience"]], "")) where IsString(body) and attributes["expected_audience"] != nil
|
||||
- set(body, Concat([body, " peer_addr=", attributes["peer_addr"]], "")) where IsString(body) and attributes["peer_addr"] != nil
|
||||
- set(body, Concat([body, " replay_scope_bootstrap_allowed=", attributes["replay_scope_bootstrap_allowed"]], "")) where IsString(body) and attributes["replay_scope_bootstrap_allowed"] != nil
|
||||
- set(body, Concat([body, " error=", attributes["error"]], "")) where IsString(body) and attributes["error"] != nil
|
||||
- set(body, Concat([body, " exception_message=", attributes["exception.message"]], "")) where IsString(body) and attributes["exception.message"] != nil
|
||||
- set(attributes["message"], body.string)
|
||||
- set(attributes["log.body"], body.string)
|
||||
|
||||
exporters:
|
||||
otlp/tempo:
|
||||
|
||||
@@ -17,9 +17,9 @@
|
||||
# =============================================================================
|
||||
#
|
||||
# Metric source: the KMS operation-policy choke point in
|
||||
# crates/kms/src/policy.rs. All label values are bounded static strings
|
||||
# (operation, op_class, outcome, error_class, backend, scope); key identifiers,
|
||||
# key material, and tokens never appear in labels.
|
||||
# crates/kms/src/policy.rs. All label values are static enum strings
|
||||
# (operation, op_class, outcome, error_class); key identifiers, key material,
|
||||
# and tokens never appear in labels.
|
||||
#
|
||||
# Response procedures: docs/operations/kms-observability-runbook.md
|
||||
#
|
||||
@@ -70,9 +70,8 @@ groups:
|
||||
# ------------------------------------------------------------------
|
||||
# 2. KmsBackendHighErrorRate
|
||||
# Sustained share of operations terminating without success
|
||||
# (fatal, budget/deadline exhaustion, admission backpressure,
|
||||
# or an open circuit). The cancelled outcome is excluded because
|
||||
# shutdowns legitimately produce it.
|
||||
# (fatal, budget_exhausted, deadline_exceeded). The cancelled
|
||||
# outcome is excluded because shutdowns legitimately produce it.
|
||||
# The traffic guard keeps a single failure on a near-idle
|
||||
# cluster from firing the alert.
|
||||
# Threshold: 5% for 10m — conservative default, calibrate
|
||||
@@ -95,11 +94,9 @@ groups:
|
||||
summary: "KMS backend non-success ratio above 5% for 10m"
|
||||
description: >-
|
||||
{{ $value | humanizePercentage }} of KMS backend operations
|
||||
are terminating in fatal, budget_exhausted,
|
||||
deadline_exceeded, backpressure_timeout,
|
||||
backpressure_rejected, or circuit_open. Object encryption
|
||||
and decryption paths depending on the KMS are degraded or
|
||||
failing.
|
||||
are terminating in fatal, budget_exhausted, or
|
||||
deadline_exceeded. Object encryption and decryption paths
|
||||
depending on the KMS are degraded or failing.
|
||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendhigherrorrate"
|
||||
|
||||
# ==========================================================================
|
||||
@@ -189,26 +186,3 @@ groups:
|
||||
Retryable failures are outlasting the retry budget, so
|
||||
callers are seeing hard failures.
|
||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendretrybudgetexhausted"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 6. KmsBackendCircuitOpen
|
||||
# Direct circuit-state signal, independent of operation traffic.
|
||||
# A transient open can recover on its first half-open probe; alert
|
||||
# only when the circuit remains open or half-open for one minute.
|
||||
# ------------------------------------------------------------------
|
||||
- alert: KmsBackendCircuitOpen
|
||||
expr: |
|
||||
rustfs_kms_backend_circuit_open > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
component: kms
|
||||
annotations:
|
||||
summary: "KMS backend circuit open ({{ $labels.backend }}/{{ $labels.scope }})"
|
||||
description: >-
|
||||
The KMS backend circuit for {{ $labels.backend }} scope
|
||||
{{ $labels.scope }} has remained open or half-open for one
|
||||
minute. Operations in this scope can terminate as
|
||||
circuit_open until the half-open probe succeeds or returns
|
||||
a non-retryable failure.
|
||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendcircuitopen"
|
||||
|
||||
@@ -24,7 +24,6 @@ on:
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/**'
|
||||
- 'scripts/release/create_or_update_release.sh'
|
||||
- 'scripts/security/check_performance_ab_workflow.sh'
|
||||
- 'scripts/security/check_preview_release_workflow.sh'
|
||||
- 'scripts/security/check_workflow_pins.sh'
|
||||
pull_request:
|
||||
@@ -37,7 +36,6 @@ on:
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/**'
|
||||
- 'scripts/release/create_or_update_release.sh'
|
||||
- 'scripts/security/check_performance_ab_workflow.sh'
|
||||
- 'scripts/security/check_preview_release_workflow.sh'
|
||||
- 'scripts/security/check_workflow_pins.sh'
|
||||
schedule:
|
||||
@@ -143,9 +141,6 @@ jobs:
|
||||
- name: Check preview release workflow policy
|
||||
run: ./scripts/security/check_preview_release_workflow.sh
|
||||
|
||||
- name: Check performance A/B workflow trust boundary
|
||||
run: ./scripts/security/check_performance_ab_workflow.sh
|
||||
|
||||
dependency-review:
|
||||
name: Dependency Review
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -102,9 +102,6 @@ jobs:
|
||||
- name: Check architecture migration rules
|
||||
run: ./scripts/check_architecture_migration_rules.sh
|
||||
|
||||
- name: Check logging guardrails
|
||||
run: ./scripts/check_logging_guardrails.sh
|
||||
|
||||
- name: Check tokio io-uring feature guard
|
||||
run: ./scripts/check_no_tokio_io_uring.sh
|
||||
|
||||
@@ -114,9 +111,6 @@ jobs:
|
||||
- name: Check body-cache whitelist guard
|
||||
run: ./scripts/check_body_cache_whitelist.sh
|
||||
|
||||
- name: Check s3s footprint ratchet
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
|
||||
+13
-72
@@ -137,9 +137,6 @@ jobs:
|
||||
- name: Check architecture migration rules
|
||||
run: ./scripts/check_architecture_migration_rules.sh
|
||||
|
||||
- name: Check logging guardrails
|
||||
run: ./scripts/check_logging_guardrails.sh
|
||||
|
||||
- name: Check tokio io-uring feature guard
|
||||
run: ./scripts/check_no_tokio_io_uring.sh
|
||||
|
||||
@@ -149,9 +146,6 @@ jobs:
|
||||
- name: Check body-cache whitelist guard
|
||||
run: ./scripts/check_body_cache_whitelist.sh
|
||||
|
||||
- name: Check s3s footprint ratchet
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
@@ -346,11 +340,9 @@ jobs:
|
||||
- name: Annotate early-stop reason
|
||||
if: failure() && github.event_name == 'pull_request'
|
||||
run: |
|
||||
{
|
||||
echo "## CI early-stop"
|
||||
echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners."
|
||||
echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "## CI early-stop" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners." >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure." >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# curl rather than `gh`: every existing `gh` call in this repo runs on
|
||||
# ubuntu-latest, and the sm-standard-* images are custom and trimmed (they
|
||||
@@ -673,17 +665,15 @@ jobs:
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
# Build the e2e test graph once. The archive is reused by the security
|
||||
# count-floor check and the smoke run below, avoiding a second compile of
|
||||
# the same e2e_test target on cold runners (backlog#1645).
|
||||
- name: Archive e2e smoke test binaries
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
run: |
|
||||
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
# Guard the security negative-auth smoke subset (backlog#1151 sec-5)
|
||||
# against a rename or deletion silently dropping it out of the e2e-smoke
|
||||
# filter. The script lists what the profile selects and fails if the count
|
||||
# of security auth-rejection tests falls below the committed floor in
|
||||
# .config/security-smoke-floor.txt (infra-12 count-floor mechanism). Run
|
||||
# before the smoke suite so a thinned gate fails fast; the `nextest list`
|
||||
# here compiles the e2e_test binaries the run below reuses.
|
||||
- name: Check security smoke subset count floor
|
||||
run: ./scripts/check_security_smoke_count.sh check
|
||||
|
||||
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
|
||||
# profile.e2e-smoke default-filter in .config/nextest.toml is the single
|
||||
@@ -691,30 +681,7 @@ jobs:
|
||||
# adding new e2e jobs here. Each test spawns its own rustfs server on a
|
||||
# random port and reuses the downloaded debug binary above.
|
||||
- name: Run e2e smoke suite
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
|
||||
run: |
|
||||
cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
|
||||
--status-level all --final-status-level all --failure-output final
|
||||
|
||||
- name: Upload e2e smoke diagnostics
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-smoke-diagnostics-${{ github.run_number }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-smoke-junit-${{ github.run_number }}
|
||||
path: target/nextest/e2e-smoke/junit.xml
|
||||
if-no-files-found: warn
|
||||
run: cargo nextest run --profile e2e-smoke -p e2e_test
|
||||
|
||||
- name: Install s3s-e2e test tool
|
||||
uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2
|
||||
@@ -770,32 +737,6 @@ jobs:
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install awscurl
|
||||
run: |
|
||||
python3 -m pip install --user --upgrade pip "awscurl==0.44"
|
||||
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify awscurl
|
||||
run: test -x "$AWSCURL_PATH"
|
||||
|
||||
- name: Install Vault
|
||||
run: |
|
||||
VAULT_VERSION="1.17.6"
|
||||
VAULT_ARCHIVE="vault_${VAULT_VERSION}_linux_amd64.zip"
|
||||
curl -fsSLo "$RUNNER_TEMP/$VAULT_ARCHIVE" "https://releases.hashicorp.com/vault/${VAULT_VERSION}/${VAULT_ARCHIVE}"
|
||||
echo "0cddc1fbbb88583b5ba5b845f9f8fae47c6fb39a6d48cd543c6ba6fd3ac1a669 $RUNNER_TEMP/$VAULT_ARCHIVE" | sha256sum --check --status
|
||||
unzip -q "$RUNNER_TEMP/$VAULT_ARCHIVE" -d "$RUNNER_TEMP/vault-bin"
|
||||
echo "RUSTFS_TEST_VAULT_BIN=$RUNNER_TEMP/vault-bin/vault" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify Vault
|
||||
run: |
|
||||
"$RUSTFS_TEST_VAULT_BIN" version
|
||||
|
||||
# Download after the cache restore so the freshly built binary from the
|
||||
# build job always wins over anything restored into target/debug.
|
||||
- name: Download debug binary
|
||||
|
||||
@@ -75,7 +75,6 @@ jobs:
|
||||
INTEROP_PACKAGE: rustfs
|
||||
INTEROP_FEATURES: rio-v2
|
||||
INTEROP_FILTER: "test(minio_generated_read_test::)"
|
||||
INTEROP_REQUIRED_TESTS: '["reads_minio_generated_sse_s3_multipart_fixture", "reads_minio_generated_sse_kms_multipart_fixture", "rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key", "rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext"]'
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
@@ -96,23 +95,20 @@ jobs:
|
||||
# is a perfectly valid filterset that matches zero tests, so the next
|
||||
# rename or module move would leave this job selecting nothing and
|
||||
# reporting success without executing a single interop assertion. Count
|
||||
# the selection and require every core reader test, while allowing new
|
||||
# reader cases to be added without changing this guard.
|
||||
# the selection and fail with a reason instead.
|
||||
#
|
||||
# Count only `filter-match.status == "matches"`: the top-level
|
||||
# `test-count` in the JSON is the package total and ignores `-E` entirely.
|
||||
- name: Assert the interop selector still matches tests
|
||||
run: |
|
||||
set -euo pipefail
|
||||
selection="$(cargo nextest list --run-ignored ignored-only \
|
||||
count="$(cargo nextest list --run-ignored all \
|
||||
-p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
|
||||
-E "$INTEROP_FILTER" --message-format json \
|
||||
| python3 -c 'import json,os,sys; d=json.load(sys.stdin); required=json.loads(os.environ["INTEROP_REQUIRED_TESTS"]); matched=[name for suite in d.get("rust-suites", {}).values() for name,test in suite.get("testcases", {}).items() if test.get("filter-match", {}).get("status") == "matches"]; missing=[test for test in required if not any(name.endswith("minio_generated_read_test::" + test) for name in matched)]; print(len(matched)); print(",".join(missing))')"
|
||||
count="$(printf '%s\n' "$selection" | sed -n '1p')"
|
||||
missing="$(printf '%s\n' "$selection" | sed -n '2p')"
|
||||
| python3 -c 'import json,sys; d=json.load(sys.stdin); print(sum(1 for s in d.get("rust-suites", {}).values() for t in s.get("testcases", {}).values() if t.get("filter-match", {}).get("status") == "matches"))')"
|
||||
echo "interop tests selected: ${count}"
|
||||
if [ -n "${missing}" ]; then
|
||||
echo "::error::Selector '${INTEROP_FILTER}' in package '${INTEROP_PACKAGE}' is missing required tests: ${missing}. The MinIO interop reader tests have moved or been renamed; fix the selector instead of running an incomplete matrix. Context: rustfs/backlog#1638."
|
||||
if [ "${count}" -eq 0 ]; then
|
||||
echo "::error::Selector '${INTEROP_FILTER}' in package '${INTEROP_PACKAGE}' matched 0 tests. The MinIO interop reader tests have moved or been renamed again; fix the selector instead of letting this job pass without running them. Context: rustfs/backlog#1638."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@@ -1,463 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Package Workflow - Build DEB/RPM packages
|
||||
#
|
||||
# This workflow builds DEB and RPM packages from pre-built Linux binaries
|
||||
# and uploads them to Cloudflare R2.
|
||||
#
|
||||
# Trigger:
|
||||
# - release published: automatically package when a GitHub release is published
|
||||
# - workflow_dispatch: manual trigger with optional tag/run_id
|
||||
#
|
||||
# Flow:
|
||||
# 1. Find the Build workflow run for the release tag
|
||||
# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
|
||||
# 3. Build DEB packages for amd64 and arm64
|
||||
# 4. Build RPM packages for x86_64 and aarch64
|
||||
# 5. Upload all packages to Cloudflare R2
|
||||
|
||||
name: Package DEB/RPM
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [ published ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Release tag to package (e.g. 1.0.0-beta.12). Leave empty for latest main build."
|
||||
required: false
|
||||
type: string
|
||||
build_run_id:
|
||||
description: "Build workflow run ID (overrides tag lookup)"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.release.tag_name || github.event.inputs.tag || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Resolve which build run to use and extract version info
|
||||
resolve:
|
||||
name: Resolve Build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
version: ${{ steps.resolve.outputs.version }}
|
||||
build_type: ${{ steps.resolve.outputs.build_type }}
|
||||
build_run_id: ${{ steps.resolve.outputs.build_run_id }}
|
||||
tag: ${{ steps.resolve.outputs.tag }}
|
||||
steps:
|
||||
- name: Resolve build run
|
||||
id: resolve
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
INPUT_TAG: ${{ github.event.inputs.tag }}
|
||||
INPUT_RUN_ID: ${{ github.event.inputs.build_run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Determine tag
|
||||
if [[ "${{ github.event_name }}" == "release" ]]; then
|
||||
TAG="${{ github.event.release.tag_name }}"
|
||||
elif [[ -n "$INPUT_TAG" ]]; then
|
||||
TAG="$INPUT_TAG"
|
||||
else
|
||||
TAG=""
|
||||
fi
|
||||
|
||||
echo "Tag: ${TAG:-<none>}"
|
||||
|
||||
# Determine build run ID
|
||||
BUILD_RUN_ID=""
|
||||
|
||||
if [[ -n "$INPUT_RUN_ID" ]]; then
|
||||
# Explicit run ID takes priority
|
||||
BUILD_RUN_ID="$INPUT_RUN_ID"
|
||||
echo "Using explicit build run ID: $BUILD_RUN_ID"
|
||||
|
||||
elif [[ -n "$TAG" ]]; then
|
||||
# Find the build run that produced this tag
|
||||
echo "Looking for build run for tag: $TAG"
|
||||
BUILD_RUN_ID=$(gh api \
|
||||
"repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=${TAG}&status=success&per_page=1" \
|
||||
--jq '.workflow_runs[0].id' 2>/dev/null || echo "")
|
||||
|
||||
if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
|
||||
# Tag might not be a branch; try event=push with head_branch matching
|
||||
BUILD_RUN_ID=$(gh api \
|
||||
"repos/${{ github.repository }}/actions/workflows/build.yml/runs?event=push&status=success&per_page=100" \
|
||||
--jq ".workflow_runs[] | select(.head_branch == \"$TAG\") | .id" 2>/dev/null | head -1 || echo "")
|
||||
fi
|
||||
|
||||
if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
|
||||
echo "❌ No successful build run found for tag: $TAG"
|
||||
exit 1
|
||||
fi
|
||||
echo "Found build run: $BUILD_RUN_ID"
|
||||
|
||||
else
|
||||
# No tag — latest successful main build
|
||||
echo "No tag specified, looking for latest main build"
|
||||
BUILD_RUN_ID=$(gh api \
|
||||
"repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=main&status=success&per_page=1" \
|
||||
--jq '.workflow_runs[0].id' 2>/dev/null || echo "")
|
||||
|
||||
if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
|
||||
echo "❌ No successful main build found"
|
||||
exit 1
|
||||
fi
|
||||
echo "Latest main build: $BUILD_RUN_ID"
|
||||
fi
|
||||
|
||||
# Determine version and build type
|
||||
if [[ -n "$TAG" ]]; then
|
||||
VERSION="$TAG"
|
||||
if [[ "$TAG" == *"-preview"* ]]; then
|
||||
BUILD_TYPE="preview"
|
||||
elif [[ "$TAG" == *"alpha"* || "$TAG" == *"beta"* || "$TAG" == *"rc"* ]]; then
|
||||
BUILD_TYPE="prerelease"
|
||||
else
|
||||
BUILD_TYPE="release"
|
||||
fi
|
||||
else
|
||||
SHORT_SHA=$(gh api "repos/${{ github.repository }}/actions/runs/${BUILD_RUN_ID}" \
|
||||
--jq '.head_sha' 2>/dev/null | head -c 7)
|
||||
VERSION="dev-${SHORT_SHA}"
|
||||
BUILD_TYPE="development"
|
||||
fi
|
||||
|
||||
{
|
||||
echo "version=$VERSION"
|
||||
echo "build_type=$BUILD_TYPE"
|
||||
echo "build_run_id=$BUILD_RUN_ID"
|
||||
echo "tag=${TAG}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "📊 Resolved:"
|
||||
echo " Version: $VERSION"
|
||||
echo " Build type: $BUILD_TYPE"
|
||||
echo " Build run ID: $BUILD_RUN_ID"
|
||||
|
||||
# Build DEB and RPM packages for each architecture
|
||||
package:
|
||||
name: Package (${{ matrix.arch }})
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- arch: x86_64
|
||||
deb_arch: amd64
|
||||
rpm_arch: x86_64
|
||||
artifact_name: "rustfs-linux-x86_64-gnu"
|
||||
- arch: aarch64
|
||||
deb_arch: arm64
|
||||
rpm_arch: aarch64
|
||||
artifact_name: "rustfs-linux-aarch64-gnu"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download binary artifact from build run
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
||||
with:
|
||||
pattern: ${{ matrix.artifact_name }}*
|
||||
path: ./binary-artifact
|
||||
run-id: ${{ needs.resolve.outputs.build_run_id }}
|
||||
github-token: ${{ github.token }}
|
||||
merge-multiple: true
|
||||
|
||||
- name: Extract binary
|
||||
id: binary
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
ZIP_FILE=$(find ./binary-artifact -name "*.zip" -type f | head -1)
|
||||
if [[ -z "$ZIP_FILE" ]]; then
|
||||
echo "❌ No binary artifact found"
|
||||
ls -la ./binary-artifact/ || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Found artifact: $ZIP_FILE"
|
||||
|
||||
mkdir -p ./bin
|
||||
unzip -o "$ZIP_FILE" -d ./bin
|
||||
|
||||
if [[ ! -f ./bin/rustfs ]]; then
|
||||
echo "❌ rustfs binary not found in archive"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
chmod +x ./bin/rustfs
|
||||
ls -lh ./bin/rustfs
|
||||
echo "✅ Binary extracted"
|
||||
|
||||
- name: Build DEB package
|
||||
id: deb
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
VERSION="${{ needs.resolve.outputs.version }}"
|
||||
DEB_ARCH="${{ matrix.deb_arch }}"
|
||||
# DEB version: replace - with ~ (1.0.0-beta.12 -> 1.0.0~beta.12)
|
||||
DEB_VERSION="${VERSION/-/~}"
|
||||
PKG_DIR="rustfs_${DEB_VERSION}_${DEB_ARCH}"
|
||||
|
||||
echo "Building DEB: ${PKG_DIR}.deb"
|
||||
|
||||
mkdir -p "${PKG_DIR}/DEBIAN"
|
||||
mkdir -p "${PKG_DIR}/usr/bin"
|
||||
mkdir -p "${PKG_DIR}/etc/default"
|
||||
mkdir -p "${PKG_DIR}/lib/systemd/system"
|
||||
mkdir -p "${PKG_DIR}/usr/share/doc/rustfs"
|
||||
|
||||
cp ./bin/rustfs "${PKG_DIR}/usr/bin/"
|
||||
chmod 755 "${PKG_DIR}/usr/bin/rustfs"
|
||||
|
||||
cp deploy/build/rustfs.service "${PKG_DIR}/lib/systemd/system/"
|
||||
|
||||
cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
cat > "${PKG_DIR}/DEBIAN/control" << EOF
|
||||
Package: rustfs
|
||||
Version: ${DEB_VERSION}
|
||||
Section: utils
|
||||
Priority: optional
|
||||
Architecture: ${DEB_ARCH}
|
||||
Depends: libc6 (>= 2.31)
|
||||
Maintainer: RustFS Team <support@rustfs.com>
|
||||
Description: High-performance distributed object storage
|
||||
RustFS is a high-performance distributed object storage software
|
||||
built using Rust. It is compatible with MinIO and S3 API.
|
||||
Homepage: https://rustfs.com
|
||||
EOF
|
||||
|
||||
cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs"
|
||||
POSTINST
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postinst"
|
||||
|
||||
cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/prerm"
|
||||
|
||||
cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postrm"
|
||||
|
||||
cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
|
||||
cp README.md "${PKG_DIR}/usr/share/doc/rustfs/"
|
||||
|
||||
fakeroot dpkg-deb --build "${PKG_DIR}"
|
||||
|
||||
DEB_FILE="${PKG_DIR}.deb"
|
||||
ls -lh "$DEB_FILE"
|
||||
echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ DEB built: $DEB_FILE"
|
||||
|
||||
- name: Build RPM package
|
||||
id: rpm
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
VERSION="${{ needs.resolve.outputs.version }}"
|
||||
RPM_ARCH="${{ matrix.rpm_arch }}"
|
||||
|
||||
echo "Building RPM for ${RPM_ARCH}"
|
||||
|
||||
sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential
|
||||
sudo gem install fpm
|
||||
|
||||
fpm -s dir -t rpm \
|
||||
--name rustfs \
|
||||
--version "$VERSION" \
|
||||
--architecture "$RPM_ARCH" \
|
||||
--depends "glibc >= 2.31" \
|
||||
--maintainer "RustFS Team <support@rustfs.com>" \
|
||||
--description "High-performance distributed object storage" \
|
||||
--url "https://rustfs.com" \
|
||||
--license "Apache-2.0" \
|
||||
--after-install <(cat <<'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTINST
|
||||
) \
|
||||
--before-remove <(cat <<'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
) \
|
||||
--after-remove <(cat <<'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
) \
|
||||
--config-files /etc/default/rustfs \
|
||||
./bin/rustfs=/usr/bin/rustfs \
|
||||
deploy/build/rustfs.service=/lib/systemd/system/rustfs.service \
|
||||
LICENSE=/usr/share/doc/rustfs/LICENSE \
|
||||
README.md=/usr/share/doc/rustfs/README.md
|
||||
|
||||
RPM_FILE=$(ls -1 rustfs-*.rpm 2>/dev/null | head -1)
|
||||
if [[ -z "$RPM_FILE" ]]; then
|
||||
echo "❌ RPM build failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ls -lh "$RPM_FILE"
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ RPM built: $RPM_FILE"
|
||||
|
||||
- name: Upload packages to artifacts
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: packages-${{ matrix.arch }}
|
||||
path: |
|
||||
*.deb
|
||||
*.rpm
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload packages to Cloudflare R2
|
||||
if: env.R2_ACCESS_KEY_ID != ''
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "⚠️ R2 credentials missing, skipping upload"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
BUILD_TYPE="${{ needs.resolve.outputs.build_type }}"
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PREFIX="artifacts/rustfs/packages/dev"
|
||||
else
|
||||
R2_PREFIX="artifacts/rustfs/packages/release"
|
||||
fi
|
||||
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
|
||||
|
||||
echo "📤 Uploading to $R2_PATH"
|
||||
|
||||
DEB_FILE="${{ steps.deb.outputs.deb_file }}"
|
||||
RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
|
||||
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading: $f"
|
||||
aws s3 cp "$f" "$R2_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
|
||||
echo "✅ Upload complete"
|
||||
|
||||
# Also upload as latest for release/prerelease
|
||||
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading latest: $(basename "$f")"
|
||||
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
needs: [ resolve, package ]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Print summary
|
||||
shell: bash
|
||||
run: |
|
||||
echo "## 📦 Package Summary" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Item | Value |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "|------|-------|" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Version | \`${{ needs.resolve.outputs.version }}\` |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Package Status | ${{ needs.package.result }} |" >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -17,10 +17,10 @@
|
||||
# Two entry points, honestly scoped:
|
||||
# * schedule (nightly, on main): post-merge detection — catches a regression
|
||||
# within 24h of landing, not before merge.
|
||||
# * workflow_dispatch: an explicitly selected trusted ref.
|
||||
# The dispatch input can run the gate with --allow-regression so a deliberate
|
||||
# correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
|
||||
# blocked (rustfs/backlog#935 correction 1).
|
||||
# * pull_request labeled `perf-ab`: opt-in pre-merge gate for a specific PR.
|
||||
# The `perf-deliberate-tradeoff` label runs the gate with --allow-regression so
|
||||
# a deliberate correctness cost (e.g. the #4221 fsync durability fix) is
|
||||
# recorded but does not block (rustfs/backlog#935 correction 1).
|
||||
|
||||
# DISABLED. This workflow is switched off in the repository's Actions settings
|
||||
# (state: disabled_manually) and does not run on any trigger, including its cron
|
||||
@@ -46,6 +46,8 @@ on:
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
pull_request:
|
||||
types: [labeled, synchronize, reopened]
|
||||
push:
|
||||
# Every main commit pre-builds and caches its release binary (perf-3) so the
|
||||
# nightly A/B restores a ready baseline instead of paying the double build.
|
||||
@@ -53,6 +55,14 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
# Per-PR: a new push cancels the previous (up to 90-minute) A/B run instead of
|
||||
# stacking them. Nightly schedule and manual dispatch get a unique group and
|
||||
# always run to completion.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
@@ -60,8 +70,8 @@ env:
|
||||
|
||||
jobs:
|
||||
# perf-3: on every push to main, build the release binary once and cache it
|
||||
# keyed by commit SHA (rustfs-baseline-<sha>). The warp-ab measurements
|
||||
# restore this instead of paying the ~32min-per-side source
|
||||
# keyed by commit SHA (rustfs-baseline-<sha>). The nightly A/B (and, later, the
|
||||
# perf-7 PR gate) restore this instead of paying the ~32min-per-side source
|
||||
# build. That double build is what pushed the expanded 24-cell nightly past its
|
||||
# ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
|
||||
# builds off the shared cargo cache keep each push cheap, and building on the
|
||||
@@ -116,11 +126,17 @@ jobs:
|
||||
|
||||
warp-ab:
|
||||
name: Warp A/B budget gate
|
||||
# Always run on schedule / manual dispatch. Never on push — that event only
|
||||
# feeds build-baseline-cache above.
|
||||
# Always run on schedule / manual dispatch. Opt-in on PRs: only when the
|
||||
# `perf-ab` label is present, and for `labeled` events only when the label
|
||||
# being added is `perf-ab` itself (adding an unrelated label to an opted-in
|
||||
# PR must not re-run the gate). Never on push — that event only feeds
|
||||
# build-baseline-cache above.
|
||||
if: >-
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch'
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
contains(github.event.pull_request.labels.*.name, 'perf-ab') &&
|
||||
(github.event.action != 'labeled' || github.event.label.name == 'perf-ab'))
|
||||
runs-on: sm-standard-2
|
||||
# With perf-3's cached baseline binary the common (cache-hit) nightly is
|
||||
# measurement-only and finishes well under 50min. This ceiling stays
|
||||
@@ -158,6 +174,10 @@ jobs:
|
||||
INPUT_ALLOW_REGRESSION: ${{ github.event.inputs.allow_regression }}
|
||||
run: |
|
||||
allow="false"
|
||||
if [[ "${{ github.event_name }}" == "pull_request" ]] \
|
||||
&& ${{ contains(github.event.pull_request.labels.*.name, 'perf-deliberate-tradeoff') }}; then
|
||||
allow="true"
|
||||
fi
|
||||
if [[ "$INPUT_ALLOW_REGRESSION" == "true" ]]; then
|
||||
allow="true"
|
||||
fi
|
||||
@@ -294,10 +314,10 @@ jobs:
|
||||
echo "candidate binary: $cand_src"
|
||||
|
||||
if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then
|
||||
args+=(--allow-regression --exemption-reason "workflow dispatch override")
|
||||
args+=(--allow-regression --exemption-reason "labeled perf-deliberate-tradeoff / dispatch override")
|
||||
fi
|
||||
# Do not let a gate FAIL abort the job here; capture status and surface
|
||||
# it after the step summary is written.
|
||||
# it after the PR comment is posted.
|
||||
set +e
|
||||
bash scripts/run_hotpath_warp_abba.sh "${args[@]}"
|
||||
echo "status=$?" >> "$GITHUB_OUTPUT"
|
||||
@@ -362,6 +382,13 @@ jobs:
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Comment gate result on PR
|
||||
if: always() && github.event_name == 'pull_request' && steps.ab.outputs.gate_md != ''
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
gh pr comment "${{ github.event.pull_request.number }}" --body-file "${{ steps.ab.outputs.gate_md }}"
|
||||
|
||||
# Scheduled failure alerting is handled by the alert-on-failure job below
|
||||
# (perf-2 consuming ci-8's schedule-failure-issue composite action).
|
||||
|
||||
@@ -370,7 +397,7 @@ jobs:
|
||||
run: |
|
||||
status="${{ steps.ab.outputs.status }}"
|
||||
if [[ "$status" != "0" ]]; then
|
||||
echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
|
||||
echo "::error::warp A/B budget gate failed (exit $status). See the step summary / PR comment / gate.md artifact." >&2
|
||||
exit "$status"
|
||||
fi
|
||||
echo "warp A/B budget gate passed."
|
||||
@@ -380,12 +407,14 @@ jobs:
|
||||
needs: [warp-ab]
|
||||
# `always()` is required: without it this job is skipped when a needed
|
||||
# job fails. Alerts only for scheduled (nightly) runs (backlog#1149
|
||||
# ci-8); manual dispatch failures are already watched by a human.
|
||||
# ci-8); PR and manual dispatch failures are already watched by a human.
|
||||
# `cancelled` is included alongside `failure` on purpose: a job that hits
|
||||
# timeout-minutes ends as `cancelled`, and the 2026-07-11..07-14 nightly
|
||||
# timeouts went silent precisely because the guard was failure-only. The
|
||||
# composite action already reports cancelled/timed-out jobs in the issue
|
||||
# body.
|
||||
# body. (Scheduled runs get a unique concurrency group with
|
||||
# cancel-in-progress off, so a cancellation here means a timeout/manual
|
||||
# abort, never a superseding run.)
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Windows Filesystem Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- "crates/ecstore/src/disk/**"
|
||||
- "crates/ecstore/src/store/init_format.rs"
|
||||
- "crates/ecstore/Cargo.toml"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- ".github/actions/setup/**"
|
||||
- ".github/workflows/windows-filesystem.yml"
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- "crates/ecstore/src/disk/**"
|
||||
- "crates/ecstore/src/store/init_format.rs"
|
||||
- "crates/ecstore/Cargo.toml"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- ".github/actions/setup/**"
|
||||
- ".github/workflows/windows-filesystem.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
rename-safety:
|
||||
name: Rename Safety
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: build-x86_64-pc-windows-msvc
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Test guarded rename publication
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib rename_all_ -- --nocapture
|
||||
|
||||
- name: Test Windows handle guards
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib windows_ -- --nocapture
|
||||
|
||||
- name: Test startup temporary-directory cleanup
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib cleanup_tmp_on_startup_ -- --nocapture
|
||||
|
||||
- name: Test fresh format publication
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib fresh_format_load_initializes_all_disks -- --nocapture
|
||||
@@ -83,7 +83,3 @@ worktrees/*
|
||||
|
||||
# Local AI-agent review artifacts (omo evidence dumps)
|
||||
.omo/
|
||||
|
||||
# insta scratch files; the accepted .snap files ARE the assertions and are committed
|
||||
*.snap.new
|
||||
*.pending-snap
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
---
|
||||
name: issue-triage
|
||||
description: Triage a GitHub issue — determine if it is already fixed, needs implementation, or should be closed. Searches related commits and PRs, verifies implementation status, and posts a triage comment or closes the issue. Use when the user provides an issue URL and asks whether it can be closed or needs work.
|
||||
---
|
||||
|
||||
# Issue Triage
|
||||
|
||||
Use this skill when the user provides a GitHub issue URL and asks "can this be closed?", "is this already implemented?", "check completion status", or similar triage questions.
|
||||
|
||||
## Workflow
|
||||
|
||||
### 1. Fetch issue context
|
||||
|
||||
```bash
|
||||
gh issue view <N> --repo <owner/repo> --json title,body,state,comments,labels,updatedAt
|
||||
```
|
||||
|
||||
Read the issue body to understand what was requested. Extract:
|
||||
- The specific feature/fix/behavior described.
|
||||
- Any linked PRs or commits mentioned in the body or comments.
|
||||
- Any checklist items or sub-issues.
|
||||
|
||||
### 2. Search for related work
|
||||
|
||||
Search git history for commits referencing the issue:
|
||||
```bash
|
||||
git log --oneline --all --grep="<N>" | head -30
|
||||
```
|
||||
|
||||
Search for related PRs:
|
||||
```bash
|
||||
gh pr list --search "fixes #<N> OR closes #<N> OR #<N>" --state all --json number,title,state,mergedAt
|
||||
```
|
||||
|
||||
If the issue mentions specific PRs, check their status:
|
||||
```bash
|
||||
gh pr view <PR_N> --json state,mergedAt,title
|
||||
```
|
||||
|
||||
### 3. Verify implementation
|
||||
|
||||
For each linked or related PR that is merged, verify the fix is actually present on the current main branch:
|
||||
```bash
|
||||
git log --oneline main | grep -i "<keyword>"
|
||||
# or
|
||||
git log --oneline main --grep="<PR_N>"
|
||||
```
|
||||
|
||||
If the issue describes a specific defect, check the relevant code to confirm the fix is in place:
|
||||
```bash
|
||||
grep -n "<pattern>" crates/<relevant>/src/<file>.rs
|
||||
```
|
||||
|
||||
For issues with checklists, verify each item individually. If sub-items are tracked as separate issues, check those too:
|
||||
```bash
|
||||
gh issue view <SUB_N> --repo <owner/repo> --json state
|
||||
```
|
||||
|
||||
### 4. Determine verdict
|
||||
|
||||
- **All items fixed and merged**: Close with a summary comment listing what was fixed and which PRs.
|
||||
- **Some items fixed, some remaining**: Comment with status of each item. Do not close.
|
||||
- **Not yet implemented**: Comment with a summary of what remains. Do not close.
|
||||
- **Superseded or no longer relevant**: Close with explanation.
|
||||
|
||||
### 5. Take action
|
||||
|
||||
Close with comment:
|
||||
```bash
|
||||
gh issue close <N> --repo <owner/repo> --comment "<body>"
|
||||
```
|
||||
|
||||
Comment without closing:
|
||||
```bash
|
||||
gh issue comment <N> --repo <owner/repo> --body-file /tmp/triage.md
|
||||
```
|
||||
|
||||
Update issue labels if needed:
|
||||
```bash
|
||||
gh issue edit <N> --repo <owner/repo> --add-label "completed" --remove-label "needs-triage"
|
||||
```
|
||||
|
||||
Always use `--body-file` for multiline content, never inline `--body`.
|
||||
|
||||
### 6. Handle multi-issue batches
|
||||
|
||||
When the user asks to check multiple issues (e.g., "check all issues by user X" or "scan backlog for closable issues"):
|
||||
1. List the issues: `gh issue list --repo <repo> --author <user> --state open --json number,title,updatedAt`
|
||||
2. For each issue, run steps 1-5 above.
|
||||
3. Report a summary table of all triaged issues with verdicts.
|
||||
|
||||
## Output format
|
||||
|
||||
### Issue Triage: #<N> — <title>
|
||||
|
||||
**State**: OPEN / CLOSED
|
||||
**Linked PRs**: <list with merge status>
|
||||
|
||||
#### Assessment
|
||||
<what was requested vs what is implemented>
|
||||
|
||||
#### Verdict
|
||||
- Close — all items resolved by <PR list>
|
||||
- Keep open — <remaining items>
|
||||
- Not started — <what needs to be done>
|
||||
|
||||
#### Action taken
|
||||
- Closed with comment / Commented / No action
|
||||
|
||||
## Notes
|
||||
|
||||
- The user may ask in Chinese ("是否可以关闭", "检查完成情况"); respond in the same language.
|
||||
- When closing, always include a summary of what was fixed and which PRs resolved it — this creates a useful audit trail.
|
||||
- For issues in `rustfs/backlog`, use `--repo rustfs/backlog`.
|
||||
- For issues in `rustfs/rustfs`, use `--repo rustfs/rustfs`.
|
||||
- If the issue has sub-issues (GitHub sub-issues API), check each one's state before declaring the parent complete.
|
||||
@@ -1,147 +0,0 @@
|
||||
---
|
||||
name: pr-review
|
||||
description: Review a GitHub PR end-to-end from a URL or number — fetch metadata, inspect the diff, run multi-role adversarial review, check CI status, and post the review comment. Use when the user provides a PR link and asks to review it.
|
||||
---
|
||||
|
||||
# PR Review
|
||||
|
||||
Use this skill when the user provides a GitHub PR URL or number and asks to review it. This covers the full review lifecycle: data gathering, code review, CI verification, and posting the result.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Read `AGENTS.md` for the repository's adversarial validation policy and change-style rules.
|
||||
- The `adversarial-validation` skill handles the review role playbooks; this skill orchestrates the workflow around it.
|
||||
|
||||
## Workflow
|
||||
|
||||
### 1. Gather PR context
|
||||
|
||||
```bash
|
||||
gh pr view <N> --json title,author,state,body,additions,deletions,changedFiles,commits,baseRefName,headRefName
|
||||
gh pr diff <N> --name-only
|
||||
```
|
||||
|
||||
Read the PR body and linked issues to understand the change's purpose. If the PR references an issue, fetch that too:
|
||||
```bash
|
||||
gh issue view <ISSUE> --json title,body,state
|
||||
```
|
||||
|
||||
### 2. Fetch the diff and classify the change
|
||||
|
||||
```bash
|
||||
git fetch origin pull/<N>/head:pr-<N>
|
||||
git diff main...pr-<N> --stat
|
||||
```
|
||||
|
||||
Classify the change by risk tier (per AGENTS.md):
|
||||
- **Exempt**: docs/comments/instruction-only, formatting, typos.
|
||||
- **Mechanical**: renames, file moves, test-only or tooling changes.
|
||||
- **Standard** (default): any behavior change.
|
||||
- **High risk**: locking, erasure coding, quorum/heal, replication, multipart, RPC, lifecycle/tiering, metadata formats, persistence/fsync, IAM/KMS/auth, on-disk/on-wire formats, S3 API-visible behavior.
|
||||
|
||||
### 3. Cluster changed files and delegate review
|
||||
|
||||
Group the changed files into logical clusters (by crate or functional area). For each cluster, spawn a subagent with a focused review prompt that includes:
|
||||
- The cluster's changed files and their diffs.
|
||||
- The applicable adversarial role probes (from the `adversarial-validation` skill).
|
||||
- The repository's AGENTS.md rules relevant to that domain.
|
||||
|
||||
For standard-tier changes: correctness adversary + simplicity adversary + test-coverage skeptic, plus every role whose domain the diff touches.
|
||||
For high-risk changes: run all seven roles.
|
||||
|
||||
Each subagent must produce findings (concrete failure scenario with file:line) or a null report ("attacked X, Y, Z — no break found").
|
||||
|
||||
### 4. Check CI status
|
||||
|
||||
```bash
|
||||
gh pr checks <N>
|
||||
```
|
||||
|
||||
If any checks fail, investigate:
|
||||
```bash
|
||||
gh run view --log-failed --job=<JOB_ID>
|
||||
```
|
||||
|
||||
Determine whether failures are pre-existing (on main), flaky, or caused by the PR.
|
||||
|
||||
### 5. Synthesize findings
|
||||
|
||||
Combine all subagent findings into a structured review:
|
||||
- **Summary**: one-paragraph overview of the change and overall assessment.
|
||||
- **Findings**: each finding with severity (critical/major/minor/nit), file:line, concrete failure scenario, and suggested fix.
|
||||
- **CI status**: pass/fail with notes on any failures.
|
||||
- **Verdict**: APPROVE, REQUEST_CHANGES, or COMMENT.
|
||||
|
||||
### 6. Post the review
|
||||
|
||||
Write the review body to a temp file and post via CLI:
|
||||
```bash
|
||||
# Request changes
|
||||
gh pr review <N> --request-changes --body-file /tmp/pr_review.md
|
||||
|
||||
# Approve
|
||||
gh pr review <N> --approve --body-file /tmp/pr_review.md
|
||||
|
||||
# Comment only (no verdict)
|
||||
gh pr review <N> --comment --body-file /tmp/pr_review.md
|
||||
```
|
||||
|
||||
For inline comments on specific lines, use the GitHub API:
|
||||
```bash
|
||||
cat > /tmp/pr_review.json <<'EOF'
|
||||
{
|
||||
"body": "review body",
|
||||
"event": "REQUEST_CHANGES",
|
||||
"comments": [
|
||||
{
|
||||
"path": "crates/foo/src/bar.rs",
|
||||
"line": 42,
|
||||
"body": "finding description"
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
gh api --method POST /repos/{owner}/{repo}/pulls/<N>/reviews --input /tmp/pr_review.json
|
||||
```
|
||||
|
||||
Always use `--body-file` or `--input`, never inline multiline `--body`.
|
||||
|
||||
### 7. Handle follow-up
|
||||
|
||||
If the review requests changes:
|
||||
- Monitor for new commits: `gh pr view <N> --json commits`
|
||||
- Re-review changed files only: `git diff pr-<N>..origin/pull/<N>/head`
|
||||
- Update the review when findings are addressed.
|
||||
|
||||
If CI was failing due to pre-existing main breakage:
|
||||
- Comment on the PR noting the failure is pre-existing.
|
||||
- Suggest updating the branch: `gh pr update-branch <N>`
|
||||
|
||||
## Output format
|
||||
|
||||
### PR Review: #<N> — <title>
|
||||
|
||||
**Author**: <author>
|
||||
**Risk tier**: exempt | mechanical | standard | high-risk
|
||||
**Changed files**: <count> across <cluster count> clusters
|
||||
|
||||
#### Summary
|
||||
<one-paragraph overview>
|
||||
|
||||
#### Findings
|
||||
| Severity | Location | Finding |
|
||||
|----------|----------|---------|
|
||||
| critical | file:line | concrete failure scenario |
|
||||
|
||||
#### CI Status
|
||||
- All checks pass / Failing: <details>
|
||||
|
||||
#### Verdict
|
||||
APPROVE / REQUEST_CHANGES / COMMENT
|
||||
|
||||
## Notes
|
||||
|
||||
- The user may ask for review in Chinese; respond in the same language but keep the review body in English per AGENTS.md rules.
|
||||
- When the user asks for "多角色对抗 review", run the full adversarial validation protocol — this skill's step 3 covers that.
|
||||
- If the PR is from a fork, check `maintainerCanModify` before attempting to push fixes.
|
||||
- For very large PRs (>50 files), cluster aggressively and delegate in parallel to keep review time reasonable.
|
||||
@@ -322,28 +322,6 @@ High risk: all seven roles.
|
||||
- Use environment variables or vault tooling for sensitive configuration.
|
||||
- For localhost-sensitive tests, verify proxy settings to avoid traffic leakage.
|
||||
|
||||
## Logging
|
||||
|
||||
Applies to **every** `tracing` macro you add or edit, including a single line
|
||||
added in passing while fixing something else — not only to log-focused changes.
|
||||
|
||||
- Fields first, message second: `event`, `component`, `subsystem`,
|
||||
`result`/`state`, then key context. The message is a short label, not a
|
||||
sentence with values interpolated into it.
|
||||
- Reuse the existing `EVENT_*` / `LOG_COMPONENT_*` / `LOG_SUBSYSTEM_*`
|
||||
constants of the module you are editing; match the shape of the log sites
|
||||
already in that file rather than introducing a second style next to them.
|
||||
- Level policy: `error` for behavior/security-affecting failures, `warn` for
|
||||
degraded or fallback paths, `info` for low-frequency lifecycle, `debug` for
|
||||
targeted diagnostics, `trace` for hot paths. Per-object and per-request
|
||||
success paths are `trace`.
|
||||
- Never log secrets, tokens, credential payloads, or merged config dumps.
|
||||
- `scripts/check_logging_guardrails.sh` enforces a subset of this on the files
|
||||
it lists; passing it is a floor, not evidence the log matches the house style.
|
||||
|
||||
See `.agents/skills/rustfs-logging-governance/SKILL.md` for the full event
|
||||
model, level policy, and guardrail-update checklist.
|
||||
|
||||
## Tools
|
||||
|
||||
### xl.meta decode tool Quick Use
|
||||
@@ -369,11 +347,6 @@ cargo run -p rustfs-filemeta --example dump_fileinfo -- "/path/to/file/xl.meta"
|
||||
absent, empty, and nil all mean "no value", never `Uuid::nil()`.
|
||||
- A remote-tier version of `None`/`""` means the tier bucket is unversioned:
|
||||
send **no** `versionId` on tier GET/DELETE.
|
||||
- Structs persisted in the scanner data-usage cache (`DataUsageCacheInfo`,
|
||||
`DataUsageEntry`) carry a hand-written map-encoded `Serialize`. MessagePack
|
||||
encodes derived structs as arrays, where an appended field makes the whole
|
||||
cache a decode error for older readers — keep new fields `#[serde(default)]`
|
||||
and keep the map encoding rather than reverting to `derive(Serialize)`.
|
||||
|
||||
## Naming Conventions
|
||||
|
||||
|
||||
Generated
+373
-408
File diff suppressed because it is too large
Load Diff
+61
-60
@@ -69,7 +69,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-rc.1"
|
||||
version = "1.0.0-beta.12"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -86,52 +86,52 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.1" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.1" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.1" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.1" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.1" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.1" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.1" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.1" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.1" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.1" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.1" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.1" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.1" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.1" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.1" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.1" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.1" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.1" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.1", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.1" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.1" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.1" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.1" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.1" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.1" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.1" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.1" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.1" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.1" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.1" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.1" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.1" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.1" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.1" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.1" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.1" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.1" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.1" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.1" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.1" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.1" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.1" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-beta.12" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.12" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.12" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.12" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-beta.12" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.12" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.12" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.12" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.12" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.12" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.12" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.12" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.12" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.12" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.12" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.12" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.12" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.12" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.12" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.12" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.12" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.12" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.12" }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.12" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.12" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.12" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.12" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.12" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.12" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.12" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.12" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.12" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.12" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.12" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.12" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.12" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.12" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.12" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.12" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.12" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.12" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.12" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.12" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.12" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.12" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.12" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -139,7 +139,7 @@ async_zip = { default-features = false, version = "0.0.18" }
|
||||
mysql_async = { default-features = false, version = "0.37" }
|
||||
async-compression = { version = "0.4.43" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.92"
|
||||
async-trait = "0.1.91"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.33"
|
||||
@@ -173,7 +173,7 @@ tower-http = { version = "0.7.0" }
|
||||
# Serialization and Data Formats
|
||||
apache-avro = "0.21.0"
|
||||
bytes = { version = "1.12.1" }
|
||||
bytesize = "2.7.0"
|
||||
bytesize = "2.6.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
form_urlencoded = "1.2.2"
|
||||
@@ -212,7 +212,7 @@ zeroize = { version = "1.9.0" }
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.35" }
|
||||
time = { version = "0.3.55" }
|
||||
time = { version = "0.3.54" }
|
||||
|
||||
# Database
|
||||
deadpool-postgres = { version = "0.14" }
|
||||
@@ -228,15 +228,15 @@ atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.10.1" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.141.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.140.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
|
||||
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||
aws-smithy-types = { version = "1.6.1" }
|
||||
base64 = "0.23.1"
|
||||
base64 = "0.23.0"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.4"
|
||||
clap = { version = "4.6.6" }
|
||||
clap = { version = "4.6.5" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.11.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -244,7 +244,7 @@ crossbeam-queue = "0.3.13"
|
||||
crossbeam-channel = "0.5.16"
|
||||
crossbeam-deque = "0.8.7"
|
||||
crossbeam-utils = "0.8.22"
|
||||
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
|
||||
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" }
|
||||
#datafusion = { default-features = false, version = "54.1.0" }
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.14"
|
||||
@@ -274,6 +274,7 @@ num_cpus = { version = "1.17.0" }
|
||||
nvml-wrapper = "0.12.1"
|
||||
parking_lot = "0.12.5"
|
||||
path-absolutize = "4.0.1"
|
||||
path-clean = "1.0.1"
|
||||
percent-encoding = "2.3.2"
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
@@ -302,7 +303,7 @@ sysinfo = "0.39.6"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.3.1"
|
||||
thiserror = "2.0.20"
|
||||
thiserror = "2.0.19"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
tracing-core = "0.1.36"
|
||||
@@ -341,20 +342,20 @@ unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.5" }
|
||||
russh-sftp = "2.4.0"
|
||||
russh-sftp = "2.3.0"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7", features = ["extended"] }
|
||||
hotpath = { version = "0.23.1", default-features = false }
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13", features = ["extended"] }
|
||||
hotpath = { version = "0.22.0", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
[workspace.metadata.cargo-shear]
|
||||
ignored = ["hotpath", "rustfs"]
|
||||
ignored = ["rustfs"]
|
||||
|
||||
[profile.dev]
|
||||
# Full debuginfo roughly doubles compile+link time and produces multi-GB
|
||||
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.12
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.12
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -55,10 +55,10 @@ hotpath.workspace = true
|
||||
rustfs-targets = { workspace = true }
|
||||
rustfs-config = { workspace = true, features = ["audit", "server-config-model"] }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
const-str = { workspace = true, features = ["std", "proc"] }
|
||||
futures = { workspace = true }
|
||||
hashbrown = { workspace = true, features = ["serde", "rayon"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use hashbrown::HashMap;
|
||||
use jiff::Timestamp;
|
||||
use rustfs_s3_types::EventName;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
@@ -151,8 +151,8 @@ pub struct AuditEntry {
|
||||
pub deployment_id: Option<String>,
|
||||
#[serde(rename = "siteName", skip_serializing_if = "Option::is_none")]
|
||||
pub site_name: Option<String>,
|
||||
#[serde(with = "jiff::fmt::serde::timestamp::millisecond::required")]
|
||||
pub time: Timestamp,
|
||||
#[serde(with = "chrono::serde::ts_milliseconds")]
|
||||
pub time: DateTime<Utc>,
|
||||
pub event: EventName,
|
||||
#[serde(rename = "type", skip_serializing_if = "Option::is_none")]
|
||||
pub entry_type: Option<String>,
|
||||
@@ -198,7 +198,7 @@ impl AuditEntryBuilder {
|
||||
pub fn new(version: impl Into<String>, event: EventName, trigger: impl Into<String>, api: ApiDetails) -> Self {
|
||||
Self(AuditEntry {
|
||||
version: version.into(),
|
||||
time: Timestamp::now(),
|
||||
time: Utc::now(),
|
||||
event,
|
||||
trigger: trigger.into(),
|
||||
api,
|
||||
@@ -232,7 +232,7 @@ impl AuditEntryBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn time(mut self, time: Timestamp) -> Self {
|
||||
pub fn time(mut self, time: DateTime<Utc>) -> Self {
|
||||
self.0.time = time;
|
||||
self
|
||||
}
|
||||
@@ -342,23 +342,4 @@ mod tests {
|
||||
assert_eq!(value["requestID"], Value::String("req-audit-123".to_string()));
|
||||
assert!(value.get("request_id").is_none(), "historical audit contract must not expose request_id");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audit_entry_time_serializes_as_epoch_milliseconds() {
|
||||
let entry = AuditEntryBuilder::new(
|
||||
"1",
|
||||
EventName::ObjectCreatedPut,
|
||||
"s3",
|
||||
ApiDetailsBuilder::new()
|
||||
.name("PutObject")
|
||||
.status("OK")
|
||||
.status_code(200)
|
||||
.build(),
|
||||
)
|
||||
.time(Timestamp::from_millisecond(1_711_423_698_870).expect("timestamp should be valid"))
|
||||
.build();
|
||||
|
||||
let value = serde_json::to_value(entry).expect("audit entry should serialize");
|
||||
assert_eq!(value["time"], Value::Number(1_711_423_698_870_i64.into()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,7 +97,7 @@ async fn test_audit_log_dispatch_performance() {
|
||||
return; // Alternatively: assert!(false, "AuditSystem failed to start");
|
||||
}
|
||||
|
||||
use jiff::Timestamp;
|
||||
use chrono::Utc;
|
||||
use rustfs_targets::EventName;
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
@@ -136,7 +136,7 @@ async fn test_audit_log_dispatch_performance() {
|
||||
version: "1".to_string(),
|
||||
deployment_id: Some(format!("test-deployment-{id}")),
|
||||
site_name: Some("test-site".to_string()),
|
||||
time: Timestamp::now(),
|
||||
time: Utc::now(),
|
||||
event: EventName::ObjectCreatedPut,
|
||||
entry_type: Some("object".to_string()),
|
||||
trigger: "api".to_string(),
|
||||
@@ -298,7 +298,7 @@ fn test_performance_requirements() {
|
||||
for i in 0..3000 {
|
||||
// Simulate event name parsing and processing
|
||||
let _event_id = format!("s3:ObjectCreated:Put_{i}");
|
||||
let _timestamp = jiff::Timestamp::now().to_string();
|
||||
let _timestamp = chrono::Utc::now().to_rfc3339();
|
||||
|
||||
// Simulate basic audit entry creation overhead
|
||||
let _entry_size = 512; // bytes
|
||||
|
||||
@@ -264,7 +264,7 @@ fn create_sample_audit_entry() -> AuditEntry {
|
||||
}
|
||||
|
||||
fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
|
||||
use jiff::Timestamp;
|
||||
use chrono::Utc;
|
||||
use rustfs_targets::EventName;
|
||||
use serde_json::json;
|
||||
|
||||
@@ -301,7 +301,7 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
|
||||
version: "1".to_string(),
|
||||
deployment_id: Some(format!("test-deployment-{id}")),
|
||||
site_name: Some("test-site".to_string()),
|
||||
time: Timestamp::now(),
|
||||
time: Utc::now(),
|
||||
event: EventName::ObjectCreatedPut,
|
||||
entry_type: Some("object".to_string()),
|
||||
trigger: "api".to_string(),
|
||||
|
||||
@@ -39,15 +39,11 @@ tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
tonic = { workspace = true, features = ["gzip", "deflate"] }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
rmp-serde = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -356,8 +356,6 @@ pub struct HealChannelRequest {
|
||||
pub recursive: Option<bool>,
|
||||
/// Whether to dry run
|
||||
pub dry_run: Option<bool>,
|
||||
/// Whether to skip namespace locking
|
||||
pub no_lock: Option<bool>,
|
||||
/// Timeout in seconds (optional)
|
||||
pub timeout_seconds: Option<u64>,
|
||||
/// Origin of the request for operational status and queue accounting
|
||||
@@ -562,7 +560,6 @@ pub fn create_heal_request(
|
||||
update_parity: None,
|
||||
recursive: None,
|
||||
dry_run: None,
|
||||
no_lock: None,
|
||||
timeout_seconds: None,
|
||||
source: HealRequestSource::Internal,
|
||||
disk: None,
|
||||
@@ -721,7 +718,6 @@ pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPri
|
||||
update_parity: None,
|
||||
recursive: None,
|
||||
dry_run: None,
|
||||
no_lock: None,
|
||||
timeout_seconds: None,
|
||||
source: HealRequestSource::AutoHeal,
|
||||
};
|
||||
|
||||
+30
-438
@@ -15,10 +15,9 @@
|
||||
use crate::heal_channel::HealScanMode;
|
||||
use crate::last_minute::{AccElem, LastMinuteLatency};
|
||||
use chrono::{DateTime, Utc};
|
||||
use jiff::Timestamp;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::{BTreeSet, HashMap},
|
||||
collections::HashMap,
|
||||
fmt::Display,
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
@@ -670,7 +669,7 @@ impl LockedLastMinuteLatency {
|
||||
#[derive(Clone, Debug)]
|
||||
struct CurrentPathState {
|
||||
path: String,
|
||||
updated_at: Timestamp,
|
||||
updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
struct CurrentPathTracker {
|
||||
@@ -679,10 +678,10 @@ struct CurrentPathTracker {
|
||||
|
||||
impl CurrentPathTracker {
|
||||
fn new(initial_path: String) -> Self {
|
||||
Self::new_at(initial_path, Timestamp::now())
|
||||
Self::new_at(initial_path, Utc::now())
|
||||
}
|
||||
|
||||
fn new_at(initial_path: String, updated_at: Timestamp) -> Self {
|
||||
fn new_at(initial_path: String, updated_at: DateTime<Utc>) -> Self {
|
||||
Self {
|
||||
state: Arc::new(RwLock::new(CurrentPathState {
|
||||
path: initial_path,
|
||||
@@ -694,7 +693,7 @@ impl CurrentPathTracker {
|
||||
async fn update_path(&self, path: String) {
|
||||
let mut state = self.state.write().await;
|
||||
state.path = path;
|
||||
state.updated_at = Timestamp::now();
|
||||
state.updated_at = Utc::now();
|
||||
}
|
||||
|
||||
async fn get_state(&self) -> CurrentPathState {
|
||||
@@ -702,36 +701,6 @@ impl CurrentPathTracker {
|
||||
}
|
||||
}
|
||||
|
||||
fn chrono_to_jiff_timestamp(dt: DateTime<Utc>) -> Timestamp {
|
||||
let seconds = dt.timestamp();
|
||||
let nanoseconds = match i32::try_from(dt.timestamp_subsec_nanos()) {
|
||||
Ok(nanoseconds) => nanoseconds,
|
||||
Err(_) => {
|
||||
return if seconds < 0 { Timestamp::MIN } else { Timestamp::MAX };
|
||||
}
|
||||
};
|
||||
|
||||
match Timestamp::new(seconds, nanoseconds) {
|
||||
Ok(timestamp) => timestamp,
|
||||
Err(_) => {
|
||||
if seconds < 0 {
|
||||
Timestamp::MIN
|
||||
} else {
|
||||
Timestamp::MAX
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
|
||||
let duration = now.duration_since(earlier);
|
||||
if duration.is_negative() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
u64::try_from(duration.as_secs()).map_or(u64::MAX, |seconds| seconds)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
struct ScannerDiskBucketScanState {
|
||||
concurrency_limit: u64,
|
||||
@@ -739,48 +708,6 @@ struct ScannerDiskBucketScanState {
|
||||
active: u64,
|
||||
}
|
||||
|
||||
type ScannerDiskBucketScanKey = (String, String);
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerDiskBucketScanSnapshot {
|
||||
pub pool: String,
|
||||
pub set: String,
|
||||
pub concurrency_limit: u64,
|
||||
pub queued: u64,
|
||||
pub active: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
struct ScannerBucketDriveResultKey {
|
||||
bucket: String,
|
||||
drive: String,
|
||||
result: String,
|
||||
}
|
||||
|
||||
impl ScannerBucketDriveResultKey {
|
||||
fn new(bucket: impl Into<String>, drive: impl Into<String>, result: impl Into<String>) -> Self {
|
||||
Self {
|
||||
bucket: bucket.into(),
|
||||
drive: drive.into(),
|
||||
result: result.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS: usize = 4096;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct ScannerBucketDriveResults {
|
||||
counts: HashMap<ScannerBucketDriveResultKey, ScannerBucketDriveResultValue>,
|
||||
eviction_index: BTreeSet<(u64, ScannerBucketDriveResultKey)>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct ScannerBucketDriveResultValue {
|
||||
count: u64,
|
||||
last_seen: u64,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metrics
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -811,11 +738,7 @@ pub struct Metrics {
|
||||
scanner_set_scan_concurrency_limit: AtomicU64,
|
||||
scanner_set_scans_queued: AtomicU64,
|
||||
scanner_set_scans_active: AtomicU64,
|
||||
scanner_disk_bucket_scan_states: Mutex<HashMap<ScannerDiskBucketScanKey, ScannerDiskBucketScanState>>,
|
||||
scanner_bucket_drive_results: Mutex<ScannerBucketDriveResults>,
|
||||
scanner_bucket_drive_result_clock: AtomicU64,
|
||||
current_scan_cycle_bucket_drive_results_start: Mutex<HashMap<ScannerBucketDriveResultKey, u64>>,
|
||||
last_scan_cycle_bucket_drive_results: Mutex<Vec<ScannerBucketDriveResultSnapshot>>,
|
||||
scanner_disk_bucket_scan_states: Mutex<HashMap<String, ScannerDiskBucketScanState>>,
|
||||
scanner_leader_lock_state: RwLock<String>,
|
||||
scanner_leader_lock_held: AtomicBool,
|
||||
scanner_leader_lock_last_error: RwLock<String>,
|
||||
@@ -1035,14 +958,6 @@ pub struct ScannerSourceWorkSnapshot {
|
||||
pub missed: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerBucketDriveResultSnapshot {
|
||||
pub bucket: String,
|
||||
pub drive: String,
|
||||
pub result: String,
|
||||
pub count: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerReplicationRepairSnapshot {
|
||||
pub source: String,
|
||||
@@ -1197,12 +1112,12 @@ pub struct ScannerLastMinute {
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct ScannerMetricsReport {
|
||||
pub collected_at: Timestamp,
|
||||
pub collected_at: DateTime<Utc>,
|
||||
pub current_cycle: u64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_active: bool,
|
||||
pub current_started: Timestamp,
|
||||
pub cycles_completed_at: Vec<Timestamp>,
|
||||
pub current_started: DateTime<Utc>,
|
||||
pub cycles_completed_at: Vec<DateTime<Utc>>,
|
||||
pub ongoing_buckets: usize,
|
||||
#[serde(default)]
|
||||
pub active_scan_paths: usize,
|
||||
@@ -1375,18 +1290,6 @@ pub struct ScannerMetricsReport {
|
||||
pub partial_cycles: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct ScannerRuntimeDetailsReport {
|
||||
#[serde(default)]
|
||||
pub disk_bucket_scan_states: Vec<ScannerDiskBucketScanSnapshot>,
|
||||
#[serde(default)]
|
||||
pub bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
||||
#[serde(default)]
|
||||
pub current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
||||
#[serde(default)]
|
||||
pub last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
||||
}
|
||||
|
||||
impl CurrentCycle {
|
||||
pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
*self = rmp_serde::from_slice(buf)?;
|
||||
@@ -1754,7 +1657,6 @@ pub fn emit_scan_cycle_superseded(duration: Duration) {
|
||||
|
||||
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
|
||||
let result = if success { "success" } else { "error" };
|
||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
|
||||
metrics::counter!(
|
||||
OTEL_SCANNER_BUCKETS_SCANNED,
|
||||
"result" => result,
|
||||
@@ -1771,7 +1673,6 @@ pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str,
|
||||
}
|
||||
|
||||
pub fn emit_scan_bucket_drive_partial(bucket: &str, disk: &str, duration: Duration) {
|
||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
|
||||
metrics::counter!(
|
||||
OTEL_SCANNER_BUCKETS_SCANNED,
|
||||
"result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
|
||||
@@ -1822,10 +1723,6 @@ impl Metrics {
|
||||
scanner_set_scans_queued: AtomicU64::new(0),
|
||||
scanner_set_scans_active: AtomicU64::new(0),
|
||||
scanner_disk_bucket_scan_states: Mutex::new(HashMap::new()),
|
||||
scanner_bucket_drive_results: Mutex::new(ScannerBucketDriveResults::default()),
|
||||
scanner_bucket_drive_result_clock: AtomicU64::new(0),
|
||||
current_scan_cycle_bucket_drive_results_start: Mutex::new(HashMap::new()),
|
||||
last_scan_cycle_bucket_drive_results: Mutex::new(Vec::new()),
|
||||
scanner_leader_lock_state: RwLock::new("unknown".to_string()),
|
||||
scanner_leader_lock_held: AtomicBool::new(false),
|
||||
scanner_leader_lock_last_error: RwLock::new(String::new()),
|
||||
@@ -2396,7 +2293,7 @@ impl Metrics {
|
||||
queued: Option<usize>,
|
||||
active: Option<usize>,
|
||||
) {
|
||||
let key = (pool.to_string(), set.to_string());
|
||||
let key = format!("{pool}/{set}");
|
||||
let mut states = self
|
||||
.scanner_disk_bucket_scan_states
|
||||
.lock()
|
||||
@@ -2413,41 +2310,6 @@ impl Metrics {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_scanner_bucket_drive_result(&self, bucket: &str, drive: &str, result: &str) {
|
||||
if bucket.is_empty() || drive.is_empty() || result.is_empty() {
|
||||
return;
|
||||
}
|
||||
let key = ScannerBucketDriveResultKey::new(bucket, drive, result);
|
||||
let mut results = self
|
||||
.scanner_bucket_drive_results
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let last_seen = self.scanner_bucket_drive_result_clock.fetch_add(1, Ordering::Relaxed);
|
||||
if let Some(previous_last_seen) = results.counts.get_mut(&key).map(|value| {
|
||||
let previous_last_seen = value.last_seen;
|
||||
value.count = value.count.saturating_add(1);
|
||||
value.last_seen = last_seen;
|
||||
previous_last_seen
|
||||
}) {
|
||||
results.eviction_index.remove(&(previous_last_seen, key.clone()));
|
||||
results.eviction_index.insert((last_seen, key));
|
||||
return;
|
||||
}
|
||||
|
||||
if results.counts.len() >= MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS
|
||||
&& let Some((_, stale_key)) = results.eviction_index.pop_first()
|
||||
{
|
||||
results.counts.remove(&stale_key);
|
||||
}
|
||||
|
||||
if results.counts.len() < MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
|
||||
results
|
||||
.counts
|
||||
.insert(key.clone(), ScannerBucketDriveResultValue { count: 1, last_seen });
|
||||
results.eviction_index.insert((last_seen, key));
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Read-side helpers
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -2619,11 +2481,6 @@ impl Metrics {
|
||||
&self.current_scan_cycle_replication_repair_work_start,
|
||||
&replication_repair_snapshot,
|
||||
);
|
||||
let bucket_drive_results = self.scanner_bucket_drive_result_counts();
|
||||
match self.current_scan_cycle_bucket_drive_results_start.lock() {
|
||||
Ok(mut start) => *start = bucket_drive_results,
|
||||
Err(poisoned) => *poisoned.into_inner() = bucket_drive_results,
|
||||
}
|
||||
self.current_scan_cycle_work_active.store(true, Ordering::Release);
|
||||
snapshot
|
||||
}
|
||||
@@ -2636,11 +2493,6 @@ impl Metrics {
|
||||
self.record_scan_cycle_work(work);
|
||||
self.record_scan_cycle_source_work(&source_work);
|
||||
self.record_scan_cycle_replication_repair_work(&replication_repair_work);
|
||||
let bucket_drive_results = self.current_cycle_bucket_drive_result_snapshots();
|
||||
match self.last_scan_cycle_bucket_drive_results.lock() {
|
||||
Ok(mut last) => *last = bucket_drive_results,
|
||||
Err(poisoned) => *poisoned.into_inner() = bucket_drive_results,
|
||||
}
|
||||
self.current_scan_cycle_work_active.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
@@ -2724,105 +2576,6 @@ impl Metrics {
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_bucket_drive_result_counts(&self) -> HashMap<ScannerBucketDriveResultKey, u64> {
|
||||
self.scanner_bucket_drive_results
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.counts
|
||||
.iter()
|
||||
.map(|(key, value)| (key.clone(), value.count))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn scanner_bucket_drive_result_snapshots(
|
||||
counts: impl IntoIterator<Item = (ScannerBucketDriveResultKey, u64)>,
|
||||
) -> Vec<ScannerBucketDriveResultSnapshot> {
|
||||
let mut snapshots = counts
|
||||
.into_iter()
|
||||
.filter(|(_, count)| *count > 0)
|
||||
.map(|(key, count)| ScannerBucketDriveResultSnapshot {
|
||||
bucket: key.bucket,
|
||||
drive: key.drive,
|
||||
result: key.result,
|
||||
count,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
snapshots.sort_by(|left, right| {
|
||||
left.bucket
|
||||
.cmp(&right.bucket)
|
||||
.then_with(|| left.drive.cmp(&right.drive))
|
||||
.then_with(|| left.result.cmp(&right.result))
|
||||
});
|
||||
snapshots
|
||||
}
|
||||
|
||||
fn scanner_bucket_drive_result_counter_snapshots(&self) -> Vec<ScannerBucketDriveResultSnapshot> {
|
||||
Self::scanner_bucket_drive_result_snapshots(self.scanner_bucket_drive_result_counts())
|
||||
}
|
||||
|
||||
fn current_cycle_bucket_drive_result_snapshots(&self) -> Vec<ScannerBucketDriveResultSnapshot> {
|
||||
let current = self.scanner_bucket_drive_result_counts();
|
||||
let start = self
|
||||
.current_scan_cycle_bucket_drive_results_start
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clone();
|
||||
Self::scanner_bucket_drive_result_snapshots(current.into_iter().filter_map(|(key, count)| {
|
||||
let delta = count.saturating_sub(start.get(&key).copied().unwrap_or_default());
|
||||
(delta > 0).then_some((key, delta))
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn scanner_runtime_details_report(&self) -> ScannerRuntimeDetailsReport {
|
||||
self.scanner_runtime_details_report_for_active(self.current_scan_cycle_work_active.load(Ordering::Acquire))
|
||||
}
|
||||
|
||||
fn scanner_runtime_details_report_for_active(&self, current_cycle_active: bool) -> ScannerRuntimeDetailsReport {
|
||||
let current_cycle_bucket_drive_results = if current_cycle_active {
|
||||
self.current_cycle_bucket_drive_result_snapshots()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
ScannerRuntimeDetailsReport {
|
||||
disk_bucket_scan_states: self.scanner_disk_bucket_scan_state_snapshots(),
|
||||
bucket_drive_results: self.scanner_bucket_drive_result_counter_snapshots(),
|
||||
current_cycle_bucket_drive_results,
|
||||
last_cycle_bucket_drive_results: self
|
||||
.last_scan_cycle_bucket_drive_results
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_disk_bucket_scan_state_snapshots(&self) -> Vec<ScannerDiskBucketScanSnapshot> {
|
||||
let mut disk_bucket_scan_states = match self.scanner_disk_bucket_scan_states.lock() {
|
||||
Ok(states) => states
|
||||
.iter()
|
||||
.map(|((pool, set), state)| ScannerDiskBucketScanSnapshot {
|
||||
pool: pool.clone(),
|
||||
set: set.clone(),
|
||||
concurrency_limit: state.concurrency_limit,
|
||||
queued: state.queued,
|
||||
active: state.active,
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
Err(poisoned) => poisoned
|
||||
.into_inner()
|
||||
.iter()
|
||||
.map(|((pool, set), state)| ScannerDiskBucketScanSnapshot {
|
||||
pool: pool.clone(),
|
||||
set: set.clone(),
|
||||
concurrency_limit: state.concurrency_limit,
|
||||
queued: state.queued,
|
||||
active: state.active,
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
};
|
||||
disk_bucket_scan_states.sort_by(|left, right| left.pool.cmp(&right.pool).then_with(|| left.set.cmp(&right.set)));
|
||||
disk_bucket_scan_states
|
||||
}
|
||||
|
||||
fn scanner_source_work_values(&self) -> Vec<ScannerSourceWorkValues> {
|
||||
ScannerWorkSource::all()
|
||||
.iter()
|
||||
@@ -3008,26 +2761,20 @@ impl Metrics {
|
||||
|
||||
/// Build a full metrics report snapshot.
|
||||
pub async fn report(&self) -> ScannerMetricsReport {
|
||||
self.report_with_runtime_details().await.0
|
||||
}
|
||||
|
||||
pub async fn report_with_runtime_details(&self) -> (ScannerMetricsReport, ScannerRuntimeDetailsReport) {
|
||||
let mut m = ScannerMetricsReport::default();
|
||||
let runtime_details;
|
||||
|
||||
let has_cycle = {
|
||||
let cycle = self.cycle_info.read().await;
|
||||
let has_cycle = if let Some(cycle) = cycle.as_ref() {
|
||||
m.current_cycle = cycle.current;
|
||||
m.cycles_completed_at = cycle.cycle_completed.iter().copied().map(chrono_to_jiff_timestamp).collect();
|
||||
m.current_started = chrono_to_jiff_timestamp(cycle.started);
|
||||
m.cycles_completed_at = cycle.cycle_completed.clone();
|
||||
m.current_started = cycle.started;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
m.current_cycle_active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
|
||||
if m.current_cycle_active {
|
||||
// Keep cycle_info before cycle-baseline locks so active scrapes cannot mix two cycle identities.
|
||||
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
|
||||
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
|
||||
let current_replication_repair_work =
|
||||
@@ -3050,20 +2797,19 @@ impl Metrics {
|
||||
m.current_cycle_replication_repair =
|
||||
self.scanner_replication_repair_work_snapshots(¤t_replication_repair_work);
|
||||
}
|
||||
runtime_details = self.scanner_runtime_details_report_for_active(m.current_cycle_active);
|
||||
has_cycle
|
||||
};
|
||||
|
||||
if !has_cycle && let Some(init_time) = crate::get_global_init_time().await {
|
||||
m.current_started = chrono_to_jiff_timestamp(init_time);
|
||||
m.current_started = init_time;
|
||||
}
|
||||
|
||||
m.collected_at = Timestamp::now();
|
||||
m.collected_at = Utc::now();
|
||||
let current_path_snapshots = self.current_path_snapshots().await;
|
||||
m.active_scan_paths = current_path_snapshots.len();
|
||||
m.oldest_active_path_age_seconds = current_path_snapshots
|
||||
.iter()
|
||||
.map(|(_, state)| timestamp_elapsed_seconds_since(m.collected_at, state.updated_at))
|
||||
.map(|(_, state)| m.collected_at.signed_duration_since(state.updated_at).num_seconds().max(0) as u64)
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
m.active_paths = current_path_snapshots
|
||||
@@ -3080,11 +2826,15 @@ impl Metrics {
|
||||
m.current_set_scan_concurrency_limit = self.scanner_set_scan_concurrency_limit.load(Ordering::Relaxed);
|
||||
m.current_set_scans_queued = self.scanner_set_scans_queued.load(Ordering::Relaxed);
|
||||
m.current_set_scans_active = self.scanner_set_scans_active.load(Ordering::Relaxed);
|
||||
let disk_bucket_scan_states = self.scanner_disk_bucket_scan_state_snapshots();
|
||||
let (disk_scan_concurrency_limit, disk_bucket_scans_queued, disk_bucket_scans_active) =
|
||||
disk_bucket_scan_states.iter().fold((0, 0, 0), |acc, state| {
|
||||
(acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active)
|
||||
});
|
||||
match self.scanner_disk_bucket_scan_states.lock() {
|
||||
Ok(states) => states.values().fold((0, 0, 0), |acc, state| {
|
||||
(acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active)
|
||||
}),
|
||||
Err(poisoned) => poisoned.into_inner().values().fold((0, 0, 0), |acc, state| {
|
||||
(acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active)
|
||||
}),
|
||||
};
|
||||
m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
|
||||
m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
|
||||
m.current_disk_bucket_scans_active = disk_bucket_scans_active;
|
||||
@@ -3253,7 +3003,7 @@ impl Metrics {
|
||||
m.pacing_pressure = scanner_pacing_pressure(&m);
|
||||
m.maintenance_control = scanner_maintenance_control(&m);
|
||||
|
||||
(m, runtime_details)
|
||||
m
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3339,22 +3089,6 @@ impl Drop for CloseDiskGuard {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_report_timestamps_serialize_as_rfc3339_utc() {
|
||||
let report = ScannerMetricsReport {
|
||||
collected_at: Timestamp::constant(1_700_000_000, 123_456_000),
|
||||
current_started: Timestamp::constant(1_699_999_940, 0),
|
||||
cycles_completed_at: vec![Timestamp::constant(1_700_000_060, 987_654_000)],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let value = serde_json::to_value(&report).expect("scanner metrics report should serialize");
|
||||
|
||||
assert_eq!(value["collected_at"].as_str(), Some("2023-11-14T22:13:20.123456Z"));
|
||||
assert_eq!(value["current_started"].as_str(), Some("2023-11-14T22:12:20Z"));
|
||||
assert_eq!(value["cycles_completed_at"][0].as_str(), Some("2023-11-14T22:14:20.987654Z"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn close_disk_guard_runs_cleanup_when_an_early_return_drops_it() {
|
||||
let (closed_tx, closed_rx) = tokio::sync::oneshot::channel();
|
||||
@@ -3413,7 +3147,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn report_counts_active_scan_paths() {
|
||||
let metrics = Metrics::new();
|
||||
let updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(12);
|
||||
let updated_at = Utc::now() - chrono::Duration::seconds(12);
|
||||
metrics.current_paths.write().await.insert(
|
||||
"disk-a".to_string(),
|
||||
Arc::new(CurrentPathTracker::new_at("bucket-a".to_string(), updated_at)),
|
||||
@@ -3435,7 +3169,7 @@ mod tests {
|
||||
let metrics = Metrics::new();
|
||||
let tracker = Arc::new(CurrentPathTracker::new_at(
|
||||
"bucket-a".to_string(),
|
||||
Timestamp::now() - jiff::SignedDuration::from_secs(60 * 60),
|
||||
Utc::now() - chrono::Duration::hours(1),
|
||||
));
|
||||
metrics
|
||||
.current_paths
|
||||
@@ -4208,7 +3942,7 @@ mod tests {
|
||||
let report = metrics.report().await;
|
||||
*crate::globals::GLOBAL_INIT_TIME.write().await = previous_init_time;
|
||||
|
||||
assert_eq!(report.current_started, chrono_to_jiff_timestamp(cycle_started));
|
||||
assert_eq!(report.current_started, cycle_started);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -4366,137 +4100,6 @@ mod tests {
|
||||
assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_includes_structured_bucket_drive_results() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scanner_bucket_drive_result("photos", "/data1", "success");
|
||||
|
||||
let cycle_start = metrics.start_scan_cycle_work();
|
||||
metrics.record_scanner_bucket_drive_result("photos", "/data1", "partial");
|
||||
|
||||
let active_report = metrics.scanner_runtime_details_report();
|
||||
assert_eq!(
|
||||
active_report.current_cycle_bucket_drive_results,
|
||||
vec![ScannerBucketDriveResultSnapshot {
|
||||
bucket: "photos".to_string(),
|
||||
drive: "/data1".to_string(),
|
||||
result: "partial".to_string(),
|
||||
count: 1,
|
||||
}]
|
||||
);
|
||||
|
||||
metrics.finish_scan_cycle_work(cycle_start);
|
||||
|
||||
let report = metrics.scanner_runtime_details_report();
|
||||
|
||||
assert_eq!(
|
||||
report.bucket_drive_results,
|
||||
vec![
|
||||
ScannerBucketDriveResultSnapshot {
|
||||
bucket: "photos".to_string(),
|
||||
drive: "/data1".to_string(),
|
||||
result: "partial".to_string(),
|
||||
count: 1,
|
||||
},
|
||||
ScannerBucketDriveResultSnapshot {
|
||||
bucket: "photos".to_string(),
|
||||
drive: "/data1".to_string(),
|
||||
result: "success".to_string(),
|
||||
count: 1,
|
||||
},
|
||||
]
|
||||
);
|
||||
assert!(report.current_cycle_bucket_drive_results.is_empty());
|
||||
assert_eq!(
|
||||
report.last_cycle_bucket_drive_results,
|
||||
vec![ScannerBucketDriveResultSnapshot {
|
||||
bucket: "photos".to_string(),
|
||||
drive: "/data1".to_string(),
|
||||
result: "partial".to_string(),
|
||||
count: 1,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_bucket_drive_results_are_bounded() {
|
||||
let metrics = Metrics::new();
|
||||
for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
|
||||
metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
|
||||
}
|
||||
metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
|
||||
|
||||
let report = metrics.scanner_runtime_details_report();
|
||||
|
||||
assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
|
||||
assert!(
|
||||
report
|
||||
.bucket_drive_results
|
||||
.iter()
|
||||
.any(|snapshot| snapshot.bucket == "overflow")
|
||||
);
|
||||
assert!(
|
||||
report
|
||||
.bucket_drive_results
|
||||
.iter()
|
||||
.all(|snapshot| snapshot.bucket != "bucket-0")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_bucket_drive_result_eviction_keeps_recent_keys() {
|
||||
let metrics = Metrics::new();
|
||||
for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
|
||||
metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
|
||||
}
|
||||
metrics.record_scanner_bucket_drive_result("bucket-0", "/data1", "success");
|
||||
metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
|
||||
|
||||
let report = metrics.scanner_runtime_details_report();
|
||||
|
||||
assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
|
||||
assert!(
|
||||
report
|
||||
.bucket_drive_results
|
||||
.iter()
|
||||
.any(|snapshot| snapshot.bucket == "bucket-0" && snapshot.count == 2)
|
||||
);
|
||||
assert!(
|
||||
report
|
||||
.bucket_drive_results
|
||||
.iter()
|
||||
.all(|snapshot| snapshot.bucket != "bucket-1")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_bucket_drive_result_eviction_survives_full_refresh() {
|
||||
let metrics = Metrics::new();
|
||||
for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
|
||||
metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
|
||||
}
|
||||
for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
|
||||
metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
|
||||
}
|
||||
metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
|
||||
|
||||
let report = metrics.scanner_runtime_details_report();
|
||||
|
||||
assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
|
||||
assert!(
|
||||
report
|
||||
.bucket_drive_results
|
||||
.iter()
|
||||
.any(|snapshot| snapshot.bucket == "overflow")
|
||||
);
|
||||
assert!(
|
||||
report
|
||||
.bucket_drive_results
|
||||
.iter()
|
||||
.all(|snapshot| snapshot.bucket != "bucket-0")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_includes_usage_freshness_status() {
|
||||
let metrics = Metrics::new();
|
||||
@@ -4631,7 +4234,7 @@ mod tests {
|
||||
let active = metrics.report().await;
|
||||
assert!(active.current_cycle_active);
|
||||
assert_eq!(active.current_cycle, 12);
|
||||
assert_eq!(active.current_started, chrono_to_jiff_timestamp(cycle_started));
|
||||
assert_eq!(active.current_started, cycle_started);
|
||||
|
||||
let idle_cycle = CurrentCycle {
|
||||
current: 0,
|
||||
@@ -4662,10 +4265,9 @@ mod tests {
|
||||
};
|
||||
let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await;
|
||||
metrics.operations[Metric::ScanObject as usize].store(1, Ordering::Relaxed);
|
||||
metrics.record_scanner_bucket_drive_result("cycle-ten", "/data1", "partial");
|
||||
|
||||
let paths = metrics.current_paths.write().await;
|
||||
let mut report = Box::pin(metrics.report_with_runtime_details());
|
||||
let mut report = Box::pin(metrics.report());
|
||||
let waker = std::task::Waker::noop();
|
||||
let mut context = std::task::Context::from_waker(waker);
|
||||
assert!(report.as_mut().poll(&mut context).is_pending());
|
||||
@@ -4682,22 +4284,12 @@ mod tests {
|
||||
})
|
||||
.await;
|
||||
metrics.operations[Metric::ScanObject as usize].store(101, Ordering::Relaxed);
|
||||
metrics.record_scanner_bucket_drive_result("cycle-eleven", "/data1", "partial");
|
||||
|
||||
drop(paths);
|
||||
let (snapshot, runtime_details) = report.await;
|
||||
let snapshot = report.await;
|
||||
|
||||
assert_eq!(snapshot.current_cycle, 10);
|
||||
assert_eq!(snapshot.current_cycle_objects_scanned, 1);
|
||||
assert_eq!(
|
||||
runtime_details.current_cycle_bucket_drive_results,
|
||||
vec![ScannerBucketDriveResultSnapshot {
|
||||
bucket: "cycle-ten".to_string(),
|
||||
drive: "/data1".to_string(),
|
||||
result: "partial".to_string(),
|
||||
count: 1,
|
||||
}]
|
||||
);
|
||||
|
||||
metrics
|
||||
.finish_scan_cycle_work_with_cycle(cycle_eleven_start, CurrentCycle::default())
|
||||
|
||||
@@ -177,9 +177,10 @@ const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
|
||||
///
|
||||
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
|
||||
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
|
||||
/// state holds roughly `authenticated RPC RPS x 601s` entries. This default is the minimum floor:
|
||||
/// explicit operator values and resource-aware auto sizing both clamp upward to at least this
|
||||
/// value. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
|
||||
/// state holds roughly `authenticated RPC RPS x 601s` entries. The default sustains about 1,700
|
||||
/// authenticated RPCs per second (about 120 MiB worst case, allocated only under sustained load);
|
||||
/// operators must size it for the node's aggregate peak RPC rate before enabling strict replay
|
||||
/// scope. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
|
||||
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
|
||||
/// the shared secret) — and increments
|
||||
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use serde::{Deserialize, Serialize, ser::SerializeMap as _};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
hash::{DefaultHasher, Hash, Hasher},
|
||||
@@ -37,10 +37,6 @@ pub const USAGE_LAST_UPDATE_FUTURE_TOLERANCE: Duration = Duration::from_secs(5 *
|
||||
/// Keeping the existing object name preserves rolling-upgrade and rollback
|
||||
/// compatibility without allowing an ambiguous snapshot to become authoritative.
|
||||
pub const DATA_USAGE_OBJECT_NAME: &str = ".usage.v2.json";
|
||||
/// Latest structurally complete scanner observation. Unlike
|
||||
/// [`DATA_USAGE_OBJECT_NAME`], this object is never authoritative for quota
|
||||
/// admission because namespace activity may have raced the scan.
|
||||
pub const DATA_USAGE_OBSERVED_OBJECT_NAME: &str = ".usage.observed.json";
|
||||
|
||||
/// Usage snapshot written by scanner implementations predating distributed
|
||||
/// leadership fencing. It is read only when neither authoritative snapshot
|
||||
@@ -55,36 +51,24 @@ pub fn usage_last_update_is_untrusted_future(existing_last_update: SystemTime, n
|
||||
existing_last_update > now + USAGE_LAST_UPDATE_FUTURE_TOLERANCE
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Clone, Copy, Default, Debug, Serialize, Deserialize, PartialEq)]
|
||||
pub struct TierStats {
|
||||
pub total_size: u64,
|
||||
pub num_versions: u64,
|
||||
pub num_objects: u64,
|
||||
pub num_versions: i32,
|
||||
pub num_objects: i32,
|
||||
}
|
||||
|
||||
impl TierStats {
|
||||
pub fn add(&self, u: &TierStats) -> TierStats {
|
||||
TierStats {
|
||||
total_size: self.total_size.saturating_add(u.total_size),
|
||||
num_versions: self.num_versions.saturating_add(u.num_versions),
|
||||
num_objects: self.num_objects.saturating_add(u.num_objects),
|
||||
total_size: self.total_size + u.total_size,
|
||||
num_versions: self.num_versions + u.num_versions,
|
||||
num_objects: self.num_objects + u.num_objects,
|
||||
}
|
||||
}
|
||||
|
||||
/// True when [`TierStats::add`] would report the exact sum instead of saturating.
|
||||
pub fn fits_add(&self, u: &TierStats) -> bool {
|
||||
self.total_size.checked_add(u.total_size).is_some()
|
||||
&& self.num_versions.checked_add(u.num_versions).is_some()
|
||||
&& self.num_objects.checked_add(u.num_objects).is_some()
|
||||
}
|
||||
|
||||
/// True when this tier contributed nothing, i.e. merging it is a no-op.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.total_size == 0 && self.num_versions == 0 && self.num_objects == 0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
|
||||
pub struct AllTierStats {
|
||||
pub tiers: HashMap<String, TierStats>,
|
||||
}
|
||||
@@ -94,35 +78,31 @@ impl AllTierStats {
|
||||
Self { tiers: HashMap::new() }
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.tiers.is_empty()
|
||||
}
|
||||
|
||||
/// Folds a scan summary's per-tier map in.
|
||||
///
|
||||
/// Scanners seed the map with a zeroed entry for every configured tier, so
|
||||
/// empty contributions are skipped to keep the persisted cache from growing
|
||||
/// one key per tier on every folder that never held tiered data.
|
||||
pub fn add_sizes(&mut self, tiers: &HashMap<String, TierStats>) {
|
||||
pub fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
|
||||
for (tier, st) in tiers {
|
||||
if st.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry = self.tiers.entry(tier.clone()).or_default();
|
||||
*entry = entry.add(st);
|
||||
self.tiers
|
||||
.insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, other: &AllTierStats) {
|
||||
self.add_sizes(&other.tiers);
|
||||
pub fn merge(&mut self, other: AllTierStats) {
|
||||
for (tier, st) in other.tiers {
|
||||
self.tiers
|
||||
.insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
|
||||
}
|
||||
}
|
||||
|
||||
/// True when [`AllTierStats::merge`] would report exact sums for every tier.
|
||||
pub fn fits_merge(&self, other: &AllTierStats) -> bool {
|
||||
other
|
||||
.tiers
|
||||
.iter()
|
||||
.all(|(tier, right)| self.tiers.get(tier).is_none_or(|left| left.fits_add(right)))
|
||||
pub fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) {
|
||||
for (tier, st) in &self.tiers {
|
||||
stats.insert(
|
||||
tier.clone(),
|
||||
TierStats {
|
||||
total_size: st.total_size,
|
||||
num_versions: st.num_versions,
|
||||
num_objects: st.num_objects,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,14 +183,6 @@ pub struct DataUsageInfo {
|
||||
pub objects_total_size: u64,
|
||||
/// Replication info across all buckets
|
||||
pub replication_info: HashMap<String, BucketTargetUsageInfo>,
|
||||
/// Usage per storage class and remote tier across all buckets.
|
||||
///
|
||||
/// Absent on snapshots written before per-tier accounting was published,
|
||||
/// and on clusters with no remote tier configured: the scanner classifies
|
||||
/// objects by tier (including `STANDARD`/`REDUCED_REDUNDANCY`) only once a
|
||||
/// tier exists, so an absent value means "not accounted", never "zero".
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tier_stats: Option<AllTierStats>,
|
||||
|
||||
/// Total number of buckets in this cluster
|
||||
pub buckets_count: u64,
|
||||
@@ -222,20 +194,6 @@ pub struct DataUsageInfo {
|
||||
/// explicit entry for every bucket, including confirmed-empty buckets.
|
||||
#[serde(default)]
|
||||
pub usage_snapshot_complete: bool,
|
||||
/// Whether no namespace activity or dirty-usage generation changed while
|
||||
/// the coordinated snapshot was being produced.
|
||||
///
|
||||
/// `false` still describes a structurally complete, useful point-in-time
|
||||
/// usage view, but follow-up scanner work remains pending. `None` is kept
|
||||
/// for snapshots written before this status became observable.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_snapshot_converged: Option<bool>,
|
||||
/// Identity of the authoritative snapshot from which a nonconverged
|
||||
/// observation started. Admin readers require an exact match before using
|
||||
/// the observation, so bucket namespace mutations fence old observations
|
||||
/// without relying on synchronized clocks.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_snapshot_authoritative_baseline: Option<DataUsageSnapshotIdentity>,
|
||||
/// Deprecated kept here for backward compatibility reasons
|
||||
pub bucket_sizes: HashMap<String, u64>,
|
||||
/// Per-disk snapshot information when available
|
||||
@@ -243,59 +201,6 @@ pub struct DataUsageInfo {
|
||||
pub disk_usage_status: Vec<DiskUsageStatus>,
|
||||
}
|
||||
|
||||
/// Stable identity fields changed by both coordinated scanner publication and
|
||||
/// backward-compatible bucket namespace cleanup.
|
||||
#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct DataUsageSnapshotIdentity {
|
||||
pub last_update: Option<SystemTime>,
|
||||
pub scanner_cycle: Option<u64>,
|
||||
pub scanner_epoch: Option<u64>,
|
||||
}
|
||||
|
||||
impl DataUsageInfo {
|
||||
pub fn snapshot_identity(&self) -> DataUsageSnapshotIdentity {
|
||||
DataUsageSnapshotIdentity {
|
||||
last_update: self.last_update,
|
||||
scanner_cycle: self.scanner_cycle,
|
||||
scanner_epoch: self.scanner_epoch,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether `candidate` was produced after `baseline`.
|
||||
///
|
||||
/// New coordinated snapshots are ordered by leadership epoch and scanner
|
||||
/// cycle. The timestamp fallback preserves ordering for legacy snapshots that
|
||||
/// predate those fields.
|
||||
pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUsageInfo) -> bool {
|
||||
match (
|
||||
candidate.scanner_epoch.zip(candidate.scanner_cycle),
|
||||
baseline.scanner_epoch.zip(baseline.scanner_cycle),
|
||||
) {
|
||||
(Some(candidate), Some(baseline)) => candidate > baseline,
|
||||
(Some(_), None) => true,
|
||||
(None, Some(_)) => false,
|
||||
(None, None) => match (candidate.last_update, baseline.last_update) {
|
||||
(Some(candidate), Some(baseline)) => candidate > baseline,
|
||||
(Some(_), None) => true,
|
||||
(None, Some(_) | None) => false,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether a nonconverged observation may safely supersede the admin
|
||||
/// view of `authoritative`.
|
||||
///
|
||||
/// The exact baseline identity is independent of clock ordering. Older binaries
|
||||
/// already advance the authoritative timestamp when deleting a bucket, so a
|
||||
/// rollback delete/recreate fences the previous bucket incarnation too.
|
||||
pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &DataUsageInfo) -> bool {
|
||||
observed.usage_snapshot_converged == Some(false)
|
||||
&& observed.is_complete_bucket_usage_snapshot()
|
||||
&& observed.usage_snapshot_authoritative_baseline.as_ref() == Some(&authoritative.snapshot_identity())
|
||||
&& data_usage_snapshot_is_newer(observed, authoritative)
|
||||
}
|
||||
|
||||
/// Metadata describing the status of a disk-level data usage snapshot.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct DiskUsageStatus {
|
||||
@@ -657,7 +562,7 @@ impl ReplicationAllStats {
|
||||
}
|
||||
|
||||
/// Data usage cache entry
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct DataUsageEntry {
|
||||
pub children: DataUsageHashMap,
|
||||
// These fields do not include any children.
|
||||
@@ -672,34 +577,6 @@ pub struct DataUsageEntry {
|
||||
/// Number of objects that failed to scan (e.g., IO errors)
|
||||
#[serde(default)]
|
||||
pub failed_objects: usize,
|
||||
/// Per-tier usage contributed by this entry, present only once a scan
|
||||
/// observed tier-classified objects.
|
||||
#[serde(default)]
|
||||
pub all_tier_stats: Option<AllTierStats>,
|
||||
}
|
||||
|
||||
impl Serialize for DataUsageEntry {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
// Keep entries map-encoded so older readers can ignore fields appended
|
||||
// by newer scanner versions during rolling upgrades. The derived
|
||||
// (array) encoding made any appended field a decode error for them.
|
||||
let mut state = serializer.serialize_map(Some(11))?;
|
||||
state.serialize_entry("children", &self.children)?;
|
||||
state.serialize_entry("size", &self.size)?;
|
||||
state.serialize_entry("objects", &self.objects)?;
|
||||
state.serialize_entry("versions", &self.versions)?;
|
||||
state.serialize_entry("delete_markers", &self.delete_markers)?;
|
||||
state.serialize_entry("obj_sizes", &self.obj_sizes)?;
|
||||
state.serialize_entry("obj_versions", &self.obj_versions)?;
|
||||
state.serialize_entry("replication_stats", &self.replication_stats)?;
|
||||
state.serialize_entry("compacted", &self.compacted)?;
|
||||
state.serialize_entry("failed_objects", &self.failed_objects)?;
|
||||
state.serialize_entry("all_tier_stats", &self.all_tier_stats)?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageEntry {
|
||||
@@ -758,22 +635,10 @@ impl DataUsageEntry {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(o_tiers) = other.all_tier_stats.as_ref().filter(|tiers| !tiers.is_empty()) {
|
||||
self.all_tier_stats.get_or_insert_with(AllTierStats::new).merge(o_tiers);
|
||||
}
|
||||
|
||||
self.obj_sizes.merge_from(&other.obj_sizes);
|
||||
self.obj_versions.merge_from(&other.obj_versions);
|
||||
}
|
||||
|
||||
/// Folds a scan summary's per-tier map into this entry.
|
||||
pub fn add_tier_sizes(&mut self, tiers: &HashMap<String, TierStats>) {
|
||||
if tiers.values().all(TierStats::is_empty) {
|
||||
return;
|
||||
}
|
||||
self.all_tier_stats.get_or_insert_with(AllTierStats::new).add_sizes(tiers);
|
||||
}
|
||||
|
||||
pub fn checked_merge(&mut self, other: &DataUsageEntry) -> bool {
|
||||
let scalar_counts_fit = self.objects.checked_add(other.objects).is_some()
|
||||
&& self.versions.checked_add(other.versions).is_some()
|
||||
@@ -833,12 +698,7 @@ impl DataUsageEntry {
|
||||
}
|
||||
};
|
||||
|
||||
let tier_stats_fit = match (&self.all_tier_stats, &other.all_tier_stats) {
|
||||
(_, None) | (None, Some(_)) => true,
|
||||
(Some(left), Some(right)) => left.fits_merge(right),
|
||||
};
|
||||
|
||||
if !scalar_counts_fit || !histograms_fit || !replication_fits || !tier_stats_fit {
|
||||
if !scalar_counts_fit || !histograms_fit || !replication_fits {
|
||||
return false;
|
||||
}
|
||||
self.merge(other);
|
||||
@@ -1178,7 +1038,6 @@ impl DataUsageCache {
|
||||
versions_total_count: flat.versions as u64,
|
||||
delete_markers_total_count: flat.delete_markers as u64,
|
||||
objects_total_size: flat.size as u64,
|
||||
tier_stats: flat.all_tier_stats.filter(|tiers| !tiers.is_empty()),
|
||||
buckets_count: u64::try_from(buckets.len()).unwrap_or(u64::MAX),
|
||||
buckets_usage,
|
||||
usage_snapshot_complete: self.info.snapshot_complete,
|
||||
@@ -1666,172 +1525,6 @@ mod tests {
|
||||
buckets_count: u64,
|
||||
}
|
||||
|
||||
fn tier_entry(tier: &str, stats: TierStats) -> DataUsageEntry {
|
||||
let mut entry = DataUsageEntry::default();
|
||||
entry.add_tier_sizes(&HashMap::from([(tier.to_string(), stats)]));
|
||||
entry
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_stats_survive_entry_merge() {
|
||||
let mut left = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 10,
|
||||
num_versions: 2,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
let mut right = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 5,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
right.add_tier_sizes(&HashMap::from([(
|
||||
"COLD".to_string(),
|
||||
TierStats {
|
||||
total_size: 7,
|
||||
num_versions: 1,
|
||||
num_objects: 0,
|
||||
},
|
||||
)]));
|
||||
|
||||
assert!(left.checked_merge(&right), "merging exact tier totals must be accepted");
|
||||
|
||||
let tiers = &left.all_tier_stats.expect("merged entry keeps tier stats").tiers;
|
||||
assert_eq!(
|
||||
tiers.get("WARM"),
|
||||
Some(&TierStats {
|
||||
total_size: 15,
|
||||
num_versions: 3,
|
||||
num_objects: 2,
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
tiers.get("COLD"),
|
||||
Some(&TierStats {
|
||||
total_size: 7,
|
||||
num_versions: 1,
|
||||
num_objects: 0,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_stats_merge_into_an_untiered_entry() {
|
||||
let mut left = DataUsageEntry::default();
|
||||
let right = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 10,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
|
||||
assert!(left.checked_merge(&right));
|
||||
|
||||
assert_eq!(
|
||||
left.all_tier_stats.expect("tier stats adopted from the merged entry").tiers["WARM"],
|
||||
TierStats {
|
||||
total_size: 10,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_merge_rejects_overflowing_tier_totals() {
|
||||
let mut left = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: u64::MAX,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
let right = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 1,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
|
||||
assert!(!left.checked_merge(&right), "saturating tier totals must not be published");
|
||||
assert_eq!(left.all_tier_stats.expect("left is untouched").tiers["WARM"].total_size, u64::MAX);
|
||||
}
|
||||
|
||||
/// Entry shape released before per-tier accounting, using the derived
|
||||
/// (array) encoding those writers produced.
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct LegacyEntry {
|
||||
children: DataUsageHashMap,
|
||||
size: usize,
|
||||
objects: usize,
|
||||
versions: usize,
|
||||
delete_markers: usize,
|
||||
obj_sizes: SizeHistogram,
|
||||
obj_versions: VersionsHistogram,
|
||||
replication_stats: Option<ReplicationAllStats>,
|
||||
compacted: bool,
|
||||
#[serde(default)]
|
||||
failed_objects: usize,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entries_are_map_encoded_so_appended_fields_stay_readable() {
|
||||
// A derived (array) encoding turns every appended field into a decode
|
||||
// error for readers built before it existed, which would cost a mixed
|
||||
// -version cluster its whole scan cache. Entries must stay map-encoded.
|
||||
let current = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 3,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
let mut encoded = Vec::new();
|
||||
current
|
||||
.serialize(&mut rmp_serde::Serializer::new(&mut encoded))
|
||||
.expect("encode current entry");
|
||||
|
||||
let legacy: LegacyEntry = rmp_serde::from_slice(&encoded).expect("legacy reader should ignore the appended field");
|
||||
assert_eq!(legacy.objects, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_array_encoded_entries_still_load() {
|
||||
let legacy = LegacyEntry {
|
||||
children: DataUsageHashMap::default(),
|
||||
size: 12,
|
||||
objects: 3,
|
||||
versions: 4,
|
||||
delete_markers: 1,
|
||||
obj_sizes: SizeHistogram::default(),
|
||||
obj_versions: VersionsHistogram::default(),
|
||||
replication_stats: None,
|
||||
compacted: false,
|
||||
failed_objects: 2,
|
||||
};
|
||||
let mut encoded = Vec::new();
|
||||
legacy
|
||||
.serialize(&mut rmp_serde::Serializer::new(&mut encoded))
|
||||
.expect("encode legacy entry");
|
||||
|
||||
let decoded: DataUsageEntry = rmp_serde::from_slice(&encoded).expect("current reader should default the missing field");
|
||||
|
||||
assert_eq!(decoded.size, 12);
|
||||
assert_eq!(decoded.failed_objects, 2);
|
||||
assert!(decoded.all_tier_stats.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
@@ -1854,8 +1547,6 @@ mod tests {
|
||||
let current = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(DataUsageSnapshotIdentity::default()),
|
||||
..Default::default()
|
||||
};
|
||||
let encoded = rmp_serde::to_vec_named(¤t).expect("encode current data usage snapshot");
|
||||
@@ -1863,76 +1554,6 @@ mod tests {
|
||||
|
||||
assert_eq!(legacy.buckets_count, 0);
|
||||
assert!(current.is_complete_bucket_usage_snapshot());
|
||||
assert_eq!(current.usage_snapshot_converged, Some(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn convergence_marker_defaults_to_unknown_for_older_snapshots() {
|
||||
let encoded = rmp_serde::to_vec_named(&DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("encode pre-convergence data usage snapshot");
|
||||
let decoded: DataUsageInfo = rmp_serde::from_slice(&encoded).expect("decode older data usage snapshot");
|
||||
|
||||
assert!(decoded.is_complete_bucket_usage_snapshot());
|
||||
assert_eq!(decoded.usage_snapshot_converged, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observation_selection_is_clock_independent_and_baseline_fenced() {
|
||||
let mut authoritative = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(600)),
|
||||
scanner_epoch: Some(7),
|
||||
scanner_cycle: Some(10),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let observed = DataUsageInfo {
|
||||
// A newer leader may have a slower wall clock.
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(300)),
|
||||
scanner_epoch: Some(8),
|
||||
scanner_cycle: Some(1),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(observed_data_usage_is_newer(&observed, &authoritative));
|
||||
|
||||
authoritative.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(601));
|
||||
assert!(
|
||||
!observed_data_usage_is_newer(&observed, &authoritative),
|
||||
"an old-binary namespace mutation must fence the prior bucket incarnation regardless of clock skew"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observation_selection_requires_nonconverged_complete_newer_data() {
|
||||
let authoritative = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(10),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let baseline = Some(authoritative.snapshot_identity());
|
||||
let candidate = |epoch, cycle, converged, complete| DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
|
||||
scanner_epoch: Some(epoch),
|
||||
scanner_cycle: Some(cycle),
|
||||
usage_snapshot_complete: complete,
|
||||
usage_snapshot_converged: converged,
|
||||
usage_snapshot_authoritative_baseline: baseline,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(observed_data_usage_is_newer(&candidate(2, 11, Some(false), true), &authoritative));
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 9, Some(false), true), &authoritative));
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(true), true), &authoritative));
|
||||
assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(false), false), &authoritative));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2280,44 +1901,6 @@ mod tests {
|
||||
assert_eq!(info.buckets_count, 2);
|
||||
assert!(info.buckets_usage.is_empty());
|
||||
assert_eq!(info.objects_total_count, 3);
|
||||
assert!(info.tier_stats.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dui_reports_tier_usage_from_the_flattened_tree() {
|
||||
let root_hash = hash_path("root");
|
||||
let bucket_hash = hash_path("bucket-a");
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "root".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.replace_hashed(&root_hash, &None, &DataUsageEntry::default());
|
||||
cache.replace_hashed(
|
||||
&bucket_hash,
|
||||
&Some(root_hash),
|
||||
&tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 40,
|
||||
num_versions: 2,
|
||||
num_objects: 2,
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
let info = cache.dui("root", &["bucket-a".to_string()]);
|
||||
|
||||
assert_eq!(
|
||||
info.tier_stats.expect("child tier usage should roll up to the root").tiers["WARM"],
|
||||
TierStats {
|
||||
total_size: 40,
|
||||
num_versions: 2,
|
||||
num_objects: 2,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -16,17 +16,91 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, signed_s3_request};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{
|
||||
AccelerateConfiguration, BucketAccelerateStatus, BucketLoggingStatus, IndexDocument, LoggingEnabled, Payer,
|
||||
RequestPaymentConfiguration, WebsiteConfiguration,
|
||||
};
|
||||
use http::Method;
|
||||
use http::header::CONTENT_TYPE;
|
||||
use serial_test::serial;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::info;
|
||||
|
||||
fn awscurl_binary_path() -> PathBuf {
|
||||
std::env::var_os("AWSCURL_PATH")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("awscurl"))
|
||||
}
|
||||
|
||||
fn awscurl_available() -> bool {
|
||||
Command::new(awscurl_binary_path()).arg("--version").output().is_ok()
|
||||
}
|
||||
|
||||
fn execute_s3_awscurl(
|
||||
method: &str,
|
||||
url: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let output = Command::new(awscurl_binary_path())
|
||||
.args([
|
||||
"--service",
|
||||
"s3",
|
||||
"--region",
|
||||
"us-east-1",
|
||||
"--access_key",
|
||||
access_key,
|
||||
"--secret_key",
|
||||
secret_key,
|
||||
"-i",
|
||||
"-X",
|
||||
method,
|
||||
url,
|
||||
])
|
||||
.output()?;
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into());
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&output.stdout).to_string())
|
||||
}
|
||||
|
||||
fn parse_status(raw: &str) -> Option<u16> {
|
||||
raw.lines()
|
||||
.filter_map(|line| {
|
||||
if line.starts_with("HTTP/") {
|
||||
line.split_whitespace().nth(1)?.parse::<u16>().ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.next_back()
|
||||
}
|
||||
|
||||
fn parse_body(raw: &str) -> String {
|
||||
if let Some(pos) = raw.rfind("\r\n\r\n") {
|
||||
return raw[pos + 4..].to_string();
|
||||
}
|
||||
if let Some(pos) = raw.rfind("\n\n") {
|
||||
return raw[pos + 2..].to_string();
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn parse_headers(raw: &str) -> String {
|
||||
let start = raw.rfind("HTTP/").unwrap_or(0);
|
||||
let tail = &raw[start..];
|
||||
if let Some(pos) = tail.find("\r\n\r\n") {
|
||||
return tail[..pos].to_string();
|
||||
}
|
||||
if let Some(pos) = tail.find("\n\n") {
|
||||
return tail[..pos].to_string();
|
||||
}
|
||||
tail.to_string()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_dummy_bucket_compatibility_endpoints() {
|
||||
@@ -396,6 +470,10 @@ mod tests {
|
||||
async fn test_dummy_bucket_endpoints_http_contracts() {
|
||||
init_logging();
|
||||
info!("Starting test: dummy-compat bucket API HTTP contracts");
|
||||
if !awscurl_available() {
|
||||
info!("Skipping test_dummy_bucket_endpoints_http_contracts: awscurl binary not found");
|
||||
return;
|
||||
}
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
@@ -410,112 +488,56 @@ mod tests {
|
||||
.await
|
||||
.expect("Failed to create bucket");
|
||||
|
||||
let logging_response = signed_s3_request(
|
||||
Method::GET,
|
||||
&format!("{}/{bucket}?logging=", env.url),
|
||||
None,
|
||||
None,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await
|
||||
.expect("GetBucketLogging HTTP request failed");
|
||||
assert_eq!(logging_response.status(), 200, "GetBucketLogging should return 200");
|
||||
let logging_body = logging_response
|
||||
.text()
|
||||
.await
|
||||
.expect("Failed to read GetBucketLogging response body");
|
||||
let logging_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?logging=", env.url), &env.access_key, &env.secret_key)
|
||||
.expect("GetBucketLogging HTTP request failed");
|
||||
assert_eq!(parse_status(&logging_raw), Some(200), "GetBucketLogging should return 200");
|
||||
let logging_body = parse_body(&logging_raw);
|
||||
assert!(
|
||||
logging_body.contains("<BucketLoggingStatus"),
|
||||
"GetBucketLogging response should contain BucketLoggingStatus XML, got: {logging_body}"
|
||||
);
|
||||
|
||||
let accel_response = signed_s3_request(
|
||||
Method::GET,
|
||||
&format!("{}/{bucket}?accelerate=", env.url),
|
||||
None,
|
||||
None,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await
|
||||
.expect("GetBucketAccelerateConfiguration HTTP request failed");
|
||||
assert_eq!(accel_response.status(), 200, "GetBucketAccelerateConfiguration should return 200");
|
||||
let accel_body = accel_response
|
||||
.text()
|
||||
.await
|
||||
.expect("Failed to read GetBucketAccelerateConfiguration response body");
|
||||
let accel_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?accelerate=", env.url), &env.access_key, &env.secret_key)
|
||||
.expect("GetBucketAccelerateConfiguration HTTP request failed");
|
||||
assert_eq!(parse_status(&accel_raw), Some(200), "GetBucketAccelerateConfiguration should return 200");
|
||||
let accel_body = parse_body(&accel_raw);
|
||||
assert!(
|
||||
accel_body.contains("<AccelerateConfiguration"),
|
||||
"GetBucketAccelerateConfiguration response should contain AccelerateConfiguration XML, got: {accel_body}"
|
||||
);
|
||||
|
||||
let payment_response = signed_s3_request(
|
||||
Method::GET,
|
||||
&format!("{}/{bucket}?requestPayment=", env.url),
|
||||
None,
|
||||
None,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await
|
||||
.expect("GetBucketRequestPayment HTTP request failed");
|
||||
assert_eq!(payment_response.status(), 200, "GetBucketRequestPayment should return 200");
|
||||
let payment_body = payment_response
|
||||
.text()
|
||||
.await
|
||||
.expect("Failed to read GetBucketRequestPayment response body");
|
||||
let payment_raw =
|
||||
execute_s3_awscurl("GET", &format!("{}/{bucket}?requestPayment=", env.url), &env.access_key, &env.secret_key)
|
||||
.expect("GetBucketRequestPayment HTTP request failed");
|
||||
assert_eq!(parse_status(&payment_raw), Some(200), "GetBucketRequestPayment should return 200");
|
||||
let payment_body = parse_body(&payment_raw);
|
||||
assert!(
|
||||
payment_body.contains("<Payer>BucketOwner</Payer>"),
|
||||
"GetBucketRequestPayment should return BucketOwner payer, got: {payment_body}"
|
||||
);
|
||||
|
||||
let website_response = signed_s3_request(
|
||||
Method::GET,
|
||||
&format!("{}/{bucket}?website=", env.url),
|
||||
None,
|
||||
None,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await
|
||||
.expect("GetBucketWebsite HTTP request failed");
|
||||
let website_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
|
||||
.expect("GetBucketWebsite HTTP request failed");
|
||||
assert_eq!(
|
||||
website_response.status(),
|
||||
404,
|
||||
parse_status(&website_raw),
|
||||
Some(404),
|
||||
"GetBucketWebsite should return 404 when website config is absent"
|
||||
);
|
||||
let website_content_type = website_response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.expect("GetBucketWebsite response should include Content-Type")
|
||||
.to_str()
|
||||
.expect("GetBucketWebsite Content-Type should be valid ASCII")
|
||||
.to_ascii_lowercase();
|
||||
let website_content_type = parse_headers(&website_raw).to_ascii_lowercase();
|
||||
assert!(
|
||||
website_content_type.contains("xml"),
|
||||
website_content_type.contains("content-type:") && website_content_type.contains("xml"),
|
||||
"GetBucketWebsite error response should be XML, got content-type: {website_content_type}"
|
||||
);
|
||||
let website_body = website_response
|
||||
.text()
|
||||
.await
|
||||
.expect("Failed to read GetBucketWebsite response body");
|
||||
let website_body = parse_body(&website_raw);
|
||||
assert!(
|
||||
website_body.contains("<Code>NoSuchWebsiteConfiguration</Code>"),
|
||||
"GetBucketWebsite should return NoSuchWebsiteConfiguration code, got: {website_body}"
|
||||
);
|
||||
|
||||
let delete_response = signed_s3_request(
|
||||
Method::DELETE,
|
||||
&format!("{}/{bucket}?website=", env.url),
|
||||
None,
|
||||
None,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await
|
||||
.expect("DeleteBucketWebsite HTTP request failed");
|
||||
assert_eq!(delete_response.status(), 204, "DeleteBucketWebsite should return 204");
|
||||
let delete_raw =
|
||||
execute_s3_awscurl("DELETE", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
|
||||
.expect("DeleteBucketWebsite HTTP request failed");
|
||||
assert_eq!(parse_status(&delete_raw), Some(204), "DeleteBucketWebsite should return 204");
|
||||
|
||||
env.stop_server();
|
||||
}
|
||||
|
||||
@@ -1,295 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression tests for bucket statistics and data usage accuracy.
|
||||
//!
|
||||
//! Covers the recurring pattern where bucket statistics (object count, size)
|
||||
//! show stale/incorrect values, remain at 0, or oscillate between complete,
|
||||
//! partial, and zero. This has regressed 10+ times.
|
||||
//!
|
||||
//! ## Regression Issues
|
||||
//!
|
||||
//! - rustfs#5615: bucket statistics remain unchanged after data expiration
|
||||
//! - rustfs#5008: Admin usage reports only one pool
|
||||
//! - rustfs#5116: Admin usage reports stale 0/0 for non-empty bucket after upgrade
|
||||
//! - rustfs#5055: console object count and size still loading
|
||||
//! - rustfs#5010: Storage usage info changed abnormally
|
||||
//! - rustfs#3662: Incorrect bucket, object count and size
|
||||
//! - rustfs#3898: DataUsageInfo undercounts versioned bucket versions
|
||||
//! - rustfs#1012: Object count in the console doesn't change
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, awscurl_get, init_logging};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use rustfs_data_usage::DataUsageInfo;
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
|
||||
async fn get_data_usage(env: &RustFSTestEnvironment) -> Result<DataUsageInfo, Box<dyn Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/datausageinfo", env.url);
|
||||
let resp = awscurl_get(&url, &env.access_key, &env.secret_key).await?;
|
||||
Ok(serde_json::from_str(&resp)?)
|
||||
}
|
||||
|
||||
/// RT-09: Verify bucket object count updates after PUT.
|
||||
///
|
||||
/// Regression pattern: bucket stats remain at 0 after objects are uploaded
|
||||
/// (rustfs#5055, rustfs#1012).
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. Create a bucket
|
||||
/// 2. Upload 10 objects
|
||||
/// 3. Query admin data usage API
|
||||
/// 4. Verify object count > 0
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_object_count_updates_after_put() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-09: bucket object count updates after PUT");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV)
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt09-stats-put";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
// Upload 10 objects
|
||||
for i in 0..10 {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(format!("stat-obj-{i:04}.txt"))
|
||||
.body(ByteStream::from_static(b"statistical data"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object");
|
||||
}
|
||||
|
||||
// Wait for scanner to process (up to 90 seconds)
|
||||
let mut found_nonzero = false;
|
||||
let mut last_query_error = None;
|
||||
for attempt in 0..18 {
|
||||
sleep(Duration::from_secs(5)).await;
|
||||
|
||||
let usage = match get_data_usage(&env).await {
|
||||
Ok(usage) => {
|
||||
last_query_error = None;
|
||||
usage
|
||||
}
|
||||
Err(err) => {
|
||||
last_query_error = Some(err.to_string());
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(bucket_usage) = usage.buckets_usage.get(bucket) {
|
||||
info!(" attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
|
||||
if bucket_usage.objects_count >= 10 {
|
||||
found_nonzero = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
found_nonzero,
|
||||
"RT-09 FAIL: bucket object count did not update after PUT 10 objects (regression: stats stuck at 0); last query error: {}",
|
||||
last_query_error.as_deref().unwrap_or("none")
|
||||
);
|
||||
|
||||
info!("RT-09 PASS: bucket object count updates after PUT");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-09b: Verify bucket stats update after DELETE.
|
||||
///
|
||||
/// Regression pattern: stats remain unchanged after objects are deleted
|
||||
/// (rustfs#5615).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_object_count_updates_after_delete() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-09b: bucket object count updates after DELETE");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV)
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt09b-stats-delete";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
// Upload 5 objects
|
||||
for i in 0..5 {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(format!("del-stat-{i}.txt"))
|
||||
.body(ByteStream::from_static(b"data"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object");
|
||||
}
|
||||
|
||||
let mut found_nonzero = false;
|
||||
for attempt in 0..18 {
|
||||
sleep(Duration::from_secs(5)).await;
|
||||
|
||||
if let Ok(usage) = get_data_usage(&env).await
|
||||
&& let Some(bucket_usage) = usage.buckets_usage.get(bucket)
|
||||
{
|
||||
info!(" baseline attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
|
||||
if bucket_usage.objects_count >= 5 {
|
||||
found_nonzero = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(found_nonzero, "RT-09b setup failed: scanner did not observe the 5 uploaded objects");
|
||||
|
||||
// Delete all objects
|
||||
for i in 0..5 {
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(format!("del-stat-{i}.txt"))
|
||||
.send()
|
||||
.await
|
||||
.expect("delete object");
|
||||
}
|
||||
|
||||
// Wait for scanner to update stats (up to 90 seconds)
|
||||
let mut found_zero = false;
|
||||
let mut last_query_error = None;
|
||||
for attempt in 0..18 {
|
||||
sleep(Duration::from_secs(5)).await;
|
||||
|
||||
let usage = match get_data_usage(&env).await {
|
||||
Ok(usage) => {
|
||||
last_query_error = None;
|
||||
usage
|
||||
}
|
||||
Err(err) => {
|
||||
last_query_error = Some(err.to_string());
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(bucket_usage) = usage.buckets_usage.get(bucket) {
|
||||
info!(" attempt {attempt}: objectsCount = {}", bucket_usage.objects_count);
|
||||
if bucket_usage.objects_count == 0 {
|
||||
found_zero = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
found_zero,
|
||||
"RT-09b FAIL: bucket object count did not update to 0 after deleting all objects (regression rustfs#5615); last query error: {}",
|
||||
last_query_error.as_deref().unwrap_or("none")
|
||||
);
|
||||
|
||||
info!("RT-09b PASS: bucket object count updates to 0 after DELETE");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-09c: Verify versioned bucket stats count all versions.
|
||||
///
|
||||
/// Regression pattern: DataUsageInfo undercounts versioned bucket versions
|
||||
/// and delete markers (rustfs#3898).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_versioned_bucket_stats_count_all_versions() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-09c: versioned bucket stats count all versions");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt09c-versioned-stats";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("enable versioning");
|
||||
|
||||
// Create 3 versions of the same object
|
||||
for i in 0..3 {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("multi-version.txt")
|
||||
.body(ByteStream::from(format!("version-{i}").into_bytes()))
|
||||
.send()
|
||||
.await
|
||||
.expect("put version");
|
||||
}
|
||||
|
||||
// Create a delete marker
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key("multi-version.txt")
|
||||
.send()
|
||||
.await
|
||||
.expect("create delete marker");
|
||||
|
||||
// Verify versions via API (immediate, no scanner wait)
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions");
|
||||
|
||||
assert_eq!(
|
||||
versions.versions().len(),
|
||||
3,
|
||||
"RT-09c FAIL: expected 3 versions, found {}",
|
||||
versions.versions().len()
|
||||
);
|
||||
assert_eq!(
|
||||
versions.delete_markers().len(),
|
||||
1,
|
||||
"RT-09c FAIL: expected 1 delete marker, found {}",
|
||||
versions.delete_markers().len()
|
||||
);
|
||||
|
||||
info!("RT-09c PASS: versioned bucket correctly tracks all versions and delete markers");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
+27
-105
@@ -47,34 +47,11 @@ use walkdir::WalkDir;
|
||||
pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
|
||||
pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
|
||||
pub const ENV_RUSTFS_BUILD_FEATURES: &str = "RUSTFS_BUILD_FEATURES";
|
||||
pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
|
||||
&[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")];
|
||||
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
||||
const RUSTFS_FULL_FEATURE: &str = "full";
|
||||
|
||||
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
||||
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
||||
Some(log_dir.join(format!("{temp_name}.log")))
|
||||
}
|
||||
|
||||
fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
||||
let log_dir = std::env::var_os("RUSTFS_E2E_LOG_DIR")?;
|
||||
if stdfs::create_dir_all(&log_dir).is_err() {
|
||||
warn!(?log_dir, "failed to create configured E2E server log directory");
|
||||
return None;
|
||||
}
|
||||
|
||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
pub(crate) fn build_test_s3_config(
|
||||
endpoint_url: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
session_token: Option<&str>,
|
||||
provider_name: &'static str,
|
||||
) -> Config {
|
||||
let credentials = Credentials::new(access_key, secret_key, session_token.map(str::to_owned), None, provider_name);
|
||||
fn build_test_s3_config(endpoint_url: &str, access_key: &str, secret_key: &str, provider_name: &'static str) -> Config {
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, provider_name);
|
||||
let mut config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
@@ -89,33 +66,6 @@ pub(crate) fn build_test_s3_config(
|
||||
config.build()
|
||||
}
|
||||
|
||||
pub(crate) fn build_test_sts_client(
|
||||
endpoint_url: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
session_token: Option<&str>,
|
||||
provider_name: &'static str,
|
||||
) -> aws_sdk_sts::Client {
|
||||
let mut config = aws_sdk_sts::Config::builder()
|
||||
.credentials_provider(aws_sdk_sts::config::Credentials::new(
|
||||
access_key,
|
||||
secret_key,
|
||||
session_token.map(str::to_owned),
|
||||
None,
|
||||
provider_name,
|
||||
))
|
||||
.region(aws_sdk_sts::config::Region::new("us-east-1"))
|
||||
.endpoint_url(endpoint_url)
|
||||
.retry_config(aws_sdk_sts::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.behavior_version_latest();
|
||||
|
||||
if endpoint_url.starts_with("http://") {
|
||||
config = config.http_client(SmithyHttpClientBuilder::new().build_http());
|
||||
}
|
||||
|
||||
aws_sdk_sts::Client::from_conf(config.build())
|
||||
}
|
||||
|
||||
pub fn workspace_root() -> PathBuf {
|
||||
let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||
path.pop(); // e2e_test
|
||||
@@ -130,38 +80,6 @@ pub fn local_http_client() -> HttpClient {
|
||||
.expect("failed to build local reqwest client")
|
||||
}
|
||||
|
||||
pub(crate) async fn signed_s3_request(
|
||||
method: http::Method,
|
||||
url: &str,
|
||||
body: Option<String>,
|
||||
content_type: Option<&str>,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string();
|
||||
let mut request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
if let Some(content_type) = content_type {
|
||||
request = request.header(CONTENT_TYPE, content_type);
|
||||
}
|
||||
|
||||
let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?;
|
||||
let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let mut request = local_http_client().request(method, url);
|
||||
for (name, value) in signed.headers() {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
if let Some(body) = body {
|
||||
request = request.body(body);
|
||||
}
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
/// Signs and sends an admin HTTP request with the given credentials.
|
||||
pub(crate) async fn admin_request(
|
||||
base_url: &str,
|
||||
@@ -172,8 +90,28 @@ pub(crate) async fn admin_request(
|
||||
secret_key: &str,
|
||||
) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{base_url}{path_and_query}");
|
||||
let content_type = body.as_ref().map(|_| "application/json");
|
||||
let response = signed_s3_request(method, &url, body, content_type, access_key, secret_key).await?;
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
|
||||
let mut request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
if body.is_some() {
|
||||
request = request.header(CONTENT_TYPE, "application/json");
|
||||
}
|
||||
|
||||
let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "admin request body is too large")?;
|
||||
let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let mut request = local_http_client().request(method, &url);
|
||||
for (name, value) in signed.headers() {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
if let Some(body) = body {
|
||||
request = request.body(body);
|
||||
}
|
||||
let response = request.send().await?;
|
||||
let status = response.status();
|
||||
let body = response.text().await?;
|
||||
Ok((status, body))
|
||||
@@ -423,7 +361,6 @@ impl RustFSTestEnvironment {
|
||||
pub async fn new() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4());
|
||||
fs::create_dir_all(&temp_dir).await?;
|
||||
let capture_log_path = configured_capture_log_path(&temp_dir);
|
||||
|
||||
// Use a unique port for each test environment
|
||||
let port = Self::find_available_port().await?;
|
||||
@@ -437,7 +374,7 @@ impl RustFSTestEnvironment {
|
||||
access_key: DEFAULT_ACCESS_KEY.to_string(),
|
||||
secret_key: DEFAULT_SECRET_KEY.to_string(),
|
||||
process: None,
|
||||
capture_log_path,
|
||||
capture_log_path: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -445,7 +382,6 @@ impl RustFSTestEnvironment {
|
||||
pub async fn with_address(address: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4());
|
||||
fs::create_dir_all(&temp_dir).await?;
|
||||
let capture_log_path = configured_capture_log_path(&temp_dir);
|
||||
|
||||
let url = format!("http://{address}");
|
||||
|
||||
@@ -456,7 +392,7 @@ impl RustFSTestEnvironment {
|
||||
access_key: DEFAULT_ACCESS_KEY.to_string(),
|
||||
secret_key: DEFAULT_SECRET_KEY.to_string(),
|
||||
process: None,
|
||||
capture_log_path,
|
||||
capture_log_path: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -611,12 +547,7 @@ impl RustFSTestEnvironment {
|
||||
|
||||
/// Create an AWS S3 client configured for this RustFS instance
|
||||
pub fn create_s3_client(&self) -> Client {
|
||||
self.create_s3_client_with_credentials(&self.access_key, &self.secret_key)
|
||||
}
|
||||
|
||||
/// Create an AWS S3 client with explicit credentials for this RustFS instance.
|
||||
pub fn create_s3_client_with_credentials(&self, access_key: &str, secret_key: &str) -> Client {
|
||||
Client::from_conf(build_test_s3_config(&self.url, access_key, secret_key, None, "e2e-test"))
|
||||
Client::from_conf(build_test_s3_config(&self.url, &self.access_key, &self.secret_key, "e2e-test"))
|
||||
}
|
||||
|
||||
/// Create test bucket
|
||||
@@ -1348,7 +1279,6 @@ impl RustFSTestClusterEnvironment {
|
||||
&self.nodes[node_idx].url,
|
||||
&self.access_key,
|
||||
&self.secret_key,
|
||||
None,
|
||||
"cluster-test",
|
||||
)))
|
||||
}
|
||||
@@ -1462,14 +1392,6 @@ mod tests {
|
||||
assert_eq!(normalize_rustfs_build_features(" , "), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capture_log_path_uses_temp_directory_basename() {
|
||||
assert_eq!(
|
||||
capture_log_path(Path::new("/tmp/e2e-logs"), "/tmp/rustfs_e2e_test_abc"),
|
||||
Some(PathBuf::from("/tmp/e2e-logs/rustfs_e2e_test_abc.log"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_feature_enables_any_required_feature() {
|
||||
assert!(rustfs_build_feature_enabled(Some("full"), "sftp"));
|
||||
|
||||
@@ -18,7 +18,7 @@ use rustfs_data_usage::DataUsageInfo;
|
||||
use serial_test::serial;
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging};
|
||||
|
||||
async fn get_data_usage_info(env: &RustFSTestEnvironment) -> Result<DataUsageInfo, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/datausageinfo", env.url);
|
||||
@@ -35,26 +35,16 @@ where
|
||||
F: FnMut(&DataUsageInfo) -> bool,
|
||||
{
|
||||
let mut last_usage = DataUsageInfo::default();
|
||||
let mut last_query_error = None;
|
||||
for _ in 0..45 {
|
||||
match get_data_usage_info(env).await {
|
||||
Ok(usage) => {
|
||||
last_query_error = None;
|
||||
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
|
||||
return Ok(usage);
|
||||
}
|
||||
last_usage = usage;
|
||||
}
|
||||
Err(err) => last_query_error = Some(err.to_string()),
|
||||
let usage = get_data_usage_info(env).await?;
|
||||
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
|
||||
return Ok(usage);
|
||||
}
|
||||
last_usage = usage;
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
|
||||
Err(format!(
|
||||
"bucket usage did not converge for {bucket}; last usage: {last_usage:?}; last query error: {}",
|
||||
last_query_error.as_deref().unwrap_or("none")
|
||||
)
|
||||
.into())
|
||||
Err(format!("bucket usage did not converge for {bucket}; last usage: {last_usage:?}").into())
|
||||
}
|
||||
|
||||
/// Regression test for data usage accuracy (issue #1012).
|
||||
@@ -66,7 +56,7 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV).await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
|
||||
@@ -84,14 +74,8 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
|
||||
.await?;
|
||||
}
|
||||
|
||||
let usage = wait_for_bucket_usage(&env, TEST_BUCKET, |usage| {
|
||||
usage
|
||||
.buckets_usage
|
||||
.get(TEST_BUCKET)
|
||||
.map(|bucket_usage| usage.objects_total_count >= 1000 && bucket_usage.objects_count >= 1000)
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.await?;
|
||||
// Query admin data usage API
|
||||
let usage = get_data_usage_info(&env).await?;
|
||||
|
||||
// Assert total object count and per-bucket count are not truncated
|
||||
let bucket_usage = usage
|
||||
@@ -124,7 +108,7 @@ async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(),
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], FAST_DATA_USAGE_SCANNER_ENV).await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "data-usage-versioned";
|
||||
@@ -200,8 +184,8 @@ async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(),
|
||||
assert_eq!(usage.versions_total_count, 3, "total version count should match bucket usage");
|
||||
assert_eq!(usage.delete_markers_total_count, 1, "total delete marker count should match bucket usage");
|
||||
|
||||
env.restart_server_preserving_data(vec![], FAST_DATA_USAGE_SCANNER_ENV)
|
||||
.await?;
|
||||
env.stop_server();
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let restarted_usage = wait_for_bucket_usage(&env, bucket, |usage| {
|
||||
usage
|
||||
|
||||
@@ -1,445 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression tests for object delete operations.
|
||||
//!
|
||||
//! Covers the recurring pattern where DELETE succeeds at the API level but the
|
||||
//! object remains visible in LIST, or deleted objects reappear after restart,
|
||||
//! or versioned delete operations fail with FileAccessDenied.
|
||||
//! This has regressed 15+ times across the entire release history.
|
||||
//!
|
||||
//! ## Regression Issues
|
||||
//!
|
||||
//! - rustfs#5375: delete object in a bucket list api also exist this object
|
||||
//! - rustfs#5349: The deleted bucket was rebuilt after some time
|
||||
//! - rustfs#5339: data not delete in Object Lock bucket
|
||||
//! - rustfs#5029: Node Does Not Remove Files After Reconnect to Cluster
|
||||
//! - rustfs#4978: DELETE fails with InternalError/FileAccessDenied on beta 10
|
||||
//! - rustfs#760: Cannot delete a versioned bucket
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
|
||||
/// RT-05: Verify DELETE → LIST → HEAD consistency.
|
||||
///
|
||||
/// Regression pattern: DELETE returns 200 but the object remains in LIST.
|
||||
/// Covers rustfs#5375.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. Create a bucket and upload an object
|
||||
/// 2. Verify the object is in LIST
|
||||
/// 3. DELETE the object
|
||||
/// 4. Verify the object is NOT in LIST
|
||||
/// 5. Verify HEAD returns 404
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_removes_object_from_list() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-05: delete removes object from list");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt05-delete-consistency";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
// Upload an object
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("to-delete.txt")
|
||||
.body(ByteStream::from_static(b"will be deleted"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object");
|
||||
|
||||
// Verify it appears in LIST
|
||||
let list = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("list objects before delete");
|
||||
|
||||
assert!(
|
||||
list.contents()
|
||||
.iter()
|
||||
.map(|o| o.key().unwrap_or(""))
|
||||
.any(|key| key == "to-delete.txt"),
|
||||
"RT-05 FAIL: object not in LIST before delete"
|
||||
);
|
||||
|
||||
// DELETE
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key("to-delete.txt")
|
||||
.send()
|
||||
.await
|
||||
.expect("delete object");
|
||||
|
||||
// Verify NOT in LIST
|
||||
let list = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("list objects after delete");
|
||||
|
||||
assert!(
|
||||
!list
|
||||
.contents()
|
||||
.iter()
|
||||
.map(|o| o.key().unwrap_or(""))
|
||||
.any(|key| key == "to-delete.txt"),
|
||||
"RT-05 FAIL: deleted object still in LIST (regression rustfs#5375)"
|
||||
);
|
||||
|
||||
// Verify HEAD returns 404
|
||||
let head = client.head_object().bucket(bucket).key("to-delete.txt").send().await;
|
||||
|
||||
assert!(head.is_err(), "RT-05 FAIL: HEAD on deleted object should return error, got success");
|
||||
|
||||
info!("RT-05 PASS: delete correctly removes object from LIST and HEAD");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-05c: Verify batch delete (DeleteObjects) consistency.
|
||||
///
|
||||
/// Regression pattern: batch delete returns success but some objects
|
||||
/// remain in LIST.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_batch_delete_removes_all_objects() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-05c: batch delete removes all objects");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt05c-batch-delete";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
// Upload multiple objects
|
||||
let keys: Vec<String> = (0..5).map(|i| format!("batch-{i:04}.txt")).collect();
|
||||
for key in &keys {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"batch-delete-me"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object");
|
||||
}
|
||||
|
||||
// Verify all in LIST
|
||||
let list = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("list before batch delete");
|
||||
|
||||
assert_eq!(
|
||||
list.contents().len(),
|
||||
5,
|
||||
"RT-05c FAIL: expected 5 objects before batch delete, found {}",
|
||||
list.contents().len()
|
||||
);
|
||||
|
||||
// Batch delete
|
||||
let objects: Vec<ObjectIdentifier> = keys
|
||||
.iter()
|
||||
.map(|k| ObjectIdentifier::builder().key(k).build().expect("build object id"))
|
||||
.collect();
|
||||
|
||||
client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(Delete::builder().set_objects(Some(objects)).build().expect("build delete"))
|
||||
.send()
|
||||
.await
|
||||
.expect("batch delete");
|
||||
|
||||
// Verify all removed
|
||||
let list = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("list after batch delete");
|
||||
|
||||
assert!(
|
||||
list.contents().is_empty(),
|
||||
"RT-05c FAIL: {} objects remain after batch delete (regression: delete objects not fully applied)",
|
||||
list.contents().len()
|
||||
);
|
||||
|
||||
info!("RT-05c PASS: batch delete removes all objects");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-05d: Verify versioned delete → permanent delete → object gone.
|
||||
///
|
||||
/// Covers the pattern where permanent deletion of a specific version
|
||||
/// fails with FileAccessDenied (rustfs#4978).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_versioned_permanent_delete() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-05d: versioned permanent delete");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt05d-permanent-delete";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("enable versioning");
|
||||
|
||||
// Upload a single object (single version)
|
||||
let put_resp = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("single-version.txt")
|
||||
.body(ByteStream::from_static(b"to-be-permanently-deleted"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object");
|
||||
|
||||
let version_id = put_resp.version_id().expect("version ID should be present").to_string();
|
||||
|
||||
// Permanently delete the specific version (rustfs#4978: FileAccessDenied)
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key("single-version.txt")
|
||||
.version_id(&version_id)
|
||||
.send()
|
||||
.await
|
||||
.expect("permanent delete should succeed (regression rustfs#4978)");
|
||||
|
||||
// Verify the object is completely gone
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions");
|
||||
|
||||
assert!(
|
||||
versions.versions().is_empty(),
|
||||
"RT-05d FAIL: version still present after permanent delete"
|
||||
);
|
||||
|
||||
info!("RT-05d PASS: versioned permanent delete succeeds");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-05e: Verify delete marker + version history interaction.
|
||||
///
|
||||
/// Covers the pattern where creating a delete marker and then listing
|
||||
/// versions shows incorrect state (rustfs#760).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_versioned_delete_marker_and_list_consistency() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-05e: versioned delete marker and list consistency");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt05e-dm-consistency";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("enable versioning");
|
||||
|
||||
// Create 3 versions
|
||||
for i in 0..3 {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("history.txt")
|
||||
.body(ByteStream::from(format!("v{i}").into_bytes()))
|
||||
.send()
|
||||
.await
|
||||
.expect("put version");
|
||||
}
|
||||
|
||||
// Create a delete marker
|
||||
let del = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key("history.txt")
|
||||
.send()
|
||||
.await
|
||||
.expect("delete (create marker)");
|
||||
|
||||
assert!(del.delete_marker().unwrap_or(false), "RT-05e FAIL: should have created a delete marker");
|
||||
|
||||
// ListObjectVersions should show 3 versions + 1 delete marker
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions");
|
||||
|
||||
assert_eq!(
|
||||
versions.versions().len(),
|
||||
3,
|
||||
"RT-05e FAIL: expected 3 versions, found {}",
|
||||
versions.versions().len()
|
||||
);
|
||||
assert_eq!(
|
||||
versions.delete_markers().len(),
|
||||
1,
|
||||
"RT-05e FAIL: expected 1 delete marker, found {}",
|
||||
versions.delete_markers().len()
|
||||
);
|
||||
|
||||
// Now delete the delete marker (restore the object)
|
||||
let dm_version = &versions.delete_markers()[0];
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key("history.txt")
|
||||
.version_id(dm_version.version_id().expect("dm version id"))
|
||||
.send()
|
||||
.await
|
||||
.expect("delete delete-marker");
|
||||
|
||||
// HEAD should succeed now (latest version is accessible)
|
||||
let head = client.head_object().bucket(bucket).key("history.txt").send().await;
|
||||
|
||||
assert!(head.is_ok(), "RT-05e FAIL: HEAD should succeed after removing delete marker");
|
||||
|
||||
info!("RT-05e PASS: versioned delete marker and list consistency");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-05f: Verify object deletion does not leave orphan data on disk.
|
||||
///
|
||||
/// Regression pattern: after delete, the object data files remain on disk
|
||||
/// (rustfs#5029: Node Does Not Remove Files After Reconnect).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_removes_object_head_returns_404() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-05f: delete → HEAD 404 consistency");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt05f-delete-head";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
// Upload, delete, verify HEAD returns 404
|
||||
let keys = vec!["small.txt", "medium.txt", "with-slash.txt", "special+chars.txt"];
|
||||
|
||||
for key in &keys {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(*key)
|
||||
.body(ByteStream::from_static(b"delete-me"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object");
|
||||
}
|
||||
|
||||
for key in &keys {
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(*key)
|
||||
.send()
|
||||
.await
|
||||
.expect("delete object");
|
||||
}
|
||||
|
||||
// All HEAD requests should return 404
|
||||
for key in &keys {
|
||||
let head = client.head_object().bucket(bucket).key(*key).send().await;
|
||||
|
||||
assert!(head.is_err(), "RT-05f FAIL: HEAD on deleted key '{key}' should return error");
|
||||
}
|
||||
|
||||
// LIST should be empty
|
||||
let list = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("list after all deletes");
|
||||
|
||||
assert!(
|
||||
list.contents().is_empty(),
|
||||
"RT-05f FAIL: {} objects remain after deleting all",
|
||||
list.contents().len()
|
||||
);
|
||||
|
||||
info!("RT-05f PASS: all deleted objects return 404 on HEAD");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,202 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression tests for distributed cluster startup and quorum.
|
||||
//!
|
||||
//! Covers the recurring pattern where multi-node clusters fail to start due to
|
||||
//! lock quorum issues, DNS resolution delays, or erasure quorum deadlocks.
|
||||
//! This has regressed 7+ times.
|
||||
//!
|
||||
//! ## Regression Issues
|
||||
//!
|
||||
//! - rustfs#5416: RustFS cannot cold-start with 2/3 quorum when Pod DNS missing
|
||||
//! - rustfs#2945: Distributed mode fails on K8s: erasure quorum deadlock
|
||||
//! - rustfs#2794: distributed deployment does not become ready
|
||||
//! - rustfs#2601: fresh pod immediately enters FaultyDisk state
|
||||
//! - rustfs#4040: Distributed startup can fail lock quorum before AppContext initializes
|
||||
//! - rustfs#5655: fix(ecstore): bootstrap fresh four-node clusters reliably
|
||||
//! - rustfs#4954: S3/health endpoint unavailability after multi-pool scale-up
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestClusterEnvironment, init_logging};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
|
||||
/// RT-10: Verify 4-node cluster starts successfully and all nodes are ready.
|
||||
///
|
||||
/// Regression pattern: distributed startup fails with quorum deadlock or
|
||||
/// lock acquisition timeout (rustfs#2945, rustfs#5655).
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. Create a 4-node cluster
|
||||
/// 2. Start all nodes simultaneously
|
||||
/// 3. Verify all nodes report healthy
|
||||
/// 4. Verify S3 operations work through any node
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_four_node_cluster_startup_and_health() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-10: 4-node cluster startup and health");
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await.expect("create 4-node cluster");
|
||||
|
||||
cluster.start().await.expect("start 4-node cluster");
|
||||
|
||||
// Create a bucket and verify it's accessible from all nodes
|
||||
cluster
|
||||
.create_test_bucket("rt10-startup")
|
||||
.await
|
||||
.expect("create bucket on cluster");
|
||||
|
||||
let clients = cluster.create_all_clients().expect("create per-node clients");
|
||||
|
||||
// Verify S3 operations work from every node
|
||||
for (i, client) in clients.iter().enumerate() {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("rt10-startup")
|
||||
.key(format!("from-node-{i}.txt"))
|
||||
.body(ByteStream::from_static(b"hello from node"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("PUT from node {i} failed: {e}"));
|
||||
}
|
||||
|
||||
// Verify all objects are visible from node 0
|
||||
let list = clients[0]
|
||||
.list_objects_v2()
|
||||
.bucket("rt10-startup")
|
||||
.send()
|
||||
.await
|
||||
.expect("list objects from node 0");
|
||||
|
||||
assert_eq!(
|
||||
list.contents().len(),
|
||||
4,
|
||||
"RT-10 FAIL: expected 4 objects (one per node), found {}",
|
||||
list.contents().len()
|
||||
);
|
||||
|
||||
info!("RT-10 PASS: 4-node cluster starts and serves S3 from all nodes");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-10b: Verify cluster handles node restart gracefully.
|
||||
///
|
||||
/// Regression pattern: after a node restart, it cannot rejoin the cluster
|
||||
/// or enters a faulty state (rustfs#2601).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_cluster_survives_node_restart() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-10b: cluster survives node restart");
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await.expect("create 4-node cluster");
|
||||
|
||||
cluster.start().await.expect("start cluster");
|
||||
|
||||
cluster.create_test_bucket("rt10b-restart").await.expect("create bucket");
|
||||
|
||||
// Write data
|
||||
let clients = cluster.create_all_clients()?;
|
||||
clients[0]
|
||||
.put_object()
|
||||
.bucket("rt10b-restart")
|
||||
.key("before-restart.txt")
|
||||
.body(ByteStream::from_static(b"persistent data"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object before restart");
|
||||
|
||||
// Stop node 3
|
||||
cluster.stop_node(3).expect("stop node 3");
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
|
||||
// Verify cluster still works with 3/4 nodes (quorum)
|
||||
clients[0]
|
||||
.put_object()
|
||||
.bucket("rt10b-restart")
|
||||
.key("during-offline.txt")
|
||||
.body(ByteStream::from_static(b"written while node 3 down"))
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT should succeed with 3/4 nodes");
|
||||
|
||||
// Restart node 3
|
||||
cluster.start_node(3).await.expect("restart node 3");
|
||||
|
||||
// Wait for node to rejoin
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
|
||||
// Verify the restarted node can serve reads
|
||||
let list = clients[3]
|
||||
.list_objects_v2()
|
||||
.bucket("rt10b-restart")
|
||||
.send()
|
||||
.await
|
||||
.expect("list from restarted node");
|
||||
|
||||
assert!(
|
||||
list.contents().len() >= 2,
|
||||
"RT-10b FAIL: restarted node sees {} objects, expected >= 2",
|
||||
list.contents().len()
|
||||
);
|
||||
|
||||
info!("RT-10b PASS: cluster survives and recovers from node restart");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-10c: Verify bucket creation persists across all nodes.
|
||||
///
|
||||
/// Regression pattern: bucket metadata is not replicated to all nodes,
|
||||
/// causing NoSuchBucket errors on some nodes (rustfs#3191).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_visible_from_all_nodes() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-10c: bucket visible from all nodes");
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await.expect("create 4-node cluster");
|
||||
|
||||
cluster.start().await.expect("start cluster");
|
||||
|
||||
cluster
|
||||
.create_test_bucket("rt10c-bucket-visibility")
|
||||
.await
|
||||
.expect("create bucket");
|
||||
|
||||
let clients = cluster.create_all_clients()?;
|
||||
|
||||
// Verify the bucket is visible from every node
|
||||
for (i, client) in clients.iter().enumerate() {
|
||||
let resp = client
|
||||
.list_objects_v2()
|
||||
.bucket("rt10c-bucket-visibility")
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("list from node {i} failed (NoSuchBucket?): {e}"));
|
||||
|
||||
assert!(resp.contents().is_empty(), "RT-10c: fresh bucket should be empty on node {i}");
|
||||
}
|
||||
|
||||
info!("RT-10c PASS: bucket visible from all 4 nodes");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -574,8 +574,7 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
|
||||
(&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
|
||||
(&Method::POST, true) if upload_id.is_some() => Operation::CompleteMultipartUpload,
|
||||
(&Method::DELETE, true) if upload_id.is_some() => Operation::AbortMultipartUpload,
|
||||
// A replication PUT addresses the source version via `?versionId=`.
|
||||
(&Method::PUT, true) if only_query_keys(&["versionId"]) => Operation::PutObject,
|
||||
(&Method::PUT, true) if only_query_keys(&[]) => Operation::PutObject,
|
||||
(&Method::GET, true) if only_query_keys(&["versionId"]) => Operation::GetObject,
|
||||
(&Method::HEAD, true) if only_query_keys(&["versionId"]) => Operation::HeadObject,
|
||||
(&Method::DELETE, true) if only_query_keys(&["versionId"]) => Operation::DeleteObject,
|
||||
|
||||
@@ -1687,44 +1687,6 @@ async fn four_node_inline_storage_and_get_boundaries() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_empty_legacy_volumes_start_as_fresh() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
for data_dir in cluster.nodes.iter().flat_map(|node| &node.data_dirs) {
|
||||
tokio::fs::create_dir_all(Path::new(data_dir).join(".minio.sys")).await?;
|
||||
}
|
||||
|
||||
cluster.start().await?;
|
||||
|
||||
// Starting is not the assertion. The regression is that an empty legacy
|
||||
// `.minio.sys` must be classified as a *fresh* volume, not as an existing
|
||||
// MinIO deployment to adopt or migrate. Pin what that classification leaves
|
||||
// on disk and in the namespace.
|
||||
let buckets = cluster.create_s3_client(0)?.list_buckets().send().await?;
|
||||
assert!(
|
||||
buckets.buckets().is_empty(),
|
||||
"a fresh classification must not adopt buckets from the pre-existing directories, got {:?}",
|
||||
buckets.buckets().iter().filter_map(|b| b.name()).collect::<Vec<_>>()
|
||||
);
|
||||
|
||||
for data_dir in cluster.nodes.iter().flat_map(|node| &node.data_dirs) {
|
||||
assert!(
|
||||
Path::new(data_dir).join(".rustfs.sys").join("format.json").is_file(),
|
||||
"each drive must be formatted as fresh: {data_dir} has no .rustfs.sys/format.json"
|
||||
);
|
||||
let mut legacy = tokio::fs::read_dir(Path::new(data_dir).join(".minio.sys")).await?;
|
||||
assert!(
|
||||
legacy.next_entry().await?.is_none(),
|
||||
"the empty legacy directory must be left untouched, not migrated into: {data_dir}"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn four_node_inline_fallback_controls() -> TestResult {
|
||||
@@ -2211,6 +2173,11 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
||||
"queue_snapshot.{field} must be readable in terminal status: {terminal}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
cold_tier_object_count(&cold_client).await? < 64,
|
||||
"queue pressure should leave at least one object untransitioned"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -21,12 +21,9 @@
|
||||
|
||||
use super::common::LocalKMSTestEnvironment;
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ServerSideEncryption,
|
||||
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
};
|
||||
use rustfs_rio::{Checksum, ChecksumType};
|
||||
use serial_test::serial;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
@@ -276,7 +273,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err
|
||||
/// Test 3: When bucket is configured with default encryption, create_multipart_upload should inherit the configuration
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
async fn test_bucket_default_encryption_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("Testing bucket default encryption impact on create_multipart_upload");
|
||||
|
||||
@@ -312,16 +309,15 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
.await
|
||||
.expect("Failed to set bucket encryption");
|
||||
|
||||
// Step 2: Declare CRC32 without specifying encryption parameters. The AWS SDK
|
||||
// calculates each UploadPart checksum and sends it as a flexible checksum.
|
||||
info!("Creating CRC32 multipart upload that should use bucket default encryption");
|
||||
let test_key = "test-multipart-bucket-default-crc32.bin";
|
||||
// Step 2: Create multipart upload (without specifying encryption parameters)
|
||||
info!("Creating multipart upload (without specifying encryption parameters, should use bucket default configuration)");
|
||||
let test_key = "test-multipart-bucket-default.txt";
|
||||
|
||||
let create_multipart_response = s3_client
|
||||
.create_multipart_upload()
|
||||
.bucket(TEST_BUCKET)
|
||||
.key(test_key)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
// Note: No encryption parameters specified here, should use bucket default configuration
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to create multipart upload");
|
||||
@@ -347,61 +343,28 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
"create_multipart_upload response should contain correct KMS key ID"
|
||||
);
|
||||
|
||||
// Step 3: Upload two parts. The first is exactly the S3 minimum size so this
|
||||
// follows the same managed SSE-KMS multipart path as issue #5756.
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
let part1: Vec<u8> = (0..PART_SIZE).map(|i| (i % 251) as u8).collect();
|
||||
let part2: Vec<u8> = (0..1024 * 1024).map(|i| ((i + 17) % 251) as u8).collect();
|
||||
let expected_body: Vec<u8> = part1.iter().chain(&part2).copied().collect();
|
||||
// Step 3: Upload a part and complete multipart upload
|
||||
info!("Uploading part and completing multipart upload");
|
||||
let test_data = b"test-multipart-bucket-default-encryption-data";
|
||||
|
||||
let upload_part = |part_number: i32, body: Vec<u8>| {
|
||||
s3_client
|
||||
.upload_part()
|
||||
.bucket(TEST_BUCKET)
|
||||
.key(test_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(part_number)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
.body(ByteStream::from(body))
|
||||
.send()
|
||||
};
|
||||
// Upload part 1
|
||||
let upload_part_response = s3_client
|
||||
.upload_part()
|
||||
.bucket(TEST_BUCKET)
|
||||
.key(test_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(test_data.to_vec().into())
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to upload part");
|
||||
|
||||
let expected_part1_crc32 = Checksum::new_from_data(ChecksumType::CRC32, &part1)
|
||||
.expect("calculate part 1 CRC32")
|
||||
.encoded;
|
||||
let upload1 = upload_part(1, part1).await.expect("Failed to upload part 1 with CRC32");
|
||||
assert_eq!(
|
||||
upload1.checksum_crc32(),
|
||||
Some(expected_part1_crc32.as_str()),
|
||||
"UploadPart must return the CRC32 calculated over plaintext"
|
||||
);
|
||||
|
||||
let expected_part2_crc32 = Checksum::new_from_data(ChecksumType::CRC32, &part2)
|
||||
.expect("calculate part 2 CRC32")
|
||||
.encoded;
|
||||
let upload2 = upload_part(2, part2).await.expect("Failed to upload part 2 with CRC32");
|
||||
assert_eq!(
|
||||
upload2.checksum_crc32(),
|
||||
Some(expected_part2_crc32.as_str()),
|
||||
"UploadPart must return the CRC32 calculated over plaintext"
|
||||
);
|
||||
let etag = upload_part_response.e_tag().unwrap().to_string();
|
||||
|
||||
// Complete multipart upload
|
||||
let completed_upload = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(upload1.e_tag().expect("No ETag for part 1"))
|
||||
.checksum_crc32(upload1.checksum_crc32().expect("No CRC32 for part 1"))
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(upload2.e_tag().expect("No ETag for part 2"))
|
||||
.checksum_crc32(upload2.checksum_crc32().expect("No CRC32 for part 2"))
|
||||
.build(),
|
||||
)
|
||||
let completed_part = aws_sdk_s3::types::CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(&etag)
|
||||
.build();
|
||||
|
||||
let complete_multipart_response = s3_client
|
||||
@@ -409,7 +372,11 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
.bucket(TEST_BUCKET)
|
||||
.key(test_key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(completed_upload)
|
||||
.multipart_upload(
|
||||
aws_sdk_s3::types::CompletedMultipartUpload::builder()
|
||||
.parts(completed_part)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to complete multipart upload");
|
||||
@@ -433,7 +400,6 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
.get_object()
|
||||
.bucket(TEST_BUCKET)
|
||||
.key(test_key)
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to get object");
|
||||
@@ -444,13 +410,6 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
Some(&ServerSideEncryption::AwsKms),
|
||||
"Final object should contain SSE-KMS encryption information"
|
||||
);
|
||||
if let Some(completed_crc32) = complete_multipart_response.checksum_crc32() {
|
||||
assert_eq!(
|
||||
get_response.checksum_crc32(),
|
||||
Some(completed_crc32),
|
||||
"GetObject should return the persisted composite CRC32 when completion reports it"
|
||||
);
|
||||
}
|
||||
|
||||
// Verify data integrity
|
||||
let downloaded_data = get_response
|
||||
@@ -459,11 +418,7 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
.await
|
||||
.expect("Failed to collect body")
|
||||
.into_bytes();
|
||||
assert_eq!(
|
||||
downloaded_data.as_ref(),
|
||||
expected_body.as_slice(),
|
||||
"Downloaded data should match the uploaded multipart body"
|
||||
);
|
||||
assert_eq!(&downloaded_data[..], test_data, "Downloaded data should match original data");
|
||||
|
||||
// Cleanup is handled automatically when the test environment is dropped
|
||||
info!("Test passed: bucket default encryption correctly applied to multipart upload");
|
||||
|
||||
@@ -1,351 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression test: a same-key CopyObject that only rewrites metadata must never re-key a
|
||||
//! managed-SSE (SSE-S3 / SSE-KMS) object.
|
||||
//!
|
||||
//! On an **unversioned** bucket the handler marks a same-name copy `metadata_only`, and the
|
||||
//! store layer then updates `xl.meta` in place without touching the data blocks. The handler
|
||||
//! nevertheless strips the source encryption metadata and generates a *fresh* DEK for the
|
||||
//! destination. Combining the two writes "new DEK + old ciphertext": the object is permanently
|
||||
//! undecryptable. The fix forces a full data rewrite whenever the copy re-derives managed
|
||||
//! encryption material, so the stored bytes always match the key metadata beside them.
|
||||
//!
|
||||
//! Companion to `copy_object_version_restore_sse_test` (issue #4238), which pins the same
|
||||
//! invariant for the versioned historical-restore path.
|
||||
|
||||
use super::common::{LocalKMSTestEnvironment, create_key_with_specific_id};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
MetadataDirective, ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration,
|
||||
ServerSideEncryptionRule,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use tracing::info;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
|
||||
init_logging();
|
||||
info!("same-key CopyObject with REPLACE metadata must not re-key an SSE-S3 object");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await.expect("failed to create local KMS env");
|
||||
let default_key_id = "rustfs-e2e-test-default-key";
|
||||
let keys_dir = kms_env.kms_keys_dir.clone();
|
||||
create_key_with_specific_id(&keys_dir, default_key_id)
|
||||
.await
|
||||
.expect("failed to create local KMS key");
|
||||
kms_env
|
||||
.base_env
|
||||
.start_rustfs_server_with_env(
|
||||
vec![
|
||||
"--kms-enable",
|
||||
"--kms-backend",
|
||||
"local",
|
||||
"--kms-key-dir",
|
||||
&keys_dir,
|
||||
"--kms-default-key-id",
|
||||
default_key_id,
|
||||
],
|
||||
&[("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")],
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
// Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service
|
||||
// the self-copy as a pure metadata update.
|
||||
let bucket = "copy-object-self-copy-sse-test";
|
||||
let key = "secrets/report.txt";
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("failed to create bucket");
|
||||
|
||||
// Content long enough that a truncated/garbled decrypt cannot coincidentally match.
|
||||
let content = b"encrypted payload that must survive a metadata-only self copy -- 0123456789";
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.content_type("text/plain; charset=utf-8")
|
||||
.metadata("stage", "before")
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.body(ByteStream::from_static(content))
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT failed");
|
||||
assert_eq!(put.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
|
||||
|
||||
// Copy the object onto itself, replacing user metadata. This is the `mc cp --attr` /
|
||||
// "edit metadata in place" shape that AWS supports on an existing object.
|
||||
let copy_out = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.metadata_directive(MetadataDirective::Replace)
|
||||
.content_type("text/plain; charset=utf-8")
|
||||
.metadata("stage", "after")
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await
|
||||
.expect("same-key CopyObject with REPLACE metadata must succeed");
|
||||
assert_eq!(copy_out.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
|
||||
|
||||
// The object must still decrypt to the original plaintext. Before the fix the stored
|
||||
// ciphertext was left untouched while the metadata carried a brand-new DEK, so this GET
|
||||
// either failed outright or returned garbage.
|
||||
let get = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect("GET after self-copy failed: the object was re-keyed without rewriting the ciphertext");
|
||||
assert_eq!(get.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
|
||||
assert_eq!(
|
||||
get.metadata().and_then(|m| m.get("stage")),
|
||||
Some(&"after".to_string()),
|
||||
"REPLACE metadata must take effect"
|
||||
);
|
||||
let body = get.body.collect().await.expect("collect body").into_bytes();
|
||||
assert_eq!(
|
||||
body.as_ref(),
|
||||
content,
|
||||
"object must still decrypt to the original plaintext after a metadata-only self copy"
|
||||
);
|
||||
|
||||
kms_env.base_env.stop_server();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
|
||||
init_logging();
|
||||
info!("same-key CopyObject that drops SSE must rewrite the data, not orphan the ciphertext");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await.expect("failed to create local KMS env");
|
||||
let default_key_id = "rustfs-e2e-test-default-key";
|
||||
let keys_dir = kms_env.kms_keys_dir.clone();
|
||||
create_key_with_specific_id(&keys_dir, default_key_id)
|
||||
.await
|
||||
.expect("failed to create local KMS key");
|
||||
kms_env
|
||||
.base_env
|
||||
.start_rustfs_server_with_env(
|
||||
vec![
|
||||
"--kms-enable",
|
||||
"--kms-backend",
|
||||
"local",
|
||||
"--kms-key-dir",
|
||||
&keys_dir,
|
||||
"--kms-default-key-id",
|
||||
default_key_id,
|
||||
],
|
||||
&[("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")],
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
// Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below
|
||||
// resolves to "no destination encryption".
|
||||
let bucket = "copy-object-self-copy-drop-sse-test";
|
||||
let key = "secrets/report.txt";
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("failed to create bucket");
|
||||
|
||||
let content = b"encrypted payload whose ciphertext must not survive as bogus plaintext -- 0123456789";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.metadata("stage", "before")
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.body(ByteStream::from_static(content))
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT failed");
|
||||
|
||||
// Self-copy with REPLACE and no SSE header. Per AWS semantics the destination ends up
|
||||
// unencrypted. The dangerous outcome is the silent one: the handler strips the source key
|
||||
// metadata while a metadata-only copy leaves the ciphertext in place, so a later GET would
|
||||
// hand back raw ciphertext as if it were plaintext — corruption with no error anywhere.
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.metadata_directive(MetadataDirective::Replace)
|
||||
.metadata("stage", "after")
|
||||
.send()
|
||||
.await
|
||||
.expect("same-key CopyObject dropping SSE must succeed");
|
||||
|
||||
let get = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect("GET after self-copy failed");
|
||||
assert_eq!(
|
||||
get.server_side_encryption(),
|
||||
None,
|
||||
"destination must be unencrypted once the copy drops SSE"
|
||||
);
|
||||
assert_eq!(
|
||||
get.metadata().and_then(|m| m.get("stage")),
|
||||
Some(&"after".to_string()),
|
||||
"REPLACE metadata must take effect"
|
||||
);
|
||||
let body = get.body.collect().await.expect("collect body").into_bytes();
|
||||
assert_eq!(
|
||||
body.as_ref(),
|
||||
content,
|
||||
"object must read back as the original plaintext, not the orphaned ciphertext"
|
||||
);
|
||||
|
||||
kms_env.base_env.stop_server();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decryptable() {
|
||||
init_logging();
|
||||
info!("bucket default encryption must also keep a same-key copy off the metadata-only path");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await.expect("failed to create local KMS env");
|
||||
let default_key_id = "rustfs-e2e-test-default-key";
|
||||
let keys_dir = kms_env.kms_keys_dir.clone();
|
||||
create_key_with_specific_id(&keys_dir, default_key_id)
|
||||
.await
|
||||
.expect("failed to create local KMS key");
|
||||
kms_env
|
||||
.base_env
|
||||
.start_rustfs_server_with_env(
|
||||
vec![
|
||||
"--kms-enable",
|
||||
"--kms-backend",
|
||||
"local",
|
||||
"--kms-key-dir",
|
||||
&keys_dir,
|
||||
"--kms-default-key-id",
|
||||
default_key_id,
|
||||
],
|
||||
&[("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")],
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
let bucket = "copy-object-self-copy-bucket-default-sse-test";
|
||||
let key = "secrets/report.txt";
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("failed to create bucket");
|
||||
|
||||
// Store the object as PLAINTEXT first: no SSE header and no bucket default rule yet. This is
|
||||
// what makes the case sharp — at copy time the source metadata carries no encryption markers,
|
||||
// so the source-side half of the guard cannot fire.
|
||||
let content = b"plaintext payload that must not be orphaned under a new DEK -- 0123456789";
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.metadata("stage", "before")
|
||||
.body(ByteStream::from_static(content))
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT failed");
|
||||
assert_eq!(put.server_side_encryption(), None, "the object must start out unencrypted");
|
||||
|
||||
// Only NOW enable bucket default encryption. The destination's encryption therefore comes
|
||||
// from the bucket rule and from nowhere else: the source is unencrypted and the copy request
|
||||
// carries no SSE header. A guard that only inspects request headers (MinIO decides
|
||||
// `isTargetEncrypted` from `crypto.S3.IsRequested(r.Header)`) would let this through, yet
|
||||
// `sse_encryption` still mints a fresh DEK from the resolved bucket default — which is why
|
||||
// the guard keys off the *effective* encryption rather than the requested one.
|
||||
let encryption_config = ServerSideEncryptionConfiguration::builder()
|
||||
.rules(
|
||||
ServerSideEncryptionRule::builder()
|
||||
.apply_server_side_encryption_by_default(
|
||||
ServerSideEncryptionByDefault::builder()
|
||||
.sse_algorithm(ServerSideEncryption::Aes256)
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build()
|
||||
.unwrap();
|
||||
client
|
||||
.put_bucket_encryption()
|
||||
.bucket(bucket)
|
||||
.server_side_encryption_configuration(encryption_config)
|
||||
.send()
|
||||
.await
|
||||
.expect("failed to set bucket default encryption");
|
||||
|
||||
// No SSE header on the copy — the bucket default alone drives the destination encryption.
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.metadata_directive(MetadataDirective::Replace)
|
||||
.metadata("stage", "after")
|
||||
.send()
|
||||
.await
|
||||
.expect("same-key CopyObject under bucket default encryption must succeed");
|
||||
|
||||
let get = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect("GET after self-copy failed: the object was re-keyed without rewriting the ciphertext");
|
||||
assert_eq!(get.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
|
||||
assert_eq!(
|
||||
get.metadata().and_then(|m| m.get("stage")),
|
||||
Some(&"after".to_string()),
|
||||
"REPLACE metadata must take effect"
|
||||
);
|
||||
let body = get.body.collect().await.expect("collect body").into_bytes();
|
||||
assert_eq!(
|
||||
body.as_ref(),
|
||||
content,
|
||||
"object must still decrypt to the original plaintext after a metadata-only self copy"
|
||||
);
|
||||
|
||||
kms_env.base_env.stop_server();
|
||||
}
|
||||
@@ -48,9 +48,6 @@ mod bucket_default_encryption_test;
|
||||
#[cfg(test)]
|
||||
mod encryption_metadata_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod copy_object_self_copy_sse_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod copy_object_version_restore_sse_test;
|
||||
|
||||
|
||||
@@ -290,14 +290,6 @@ mod overwrite_cleanup_regression_test;
|
||||
#[cfg(test)]
|
||||
mod list_buckets_double_slash_test;
|
||||
|
||||
// Regression coverage for bucket-scoped ListBuckets authorization fallback.
|
||||
#[cfg(test)]
|
||||
mod list_buckets_auth_test;
|
||||
|
||||
// ListBuckets visibility follows IAM authorization, not bucket policy.
|
||||
#[cfg(test)]
|
||||
mod list_buckets_iam_filter_test;
|
||||
|
||||
// Regression test for backlog#629(b): region-aware CreateBucket SigV4.
|
||||
#[cfg(test)]
|
||||
mod create_bucket_region_test;
|
||||
@@ -306,32 +298,4 @@ mod create_bucket_region_test;
|
||||
#[cfg(test)]
|
||||
mod copy_source_invalid_date_test;
|
||||
|
||||
// P0 regression: event notification startup race (rustfs#5387, #5681, #5401, #5183, #5115, #4796)
|
||||
#[cfg(test)]
|
||||
mod notification_startup_regression_test;
|
||||
|
||||
// P0 regression: lifecycle/ILM object expiration (rustfs#5407, #5167, #4963, #5615, #4879)
|
||||
#[cfg(test)]
|
||||
mod lifecycle_regression_test;
|
||||
|
||||
// P0 regression: delete operations consistency (rustfs#5375, #5349, #5339, #5029, #4978, #760)
|
||||
#[cfg(test)]
|
||||
mod delete_regression_test;
|
||||
|
||||
// P1 regression: listing/metacache completeness (rustfs#5166, #5156, #5051, #4810, #4648, #3191)
|
||||
#[cfg(test)]
|
||||
mod listing_regression_test;
|
||||
|
||||
// P1 regression: bucket statistics accuracy (rustfs#5615, #5008, #5116, #5055, #3898, #1012)
|
||||
#[cfg(test)]
|
||||
mod bucket_stats_regression_test;
|
||||
|
||||
// P1 regression: distributed startup/quorum (rustfs#5416, #2945, #2794, #2601, #4040, #5655)
|
||||
#[cfg(test)]
|
||||
mod distributed_startup_regression_test;
|
||||
|
||||
// P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024)
|
||||
#[cfg(test)]
|
||||
mod tier_transition_regression_test;
|
||||
|
||||
pub mod tls_gen;
|
||||
|
||||
@@ -1,360 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression tests for lifecycle/ILM object expiration and transition.
|
||||
//!
|
||||
//! Covers the recurring pattern where ILM expiration rules do not actually
|
||||
//! delete objects, or lifecycle rule parameters are silently corrupted.
|
||||
//! This has regressed 6+ times.
|
||||
//!
|
||||
//! ## Regression Issues
|
||||
//!
|
||||
//! - rustfs#5407: lifecycle not delete any bucket object
|
||||
//! - rustfs#5167: lifecycle not delete object
|
||||
//! - rustfs#4963: lifecycle rule 3 days → effective value 0 days
|
||||
//! - rustfs#5615: bucket statistics remain unchanged after data expiration
|
||||
//! - rustfs#4879: ILM serial lane: restore transition never completes
|
||||
//! - rustfs#5442: Uncheck of Replicate Delete still deletes the file
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketLifecycleConfiguration, BucketVersioningStatus, ExpirationStatus, LifecycleExpiration, LifecycleRule,
|
||||
LifecycleRuleFilter, NoncurrentVersionExpiration, VersioningConfiguration,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
|
||||
async fn setup_versioned_bucket(client: &Client, bucket: &str) -> TestResult {
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("create bucket: {e}"))?;
|
||||
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("enable versioning: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-03: Verify that a lifecycle expiration rule actually deletes objects.
|
||||
///
|
||||
/// Regression pattern: lifecycle rules are accepted but the scanner never
|
||||
/// processes them, leaving expired objects in place.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. Create a versioned bucket
|
||||
/// 2. Upload several objects
|
||||
/// 3. Apply a lifecycle rule with 1-day expiration
|
||||
/// 4. Wait for the scanner to process
|
||||
/// 5. Verify objects are still present (they shouldn't expire yet — 1 day)
|
||||
/// 6. Verify the lifecycle rule was persisted correctly (not corrupted to 0 days)
|
||||
///
|
||||
/// This tests the rule persistence path (rustfs#4963: 3 days → 0 days).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_lifecycle_expiration_rule_persists_correctly() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-03: lifecycle expiration rule persists correctly");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt03-lifecycle-persist";
|
||||
setup_versioned_bucket(&client, bucket).await?;
|
||||
|
||||
// Apply a lifecycle rule with 1-day expiration on a prefix
|
||||
let rule = LifecycleRule::builder()
|
||||
.id("expire-after-1-day")
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.filter(LifecycleRuleFilter::builder().prefix("logs/").build())
|
||||
.expiration(LifecycleExpiration::builder().days(1).build())
|
||||
.build()
|
||||
.expect("build lifecycle rule");
|
||||
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.lifecycle_configuration(
|
||||
BucketLifecycleConfiguration::builder()
|
||||
.rules(rule)
|
||||
.build()
|
||||
.expect("build lifecycle config"),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("put lifecycle configuration");
|
||||
|
||||
// Read back and verify the rule was not corrupted (rustfs#4963: days → 0)
|
||||
let resp = client
|
||||
.get_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("get lifecycle configuration");
|
||||
|
||||
let rules = resp.rules();
|
||||
assert_eq!(rules.len(), 1, "RT-03 FAIL: expected exactly 1 lifecycle rule");
|
||||
|
||||
let retrieved = &rules[0];
|
||||
assert_eq!(retrieved.id(), Some("expire-after-1-day"), "RT-03 FAIL: rule ID mismatch");
|
||||
assert_eq!(retrieved.status(), &ExpirationStatus::Enabled, "RT-03 FAIL: rule should be Enabled");
|
||||
|
||||
let exp = retrieved.expiration().expect("expiration should be set");
|
||||
assert_eq!(
|
||||
exp.days(),
|
||||
Some(1),
|
||||
"RT-03 FAIL: expiration days corrupted (regression rustfs#4963: expected 1, got {:?})",
|
||||
exp.days()
|
||||
);
|
||||
|
||||
info!("RT-03 PASS: lifecycle expiration rule persists correctly");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-03b: Verify lifecycle rule with noncurrent version expiration.
|
||||
///
|
||||
/// Covers the pattern where noncurrent version expiration rules are
|
||||
/// accepted but old versions are never cleaned up.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_lifecycle_noncurrent_version_expiration_rule_persists() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-03b: noncurrent version expiration rule persists");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt03b-noncurrent-expire";
|
||||
setup_versioned_bucket(&client, bucket).await?;
|
||||
|
||||
// Create multiple versions of the same object
|
||||
for i in 0..3 {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("versioned-obj.txt")
|
||||
.body(ByteStream::from(format!("version-{i}").into_bytes()))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object version");
|
||||
}
|
||||
|
||||
// Verify we have 3 versions
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions");
|
||||
|
||||
let count = versions.versions().len();
|
||||
assert_eq!(count, 3, "RT-03b FAIL: expected 3 versions, found {count}");
|
||||
|
||||
// Apply noncurrent version expiration rule
|
||||
let rule = LifecycleRule::builder()
|
||||
.id("expire-noncurrent-after-1-day")
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.filter(LifecycleRuleFilter::builder().prefix("").build())
|
||||
.noncurrent_version_expiration(NoncurrentVersionExpiration::builder().noncurrent_days(1).build())
|
||||
.build()
|
||||
.expect("build lifecycle rule");
|
||||
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.lifecycle_configuration(
|
||||
BucketLifecycleConfiguration::builder()
|
||||
.rules(rule)
|
||||
.build()
|
||||
.expect("build lifecycle config"),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("put lifecycle configuration");
|
||||
|
||||
// Read back and verify
|
||||
let resp = client
|
||||
.get_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("get lifecycle configuration");
|
||||
|
||||
let rules = resp.rules();
|
||||
assert_eq!(rules.len(), 1, "RT-03b FAIL: expected 1 rule");
|
||||
|
||||
let nc_exp = rules[0]
|
||||
.noncurrent_version_expiration()
|
||||
.expect("noncurrent expiration should be set");
|
||||
assert_eq!(nc_exp.noncurrent_days(), Some(1), "RT-03b FAIL: noncurrent days corrupted");
|
||||
|
||||
info!("RT-03b PASS: noncurrent version expiration rule persists correctly");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-04: Verify lifecycle rule with prefix filter persists after restart.
|
||||
///
|
||||
/// Covers the pattern where lifecycle rules are accepted but silently lost
|
||||
/// after restart. Transition rules require a configured remote tier
|
||||
/// (tested in reliant/tiering.rs), so this test uses expiration only.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_lifecycle_prefix_rule_persists() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-04: lifecycle prefix rule persists");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt04-lifecycle-prefix";
|
||||
setup_versioned_bucket(&client, bucket).await?;
|
||||
|
||||
let rule = LifecycleRule::builder()
|
||||
.id("expire-archive-after-7-days")
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.filter(LifecycleRuleFilter::builder().prefix("archive/").build())
|
||||
.expiration(LifecycleExpiration::builder().days(7).build())
|
||||
.build()
|
||||
.expect("build lifecycle rule");
|
||||
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.lifecycle_configuration(
|
||||
BucketLifecycleConfiguration::builder()
|
||||
.rules(rule)
|
||||
.build()
|
||||
.expect("build lifecycle config"),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("put lifecycle configuration");
|
||||
|
||||
// Restart server
|
||||
env.restart_server_preserving_data(vec![], &[]).await.expect("restart RustFS");
|
||||
|
||||
// Verify the rule survived restart
|
||||
let resp = client
|
||||
.get_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("get lifecycle after restart");
|
||||
|
||||
let rules = resp.rules();
|
||||
assert_eq!(rules.len(), 1, "RT-04 FAIL: expected 1 rule after restart");
|
||||
|
||||
let exp = rules[0].expiration().expect("expiration should be set");
|
||||
assert_eq!(exp.days(), Some(7), "RT-04 FAIL: expiration days corrupted after restart");
|
||||
|
||||
info!("RT-04 PASS: lifecycle prefix rule persists after restart");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-05b: Verify delete marker creation in versioned bucket.
|
||||
///
|
||||
/// Regression pattern: DELETE on a versioned object fails or does not
|
||||
/// create a delete marker, or the delete marker is not visible in LIST.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delete_marker_creation_and_visibility() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-05b: delete marker creation and visibility");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt05b-delete-marker";
|
||||
setup_versioned_bucket(&client, bucket).await?;
|
||||
|
||||
// Put an object
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("marker-test.txt")
|
||||
.body(ByteStream::from_static(b"to-be-deleted"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object");
|
||||
|
||||
// Delete without specifying versionId → should create a delete marker
|
||||
let del_resp = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key("marker-test.txt")
|
||||
.send()
|
||||
.await
|
||||
.expect("delete object");
|
||||
|
||||
// The response should indicate a delete marker was created
|
||||
assert!(
|
||||
del_resp.delete_marker().unwrap_or(false),
|
||||
"RT-05b FAIL: DELETE on versioned object did not create a delete marker"
|
||||
);
|
||||
|
||||
// ListObjectVersions should show both the original version and the delete marker
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions");
|
||||
|
||||
let delete_markers: Vec<_> = versions
|
||||
.delete_markers()
|
||||
.iter()
|
||||
.filter(|dm| dm.key() == Some("marker-test.txt"))
|
||||
.collect();
|
||||
|
||||
assert_eq!(
|
||||
delete_markers.len(),
|
||||
1,
|
||||
"RT-05b FAIL: expected 1 delete marker, found {}",
|
||||
delete_markers.len()
|
||||
);
|
||||
|
||||
info!("RT-05b PASS: delete marker created and visible");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression coverage for the MinIO-compatible filtered ListBuckets fallback.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
|
||||
use std::error::Error;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_scoped_policy_returns_only_authorized_bucket() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let root_client = env.create_s3_client();
|
||||
let allowed_bucket = "list-buckets-authorized";
|
||||
let hidden_bucket = "list-buckets-hidden";
|
||||
let user = "listbucketsuser";
|
||||
let secret = "listbucketssecret";
|
||||
let policy = "list-buckets-scoped";
|
||||
|
||||
root_client.create_bucket().bucket(allowed_bucket).send().await?;
|
||||
root_client.create_bucket().bucket(hidden_bucket).send().await?;
|
||||
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={policy}"),
|
||||
Some(
|
||||
serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{allowed_bucket}"),
|
||||
format!("arn:aws:s3:::{allowed_bucket}/*")
|
||||
]
|
||||
}]
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={user}"),
|
||||
Some(serde_json::json!({ "secretKey": secret, "status": "enabled" }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::POST,
|
||||
"/rustfs/admin/v3/idp/builtin/policy/attach",
|
||||
Some(serde_json::json!({ "policies": [policy], "user": user }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let client = env.create_s3_client_with_credentials(user, secret);
|
||||
// Capture ListBuckets first so the direct-access control cannot warm bucket metadata and mask the regression.
|
||||
let listed = client.list_buckets().send().await;
|
||||
client.list_objects_v2().bucket(allowed_bucket).send().await?;
|
||||
|
||||
let listed = listed?;
|
||||
let names = listed
|
||||
.buckets()
|
||||
.iter()
|
||||
.filter_map(|bucket| bucket.name().map(ToOwned::to_owned))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(names, vec![allowed_bucket]);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,459 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, build_test_s3_config, build_test_sts_client, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use serial_test::serial;
|
||||
use tokio::time::{Duration, Instant};
|
||||
|
||||
fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
|
||||
Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
access_key,
|
||||
secret_key,
|
||||
session_token,
|
||||
"list-buckets-iam-filter",
|
||||
))
|
||||
}
|
||||
|
||||
fn bucket_names(buckets: &[aws_sdk_s3::types::Bucket]) -> Vec<String> {
|
||||
let mut names = buckets
|
||||
.iter()
|
||||
.filter_map(|bucket| bucket.name().map(str::to_owned))
|
||||
.collect::<Vec<_>>();
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
async fn create_user(
|
||||
env: &RustFSTestEnvironment,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let body = serde_json::json!({ "secretKey": secret_key, "status": "enabled" }).to_string();
|
||||
admin_ok(
|
||||
env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={access_key}"),
|
||||
Some(body),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn create_service_account(
|
||||
env: &RustFSTestEnvironment,
|
||||
target_user: &str,
|
||||
policy: Option<&serde_json::Value>,
|
||||
) -> Result<(String, String), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let request = match policy {
|
||||
Some(policy) => serde_json::json!({ "targetUser": target_user, "policy": policy }),
|
||||
None => serde_json::json!({ "targetUser": target_user }),
|
||||
};
|
||||
let response = admin_ok(env, http::Method::PUT, "/rustfs/admin/v3/add-service-accounts", Some(request.to_string())).await?;
|
||||
let response: serde_json::Value = serde_json::from_str(&response)?;
|
||||
let access_key = response["credentials"]["accessKey"]
|
||||
.as_str()
|
||||
.ok_or("service account response should contain credentials.accessKey")?
|
||||
.to_owned();
|
||||
let secret_key = response["credentials"]["secretKey"]
|
||||
.as_str()
|
||||
.ok_or("service account response should contain credentials.secretKey")?
|
||||
.to_owned();
|
||||
Ok((access_key, secret_key))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn list_buckets_filters_with_iam_bucket_resources() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.capture_log_path = Some(format!("{}/server.log", env.temp_dir));
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUST_LOG", "rustfs=debug,rustfs_notify=debug")])
|
||||
.await?;
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
for bucket in [
|
||||
"benchmark-artifacts",
|
||||
"benchmark-denied",
|
||||
"benchmark-location-only",
|
||||
"benchmark-test1",
|
||||
"testuser1-artifacts",
|
||||
] {
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
}
|
||||
assert_eq!(
|
||||
bucket_names(admin_client.list_buckets().send().await?.buckets()),
|
||||
vec![
|
||||
"benchmark-artifacts",
|
||||
"benchmark-denied",
|
||||
"benchmark-location-only",
|
||||
"benchmark-test1",
|
||||
"testuser1-artifacts"
|
||||
]
|
||||
);
|
||||
|
||||
let access_key = "benchmark";
|
||||
let secret_key = "benchmark-secret-1234567890";
|
||||
create_user(&env, access_key, secret_key).await?;
|
||||
|
||||
let policy_name = "benchmark-bucket-prefix";
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-*", "arn:aws:s3:::benchmark-*/*"],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:prefix": [""],
|
||||
"s3:delimiter": ["/"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-denied"]
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-location-only"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["sts:AssumeRole"],
|
||||
"Resource": ["arn:aws:s3:::*"]
|
||||
}
|
||||
]
|
||||
})
|
||||
.to_string();
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
|
||||
Some(policy),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={access_key}&isGroup=false"),
|
||||
Some(String::new()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let bucket_policy_allow = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [access_key] },
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::testuser1-artifacts"]
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_client
|
||||
.put_bucket_policy()
|
||||
.bucket("testuser1-artifacts")
|
||||
.policy(bucket_policy_allow)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let bucket_policy_deny = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Deny",
|
||||
"Principal": { "AWS": [access_key] },
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-artifacts"]
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_client
|
||||
.put_bucket_policy()
|
||||
.bucket("benchmark-artifacts")
|
||||
.policy(bucket_policy_deny)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let benchmark_client = user_client(&env, access_key, secret_key, None);
|
||||
benchmark_client
|
||||
.list_objects_v2()
|
||||
.bucket("testuser1-artifacts")
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
bucket_names(benchmark_client.list_buckets().send().await?.buckets()),
|
||||
vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
|
||||
);
|
||||
let log_path = env.capture_log_path.as_deref().expect("server log path should be configured");
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
let audit_log = loop {
|
||||
let audit_log = tokio::fs::read_to_string(log_path).await?;
|
||||
if [
|
||||
"iam_implicit_deny",
|
||||
"s3_authorization_denied",
|
||||
"ListAllMyBucketsAction",
|
||||
"benchmark",
|
||||
"DEBUG",
|
||||
]
|
||||
.iter()
|
||||
.all(|field| audit_log.contains(field))
|
||||
|| Instant::now() >= deadline
|
||||
{
|
||||
break audit_log;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
};
|
||||
assert_eq!(audit_log.matches("iam_implicit_deny").count(), 1, "{audit_log}");
|
||||
for field in ["s3_authorization_denied", "ListAllMyBucketsAction", "benchmark", "DEBUG"] {
|
||||
assert!(audit_log.contains(field), "missing {field} in {audit_log}");
|
||||
}
|
||||
|
||||
let denied_access_key = "no-bucket-access";
|
||||
let denied_secret_key = "no-bucket-access-secret-1234567890";
|
||||
create_user(&env, denied_access_key, denied_secret_key).await?;
|
||||
let denied = user_client(&env, denied_access_key, denied_secret_key, None)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a user without IAM bucket permissions must be denied");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
|
||||
let put_only_policy_name = "put-only-no-bucket-discovery";
|
||||
let put_only_policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-*/*"]
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={put_only_policy_name}"),
|
||||
Some(put_only_policy),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!(
|
||||
"/rustfs/admin/v3/set-user-or-group-policy?policyName={put_only_policy_name}&userOrGroup={denied_access_key}&isGroup=false"
|
||||
),
|
||||
Some(String::new()),
|
||||
)
|
||||
.await?;
|
||||
let denied = user_client(&env, denied_access_key, denied_secret_key, None)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an unrelated IAM action must not reveal bucket names");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
|
||||
let list_all_policy_name = "list-all-buckets";
|
||||
let list_all_policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListAllMyBuckets"],
|
||||
"Resource": ["arn:aws:s3:::*"]
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={list_all_policy_name}"),
|
||||
Some(list_all_policy),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!(
|
||||
"/rustfs/admin/v3/set-user-or-group-policy?policyName={list_all_policy_name}&userOrGroup={denied_access_key}&isGroup=false"
|
||||
),
|
||||
Some(String::new()),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
bucket_names(
|
||||
user_client(&env, denied_access_key, denied_secret_key, None)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
),
|
||||
vec![
|
||||
"benchmark-artifacts",
|
||||
"benchmark-denied",
|
||||
"benchmark-location-only",
|
||||
"benchmark-test1",
|
||||
"testuser1-artifacts"
|
||||
]
|
||||
);
|
||||
|
||||
let group_user = "benchmark-group-user";
|
||||
let group_secret = "benchmark-group-secret-1234567890";
|
||||
let group_name = "benchmark-group";
|
||||
create_user(&env, group_user, group_secret).await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/update-group-members",
|
||||
Some(
|
||||
serde_json::json!({
|
||||
"group": group_name,
|
||||
"members": [group_user],
|
||||
"isRemove": false,
|
||||
"groupStatus": "enabled"
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
&env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
|
||||
Some(String::new()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
bucket_names(
|
||||
user_client(&env, group_user, group_secret, None)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
),
|
||||
vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
|
||||
);
|
||||
|
||||
let (service_access_key, service_secret_key) = create_service_account(&env, group_user, None).await?;
|
||||
assert_eq!(
|
||||
bucket_names(
|
||||
user_client(&env, &service_access_key, &service_secret_key, None)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
),
|
||||
vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
|
||||
);
|
||||
|
||||
let service_account_policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-test1"],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:prefix": [""],
|
||||
"s3:delimiter": ["/"]
|
||||
}
|
||||
}
|
||||
}]
|
||||
});
|
||||
let (restricted_service_access_key, restricted_service_secret_key) =
|
||||
create_service_account(&env, group_user, Some(&service_account_policy)).await?;
|
||||
assert_eq!(
|
||||
bucket_names(
|
||||
user_client(&env, &restricted_service_access_key, &restricted_service_secret_key, None,)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
),
|
||||
vec!["benchmark-test1"]
|
||||
);
|
||||
|
||||
let sts_client = build_test_sts_client(&env.url, group_user, group_secret, None, "list-buckets-iam-filter-sts");
|
||||
let inherited = sts_client
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/list-buckets")
|
||||
.role_session_name("list-buckets-iam-filter-inherited")
|
||||
.send()
|
||||
.await?;
|
||||
let inherited = inherited
|
||||
.credentials()
|
||||
.ok_or("AssumeRole response should contain inherited temporary credentials")?;
|
||||
assert_eq!(
|
||||
bucket_names(
|
||||
user_client(
|
||||
&env,
|
||||
inherited.access_key_id(),
|
||||
inherited.secret_access_key(),
|
||||
Some(inherited.session_token()),
|
||||
)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
),
|
||||
vec!["benchmark-artifacts", "benchmark-location-only", "benchmark-test1"]
|
||||
);
|
||||
|
||||
let session_policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::benchmark-test1"],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:prefix": [""],
|
||||
"s3:delimiter": ["/"]
|
||||
}
|
||||
}
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
let assumed = sts_client
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/list-buckets")
|
||||
.role_session_name("list-buckets-iam-filter")
|
||||
.policy(session_policy)
|
||||
.send()
|
||||
.await?;
|
||||
let temporary = assumed
|
||||
.credentials()
|
||||
.ok_or("AssumeRole response should contain temporary credentials")?;
|
||||
assert_eq!(
|
||||
bucket_names(
|
||||
user_client(
|
||||
&env,
|
||||
temporary.access_key_id(),
|
||||
temporary.secret_access_key(),
|
||||
Some(temporary.session_token()),
|
||||
)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
),
|
||||
vec!["benchmark-test1"]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,357 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression tests for object listing and metacache consistency.
|
||||
//!
|
||||
//! Covers the recurring pattern where ListObjectsV2 returns incomplete results,
|
||||
//! silently truncates with IsTruncated=false, or corrupts the metadata cache.
|
||||
//! This has regressed 8+ times.
|
||||
//!
|
||||
//! ## Regression Issues
|
||||
//!
|
||||
//! - rustfs#5166: Metacache listing quorum failed timeout after cluster startup
|
||||
//! - rustfs#5156: Metacache producer failed
|
||||
//! - rustfs#5051: ListObjectsV2 returns empty results for shallow prefixes
|
||||
//! - rustfs#4810: walk_dir timeout silently truncates listings (200, IsTruncated=false)
|
||||
//! - rustfs#4648: Object listing oscillates between complete, partial, and zero
|
||||
//! - rustfs#3191: ListObjectsV2 timeout corrupts metadata cache → NoSuchBucket
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use serial_test::serial;
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
|
||||
/// RT-06: Verify ListObjectsV2 pagination completeness for medium-sized bucket.
|
||||
///
|
||||
/// Regression pattern: listing returns 200 with IsTruncated=false but
|
||||
/// misses objects (rustfs#4810: walk_dir timeout truncation).
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. Upload 100 objects with known keys
|
||||
/// 2. List all objects via pagination (max_keys=10)
|
||||
/// 3. Verify all 100 keys are returned exactly once
|
||||
/// 4. Verify no duplicates or skipped keys
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_list_objects_v2_completeness_100_objects() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-06: listing completeness with 100 objects");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt06-list-completeness";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
// Upload 100 objects
|
||||
let expected_keys: Vec<String> = (0..100).map(|i| format!("obj-{i:04}.txt")).collect();
|
||||
for key in &expected_keys {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"data"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object");
|
||||
}
|
||||
|
||||
// Paginate through all objects (small page size to force multiple pages)
|
||||
let mut all_keys: Vec<String> = Vec::new();
|
||||
let mut continuation_token: Option<String> = None;
|
||||
|
||||
loop {
|
||||
let mut req = client.list_objects_v2().bucket(bucket).max_keys(10);
|
||||
|
||||
if let Some(ref token) = continuation_token {
|
||||
req = req.continuation_token(token);
|
||||
}
|
||||
|
||||
let resp = req.send().await.expect("list objects page");
|
||||
|
||||
for obj in resp.contents() {
|
||||
all_keys.push(obj.key().unwrap_or("").to_string());
|
||||
}
|
||||
|
||||
if !resp.is_truncated().unwrap_or(false) {
|
||||
break;
|
||||
}
|
||||
continuation_token = resp.next_continuation_token().map(|s| s.to_string());
|
||||
}
|
||||
|
||||
// Verify completeness and uniqueness
|
||||
let unique_keys: HashSet<&str> = all_keys.iter().map(|s| s.as_str()).collect();
|
||||
|
||||
assert_eq!(
|
||||
all_keys.len(),
|
||||
100,
|
||||
"RT-06 FAIL: expected 100 objects, listed {} (regression: walk_dir truncation)",
|
||||
all_keys.len()
|
||||
);
|
||||
assert_eq!(
|
||||
unique_keys.len(),
|
||||
100,
|
||||
"RT-06 FAIL: found {} unique keys but listed {} total (duplicates!)",
|
||||
unique_keys.len(),
|
||||
all_keys.len()
|
||||
);
|
||||
|
||||
for key in &expected_keys {
|
||||
assert!(
|
||||
unique_keys.contains(key.as_str()),
|
||||
"RT-06 FAIL: key '{key}' missing from listing (regression rustfs#4810)"
|
||||
);
|
||||
}
|
||||
|
||||
info!("RT-06 PASS: all 100 objects listed completely and uniquely");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-06b: Verify listing with prefix filter returns correct subset.
|
||||
///
|
||||
/// Regression pattern: prefix filter returns empty or includes wrong keys
|
||||
/// (rustfs#5051: empty results for shallow prefixes).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_list_objects_v2_prefix_filter_correctness() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-06b: prefix filter correctness");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt06b-prefix-filter";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
// Upload objects with different prefixes
|
||||
for i in 0..5 {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(format!("logs/app-{i:04}.log"))
|
||||
.body(ByteStream::from_static(b"log data"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put log object");
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(format!("data/file-{i:04}.csv"))
|
||||
.body(ByteStream::from_static(b"csv data"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put data object");
|
||||
}
|
||||
|
||||
// List with prefix "logs/" — should return exactly 5
|
||||
let resp = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.prefix("logs/")
|
||||
.send()
|
||||
.await
|
||||
.expect("list with prefix");
|
||||
|
||||
assert_eq!(
|
||||
resp.contents().len(),
|
||||
5,
|
||||
"RT-06b FAIL: expected 5 objects with prefix 'logs/', found {} (regression rustfs#5051)",
|
||||
resp.contents().len()
|
||||
);
|
||||
|
||||
for obj in resp.contents() {
|
||||
assert!(
|
||||
obj.key().unwrap_or("").starts_with("logs/"),
|
||||
"RT-06b FAIL: object '{}' does not match prefix 'logs/'",
|
||||
obj.key().unwrap_or("?")
|
||||
);
|
||||
}
|
||||
|
||||
// List with prefix "data/" — should return exactly 5
|
||||
let resp = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.prefix("data/")
|
||||
.send()
|
||||
.await
|
||||
.expect("list with data/ prefix");
|
||||
|
||||
assert_eq!(
|
||||
resp.contents().len(),
|
||||
5,
|
||||
"RT-06b FAIL: expected 5 objects with prefix 'data/', found {}",
|
||||
resp.contents().len()
|
||||
);
|
||||
|
||||
// List with prefix "nonexistent/" — should return 0
|
||||
let resp = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.prefix("nonexistent/")
|
||||
.send()
|
||||
.await
|
||||
.expect("list with nonexistent prefix");
|
||||
|
||||
assert!(
|
||||
resp.contents().is_empty(),
|
||||
"RT-06b FAIL: expected 0 objects with prefix 'nonexistent/', found {}",
|
||||
resp.contents().len()
|
||||
);
|
||||
|
||||
info!("RT-06b PASS: prefix filter returns correct subset");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-06c: Verify listing with delimiter and CommonPrefixes.
|
||||
///
|
||||
/// Regression pattern: delimiter handling produces incorrect CommonPrefixes
|
||||
/// or misses objects at the delimiter boundary.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_list_objects_v2_delimiter_common_prefixes() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-06c: delimiter and CommonPrefixes");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt06c-delimiter";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
// Create a hierarchical structure
|
||||
let keys = vec!["a.txt", "dir1/b.txt", "dir1/sub1/c.txt", "dir1/sub2/d.txt", "dir2/e.txt"];
|
||||
|
||||
for key in &keys {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(*key)
|
||||
.body(ByteStream::from_static(b"content"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object");
|
||||
}
|
||||
|
||||
// List with delimiter "/" at root level
|
||||
let resp = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.delimiter("/")
|
||||
.send()
|
||||
.await
|
||||
.expect("list with delimiter");
|
||||
|
||||
// Should have 1 object (a.txt) and 2 common prefixes (dir1/, dir2/)
|
||||
let contents: Vec<_> = resp.contents().iter().map(|o| o.key().unwrap_or("")).collect();
|
||||
let prefixes: Vec<_> = resp.common_prefixes().iter().map(|p| p.prefix().unwrap_or("")).collect();
|
||||
|
||||
assert!(contents.contains(&"a.txt"), "RT-06c FAIL: root object 'a.txt' missing from listing");
|
||||
assert_eq!(contents.len(), 1, "RT-06c FAIL: expected 1 root-level object, found {}", contents.len());
|
||||
assert_eq!(prefixes.len(), 2, "RT-06c FAIL: expected 2 common prefixes, found {:?}", prefixes);
|
||||
assert!(prefixes.contains(&"dir1/"), "RT-06c FAIL: 'dir1/' missing from CommonPrefixes");
|
||||
assert!(prefixes.contains(&"dir2/"), "RT-06c FAIL: 'dir2/' missing from CommonPrefixes");
|
||||
|
||||
info!("RT-06c PASS: delimiter and CommonPrefixes correct");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-06d: Verify listing returns correct IsTruncated flag.
|
||||
///
|
||||
/// Regression pattern: IsTruncated=false when there are more objects
|
||||
/// (rustfs#4810: walk_dir timeout truncation with false IsTruncated).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_list_objects_v2_is_truncated_correctness() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-06d: IsTruncated correctness");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt06d-truncated";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
// Upload 15 objects
|
||||
for i in 0..15 {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(format!("item-{i:04}.txt"))
|
||||
.body(ByteStream::from_static(b"data"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object");
|
||||
}
|
||||
|
||||
// List with max_keys=5 — should be truncated
|
||||
let resp = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.max_keys(5)
|
||||
.send()
|
||||
.await
|
||||
.expect("list with max_keys=5");
|
||||
|
||||
assert!(
|
||||
resp.is_truncated().unwrap_or(false),
|
||||
"RT-06d FAIL: IsTruncated should be true with 15 objects and max_keys=5"
|
||||
);
|
||||
assert_eq!(resp.contents().len(), 5, "RT-06d FAIL: expected 5 objects in first page");
|
||||
assert!(
|
||||
resp.next_continuation_token().is_some(),
|
||||
"RT-06d FAIL: NextContinuationToken should be present when truncated"
|
||||
);
|
||||
|
||||
// List with max_keys=100 — should NOT be truncated
|
||||
let resp = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.max_keys(100)
|
||||
.send()
|
||||
.await
|
||||
.expect("list with max_keys=100");
|
||||
|
||||
assert!(
|
||||
!resp.is_truncated().unwrap_or(false),
|
||||
"RT-06d FAIL: IsTruncated should be false with 15 objects and max_keys=100"
|
||||
);
|
||||
assert_eq!(resp.contents().len(), 15, "RT-06d FAIL: expected 15 objects with max_keys=100");
|
||||
|
||||
info!("RT-06d PASS: IsTruncated flag is correct");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -62,33 +62,6 @@ fn md5_hex(input: impl AsRef<[u8]>) -> String {
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
async fn create_restricted_user(
|
||||
env: &RustFSTestEnvironment,
|
||||
username: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/add-user?accessKey={username}", env.url);
|
||||
let body = serde_json::json!({
|
||||
"secretKey": secret_key,
|
||||
"status": "enabled"
|
||||
})
|
||||
.to_string();
|
||||
crate::common::awscurl_put(&url, &body, &env.access_key, &env.secret_key).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn restricted_user_client(env: &RustFSTestEnvironment, username: &str, secret_key: &str) -> aws_sdk_s3::Client {
|
||||
let credentials = aws_sdk_s3::config::Credentials::new(username, secret_key, None, None, "snowball-pax-auth-test");
|
||||
let config = aws_sdk_s3::Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(aws_sdk_s3::config::Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
aws_sdk_s3::Client::from_conf(config)
|
||||
}
|
||||
|
||||
/// Env var consumed by the local SSE-S3 DEK provider when KMS is not configured.
|
||||
///
|
||||
/// Since rustfs#3564 the server fails closed on managed SSE (SSE-S3 or
|
||||
@@ -3584,8 +3557,8 @@ async fn test_anonymous_post_object_rejects_expires_field_missing_from_policy_co
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_object_lock_retention_without_permission()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
async fn test_anonymous_post_object_accepts_object_lock_retention_fields() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
@@ -3594,6 +3567,8 @@ async fn test_anonymous_post_object_rejects_object_lock_retention_without_permis
|
||||
let bucket = "anon-post-policy-object-lock-retention";
|
||||
let object_key = "uploads/object-lock-retention.txt";
|
||||
let retain_until = "2037-10-21T07:28:00Z";
|
||||
let expected_body = b"post-policy-object-lock-retention-body".to_vec();
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client
|
||||
.create_bucket()
|
||||
@@ -3618,7 +3593,7 @@ async fn test_anonymous_post_object_rejects_object_lock_retention_without_permis
|
||||
.text("x-amz-object-lock-retain-until-date", retain_until)
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-object-lock-retention-body".to_vec())
|
||||
reqwest::multipart::Part::bytes(expected_body.clone())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
@@ -3632,8 +3607,26 @@ async fn test_anonymous_post_object_rejects_object_lock_retention_without_permis
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert_eq!(status, reqwest::StatusCode::NO_CONTENT);
|
||||
assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}");
|
||||
|
||||
let retention = admin_client
|
||||
.get_object_retention()
|
||||
.bucket(bucket)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
let retention = retention.retention().expect("retention should be present");
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until_out = retention
|
||||
.retain_until_date()
|
||||
.expect("retain_until_date should be present")
|
||||
.fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime)?;
|
||||
assert_eq!(retain_until_out, retain_until);
|
||||
|
||||
let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?;
|
||||
let uploaded = get_out.body.collect().await?.into_bytes();
|
||||
assert_eq!(uploaded.as_ref(), expected_body.as_slice());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -3822,8 +3815,8 @@ async fn test_anonymous_post_object_rejects_object_lock_retention_missing_from_p
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_object_lock_legal_hold_without_permission()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
async fn test_anonymous_post_object_accepts_object_lock_legal_hold_field() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
@@ -3831,6 +3824,8 @@ async fn test_anonymous_post_object_rejects_object_lock_legal_hold_without_permi
|
||||
|
||||
let bucket = "anon-post-policy-object-lock-legal-hold";
|
||||
let object_key = "uploads/object-lock-legal-hold.txt";
|
||||
let expected_body = b"post-policy-object-lock-legal-hold-body".to_vec();
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client
|
||||
.create_bucket()
|
||||
@@ -3853,7 +3848,7 @@ async fn test_anonymous_post_object_rejects_object_lock_legal_hold_without_permi
|
||||
.text("x-amz-object-lock-legal-hold", "ON")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-object-lock-legal-hold-body".to_vec())
|
||||
reqwest::multipart::Part::bytes(expected_body.clone())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
@@ -3867,8 +3862,26 @@ async fn test_anonymous_post_object_rejects_object_lock_legal_hold_without_permi
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert_eq!(status, reqwest::StatusCode::NO_CONTENT);
|
||||
assert!(response_body.is_empty(), "204 response should not contain a body, got: {response_body}");
|
||||
|
||||
let legal_hold = admin_client
|
||||
.get_object_legal_hold()
|
||||
.bucket(bucket)
|
||||
.key(object_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
legal_hold
|
||||
.legal_hold()
|
||||
.and_then(|value| value.status())
|
||||
.map(|value| value.as_str()),
|
||||
Some("ON")
|
||||
);
|
||||
|
||||
let get_out = admin_client.get_object().bucket(bucket).key(object_key).send().await?;
|
||||
let uploaded = get_out.body.collect().await?.into_bytes();
|
||||
assert_eq!(uploaded.as_ref(), expected_body.as_slice());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -5645,70 +5658,6 @@ async fn test_signed_put_object_extract_preserves_object_lock_retention() -> Res
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_pax_retention_overrides_request_retention()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "signed-extract-pax-retention-precedence";
|
||||
let archive_key = "retention.tar";
|
||||
let extracted_key = "alpha.txt";
|
||||
let request_retain_until = aws_sdk_s3::primitives::DateTime::from_secs(2_114_380_800);
|
||||
let pax_retain_until = "2040-01-01T00:00:00Z";
|
||||
|
||||
let client = env.create_s3_client();
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let pax = HashMap::from([
|
||||
("minio.metadata.x-amz-object-lock-mode", "COMPLIANCE".to_string()),
|
||||
("minio.metadata.x-amz-object-lock-retain-until-date", pax_retain_until.to_string()),
|
||||
]);
|
||||
let archive = make_tar_with_pax_entry(extracted_key, b"alpha-body", None, &pax).await;
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(archive_key)
|
||||
.object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Governance)
|
||||
.object_lock_retain_until_date(request_retain_until)
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let retention = client
|
||||
.get_object_retention()
|
||||
.bucket(bucket)
|
||||
.key(extracted_key)
|
||||
.send()
|
||||
.await?
|
||||
.retention()
|
||||
.expect("retention should be present")
|
||||
.clone();
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("COMPLIANCE"));
|
||||
assert_eq!(
|
||||
retention
|
||||
.retain_until_date()
|
||||
.expect("retain_until_date should be present")
|
||||
.fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime)?,
|
||||
pax_retain_until
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
@@ -5833,316 +5782,6 @@ async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retention_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
if !crate::common::awscurl_available() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "signed-extract-pax-auth";
|
||||
let put_only_user = "snowball-put-only";
|
||||
let put_only_secret = "snowball-put-only-secret";
|
||||
let conditional_user = "snowball-retention-condition";
|
||||
let conditional_secret = "snowball-retention-condition-secret";
|
||||
let wrong_action_user = "snowball-wrong-action";
|
||||
let wrong_action_secret = "snowball-wrong-action-secret";
|
||||
let version_condition_user = "snowball-version-condition";
|
||||
let version_condition_secret = "snowball-version-condition-secret";
|
||||
let pax_context_user = "snowball-pax-context";
|
||||
let pax_context_secret = "snowball-pax-context-secret";
|
||||
let conditional_version_id = Uuid::new_v4().to_string();
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
create_restricted_user(&env, put_only_user, put_only_secret).await?;
|
||||
create_restricted_user(&env, conditional_user, conditional_secret).await?;
|
||||
create_restricted_user(&env, wrong_action_user, wrong_action_secret).await?;
|
||||
create_restricted_user(&env, version_condition_user, version_condition_secret).await?;
|
||||
create_restricted_user(&env, pax_context_user, pax_context_secret).await?;
|
||||
|
||||
let object_resource = format!("arn:aws:s3:::{bucket}/*");
|
||||
let context_archive_resources = [
|
||||
format!("arn:aws:s3:::{bucket}/tag-context.tar"),
|
||||
format!("arn:aws:s3:::{bucket}/lock-context.tar"),
|
||||
];
|
||||
let tag_entry_resource = format!("arn:aws:s3:::{bucket}/tag-context-entry.txt");
|
||||
let lock_entry_resource = format!("arn:aws:s3:::{bucket}/lock-context-entry.txt");
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "PutOnly",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [put_only_user] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [object_resource.clone()]
|
||||
},
|
||||
{
|
||||
"Sid": "RetentionWithLimit",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [conditional_user] },
|
||||
"Action": ["s3:PutObject", "s3:PutObjectRetention"],
|
||||
"Resource": [object_resource.clone()]
|
||||
},
|
||||
{
|
||||
"Sid": "DenyRetentionBeyondCutoff",
|
||||
"Effect": "Deny",
|
||||
"Principal": { "AWS": [conditional_user] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [object_resource.clone()],
|
||||
"Condition": {
|
||||
"DateGreaterThan": {
|
||||
"s3:object-lock-retain-until-date": "2030-01-01T00:00:00Z"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Sid": "WrongAdditionalAction",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [wrong_action_user] },
|
||||
"Action": ["s3:PutObject", "s3:PutObjectLegalHold"],
|
||||
"Resource": [object_resource.clone()]
|
||||
},
|
||||
{
|
||||
"Sid": "VersionConditionPut",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [version_condition_user] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [object_resource.clone()]
|
||||
},
|
||||
{
|
||||
"Sid": "VersionConditionReplicate",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [version_condition_user] },
|
||||
"Action": ["s3:ReplicateObject"],
|
||||
"Resource": [object_resource],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:VersionId": conditional_version_id.clone()
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Sid": "PaxContextArchives",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectTagging"],
|
||||
"Resource": context_archive_resources
|
||||
},
|
||||
{
|
||||
"Sid": "PaxTagContextPut",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [tag_entry_resource.clone()],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:RequestObjectTag/classification": "public"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Sid": "PaxTagContextAction",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObjectTagging"],
|
||||
"Resource": [tag_entry_resource]
|
||||
},
|
||||
{
|
||||
"Sid": "PaxLockContextPut",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [lock_entry_resource.clone()],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:object-lock-mode": "COMPLIANCE"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Sid": "PaxLockContextAction",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObjectRetention"],
|
||||
"Resource": [lock_entry_resource]
|
||||
}
|
||||
]
|
||||
})
|
||||
.to_string();
|
||||
admin_client.put_bucket_policy().bucket(bucket).policy(policy).send().await?;
|
||||
|
||||
let put_only_client = restricted_user_client(&env, put_only_user, put_only_secret);
|
||||
let conditional_client = restricted_user_client(&env, conditional_user, conditional_secret);
|
||||
let wrong_action_client = restricted_user_client(&env, wrong_action_user, wrong_action_secret);
|
||||
let cases = [
|
||||
(
|
||||
"legal-hold.tar",
|
||||
put_only_client,
|
||||
HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]),
|
||||
),
|
||||
(
|
||||
"retention-condition.tar",
|
||||
conditional_client,
|
||||
HashMap::from([
|
||||
("minio.metadata.x-amz-object-lock-mode", "COMPLIANCE".to_string()),
|
||||
("minio.metadata.x-amz-object-lock-retain-until-date", "2099-01-01T00:00:00Z".to_string()),
|
||||
]),
|
||||
),
|
||||
(
|
||||
"version-id.tar",
|
||||
wrong_action_client,
|
||||
HashMap::from([("minio.versionId", Uuid::new_v4().to_string())]),
|
||||
),
|
||||
];
|
||||
|
||||
for (archive_key, client, pax) in cases {
|
||||
let archive = make_tar_with_pax_entry("entry.txt", b"must-not-write", None, &pax).await;
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(archive_key)
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect_err("missing, conditional, or wrong PAX privilege must be rejected");
|
||||
assert_eq!(
|
||||
err.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("AccessDenied"),
|
||||
"{archive_key}"
|
||||
);
|
||||
}
|
||||
|
||||
let version_condition_client = restricted_user_client(&env, version_condition_user, version_condition_secret);
|
||||
let mismatching_version_pax = HashMap::from([("minio.versionId", Uuid::new_v4().to_string())]);
|
||||
let archive = make_tar_with_pax_entry("version-mismatch-entry.txt", b"must-not-write", None, &mismatching_version_pax).await;
|
||||
let err = version_condition_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("version-mismatch.tar")
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a mismatching PAX version ID must fail the replication condition");
|
||||
assert_eq!(err.as_service_error().and_then(|error| error.meta().code()), Some("AccessDenied"));
|
||||
let err = admin_client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key("version-mismatch-entry.txt")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a denied PAX entry must not be written");
|
||||
assert!(matches!(
|
||||
err.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("NoSuchKey" | "NotFound")
|
||||
));
|
||||
|
||||
let matching_version_pax = HashMap::from([("minio.versionId", conditional_version_id)]);
|
||||
let archive = make_tar_with_pax_entry("condition-entry.txt", b"condition-body", None, &matching_version_pax).await;
|
||||
version_condition_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("version-condition.tar")
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
let stored = admin_client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("condition-entry.txt")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), b"condition-body");
|
||||
|
||||
let pax_context_client = restricted_user_client(&env, pax_context_user, pax_context_secret);
|
||||
let tag_pax = HashMap::from([("minio.metadata.x-amz-tagging", "classification=public".to_string())]);
|
||||
let archive = make_tar_with_pax_entry("tag-context-entry.txt", b"tag-context-body", None, &tag_pax).await;
|
||||
pax_context_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("tag-context.tar")
|
||||
.tagging("classification=restricted")
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
let tags = admin_client
|
||||
.get_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key("tag-context-entry.txt")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
tags.tag_set()
|
||||
.iter()
|
||||
.any(|tag| tag.key() == "classification" && tag.value() == "public")
|
||||
);
|
||||
|
||||
let pax_retain_until = "2040-01-01T00:00:00Z";
|
||||
let lock_pax = HashMap::from([
|
||||
("minio.metadata.x-amz-object-lock-mode", "COMPLIANCE".to_string()),
|
||||
("minio.metadata.x-amz-object-lock-retain-until-date", pax_retain_until.to_string()),
|
||||
]);
|
||||
let archive = make_tar_with_pax_entry("lock-context-entry.txt", b"lock-context-body", None, &lock_pax).await;
|
||||
pax_context_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("lock-context.tar")
|
||||
.object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Governance)
|
||||
.object_lock_retain_until_date(aws_sdk_s3::primitives::DateTime::from_secs(2_114_380_800))
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
let retention = admin_client
|
||||
.get_object_retention()
|
||||
.bucket(bucket)
|
||||
.key("lock-context-entry.txt")
|
||||
.send()
|
||||
.await?
|
||||
.retention()
|
||||
.expect("PAX retention should be present")
|
||||
.clone();
|
||||
assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("COMPLIANCE"));
|
||||
assert_eq!(
|
||||
retention
|
||||
.retain_until_date()
|
||||
.expect("PAX retain-until should be present")
|
||||
.fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime)?,
|
||||
pax_retain_until
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_accepts_compat_header() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression tests for the event notification startup race.
|
||||
//!
|
||||
//! Covers the recurring pattern where webhook/audit targets fail to load at boot
|
||||
//! due to startup ordering (notification runtime starts before server config is
|
||||
//! loaded). This has regressed 9+ times across beta.3 ~ beta.12.
|
||||
//!
|
||||
//! ## Regression Issues
|
||||
//!
|
||||
//! - rustfs#5387: webhook notifications broken again in beta.9+
|
||||
//! - rustfs#5681: Audit webhook targets are not loaded at boot
|
||||
//! - rustfs#5401: Event Destinations broken again
|
||||
//! - rustfs#5183: Audit webhooks stay offline after restart
|
||||
//! - rustfs#5115: init_event_notifier loses startup race against server config load
|
||||
//! - rustfs#4796: Pulsar event destinations offline after restart
|
||||
//! - rustfs#5428: MQTT bucket notifications stop on restarted cluster node
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
|
||||
/// RT-01: Verify that the notification runtime initializes correctly at boot.
|
||||
///
|
||||
/// Regression pattern: notification runtime initializes before server config
|
||||
/// is fully loaded, causing webhook targets to never come online.
|
||||
///
|
||||
/// This test verifies the startup ordering by checking that the server
|
||||
/// starts successfully with notification enabled and can serve S3 requests.
|
||||
/// A full webhook delivery test is in notification_webhook_test.rs.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_notification_enabled_server_starts_cleanly() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-01: notification enabled server starts cleanly");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false"), ("RUSTFS_NOTIFY_ENABLE", "true")])
|
||||
.await
|
||||
.expect("start RustFS with notifications enabled");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt01-notify-startup";
|
||||
|
||||
// Server should be healthy and able to serve S3 requests
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("create bucket with notifications enabled");
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("test.txt")
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"test"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object with notifications enabled");
|
||||
|
||||
info!("RT-01 PASS: notification enabled server starts and serves S3");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-02: Verify notification config persists after server restart.
|
||||
///
|
||||
/// Regression pattern: after a node restart, notification targets stay
|
||||
/// offline permanently because the config is not re-loaded.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. Start server with notification enabled
|
||||
/// 2. Create bucket and configure notification
|
||||
/// 3. Restart server
|
||||
/// 4. Verify notification config still exists
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_notification_config_survives_restart() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-02: notification config survives restart");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false"), ("RUSTFS_NOTIFY_ENABLE", "true")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt02-notify-restart";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
// Enable versioning (required for notification configuration)
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("enable versioning");
|
||||
|
||||
// Note: We can't fully test notification config persistence without a
|
||||
// configured target. But we verify the server restarts cleanly with
|
||||
// notification enabled, which is the core regression scenario.
|
||||
env.restart_server_preserving_data(vec![], &[])
|
||||
.await
|
||||
.expect("restart RustFS with notifications enabled");
|
||||
|
||||
// Verify bucket still exists and is accessible after restart
|
||||
let list = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("list objects after restart");
|
||||
|
||||
assert!(list.contents().is_empty(), "RT-02: bucket should be empty after restart");
|
||||
|
||||
// Verify we can still write objects (notification runtime initialized)
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("after-restart.txt")
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"post-restart"))
|
||||
.send()
|
||||
.await
|
||||
.expect("put object after restart — notification runtime must be initialized");
|
||||
|
||||
info!("RT-02 PASS: server with notifications survives restart");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,7 @@
|
||||
//! * PUT / multipart-complete / DeleteObject / DeleteObjects each deliver one event with the correct
|
||||
//! eventName, bucket, key, versionId and eTag.
|
||||
//! * prefix/suffix filters drop non-matching keys (rule-engine gate).
|
||||
//! * an event queued while the target endpoint rejects delivery is redelivered
|
||||
//! * an event queued while the target endpoint is unreachable is redelivered
|
||||
//! from the on-disk store once the endpoint recovers (store-and-forward).
|
||||
//! * responseElements and the S3 response use the canonical request ID while
|
||||
//! requestParameters preserve a conflicting client-supplied value.
|
||||
@@ -897,10 +897,11 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// An event queued while the target endpoint rejects delivery survives on the
|
||||
/// An event queued while the target endpoint is unreachable survives on the
|
||||
/// durable store and is redelivered once the endpoint comes back.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
#[ignore = "FAILING deterministically on main since it landed (#4821): the target is created but never appears in /rustfs/admin/v3/target/arns, so wait_for_target_registered times out. Quarantined per the flake policy; remove with the fix for rustfs#4852"]
|
||||
async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -931,55 +932,28 @@ async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult {
|
||||
wait_for_target_registered(&env, target).await?;
|
||||
put_notification_config(&client, bucket, target, "uploads/", ".dat").await?;
|
||||
|
||||
// Replace the healthy setup listener with one that rejects the first POST.
|
||||
// Waiting for that response below proves the queued event reached a failed
|
||||
// delivery attempt before the endpoint recovers.
|
||||
// Take the endpoint down (drops the listener, so connections are refused —
|
||||
// a retryable NotConnected), then PUT: the event cannot be delivered and
|
||||
// must survive on the durable queue store.
|
||||
setup_handle.abort();
|
||||
let _ = setup_handle.await;
|
||||
|
||||
let listener = TcpListener::bind(("0.0.0.0", port)).await?;
|
||||
let key = "uploads/redeliver.dat";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"queued while target rejects"))
|
||||
.body(ByteStream::from_static(b"queued while target down"))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let mut failure_handle = tokio::spawn(async move {
|
||||
loop {
|
||||
let (mut stream, _) = listener.accept().await?;
|
||||
let (method, _) = timeout(Duration::from_secs(5), read_http_message(&mut stream)).await??;
|
||||
if method == "HEAD" {
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
|
||||
.await?;
|
||||
stream.shutdown().await?;
|
||||
continue;
|
||||
}
|
||||
if method == "POST" {
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 503 Service Unavailable\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
|
||||
.await?;
|
||||
stream.shutdown().await?;
|
||||
return Ok::<(), BoxError>(());
|
||||
}
|
||||
}
|
||||
});
|
||||
// Hold the endpoint down long enough for at least one replay attempt to
|
||||
// fail (the replay worker scans the store every 500ms), so recovery below
|
||||
// exercises real redelivery rather than a first-attempt success.
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
|
||||
let rejected = match timeout(Duration::from_secs(20), &mut failure_handle).await {
|
||||
Ok(rejected) => rejected,
|
||||
Err(_) => {
|
||||
failure_handle.abort();
|
||||
let _ = failure_handle.await;
|
||||
return Err("webhook replay did not reach the rejecting endpoint".into());
|
||||
}
|
||||
};
|
||||
rejected??;
|
||||
|
||||
// Bring the endpoint back on the same port; the replay worker rescans the
|
||||
// durable queue and delivers the retained event.
|
||||
// Bring the endpoint back on the same port; the replay worker retries with
|
||||
// exponential backoff and delivers the queued event.
|
||||
let listener = TcpListener::bind(("0.0.0.0", port)).await?;
|
||||
let (tx, mut rx) = mpsc::unbounded_channel();
|
||||
let handle = serve_event_collector(listener, tx);
|
||||
|
||||
@@ -12,11 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_post, awscurl_put, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_get, awscurl_post, awscurl_put, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use http::{Method, StatusCode};
|
||||
use serial_test::serial;
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
use tracing::{debug, info};
|
||||
|
||||
fn skip_without_awscurl() -> bool {
|
||||
@@ -39,8 +37,7 @@ impl QuotaTestEnv {
|
||||
pub async fn new() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let bucket_name = format!("quota-test-{}", uuid::Uuid::new_v4());
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")])
|
||||
.await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let client = env.create_s3_client();
|
||||
|
||||
Ok(Self {
|
||||
@@ -70,7 +67,18 @@ impl QuotaTestEnv {
|
||||
}
|
||||
|
||||
pub async fn set_bucket_quota(&self, quota_bytes: u64) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.set_bucket_quota_for(&self.bucket_name, quota_bytes).await
|
||||
let url = format!("{}/rustfs/admin/v3/quota/{}", self.env.url, self.bucket_name);
|
||||
let quota_config = serde_json::json!({
|
||||
"quota": quota_bytes,
|
||||
"quota_type": "HARD"
|
||||
});
|
||||
|
||||
let response = awscurl_put(&url, "a_config.to_string(), &self.env.access_key, &self.env.secret_key).await?;
|
||||
if response.contains("error") {
|
||||
Err(format!("Failed to set quota: {}", response).into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_bucket_quota(&self) -> Result<Option<u64>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
@@ -170,29 +178,6 @@ impl QuotaTestEnv {
|
||||
bucket: &str,
|
||||
quota_bytes: u64,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let stats_path = format!("/rustfs/admin/v3/quota-stats/{bucket}");
|
||||
let readiness = async {
|
||||
loop {
|
||||
let (status, response) =
|
||||
admin_request(&self.env.url, Method::GET, &stats_path, None, &self.env.access_key, &self.env.secret_key)
|
||||
.await?;
|
||||
if status.is_success() {
|
||||
return Ok::<(), Box<dyn std::error::Error + Send + Sync>>(());
|
||||
}
|
||||
if status != StatusCode::SERVICE_UNAVAILABLE {
|
||||
return Err(format!("quota usage readiness failed for {bucket}: {status} {response}").into());
|
||||
}
|
||||
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
};
|
||||
match timeout(Duration::from_secs(30), readiness).await {
|
||||
Ok(result) => result?,
|
||||
Err(_) => {
|
||||
return Err(format!("quota usage did not become authoritative for {bucket} within 30 seconds").into());
|
||||
}
|
||||
}
|
||||
|
||||
let url = format!("{}/rustfs/admin/v3/quota/{}", self.env.url, bucket);
|
||||
let quota_config = serde_json::json!({
|
||||
"quota": quota_bytes,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,10 +12,13 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, build_test_s3_config, build_test_sts_client, init_logging};
|
||||
use aws_sdk_sts::Client;
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
|
||||
use aws_sdk_sts::config::retry::RetryConfig;
|
||||
use aws_sdk_sts::config::{Credentials, Region};
|
||||
use aws_sdk_sts::error::ProvideErrorMetadata;
|
||||
use aws_sdk_sts::operation::RequestId;
|
||||
use aws_sdk_sts::{Client, Config};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use bytes::Bytes;
|
||||
use http::header::{AUTHORIZATION, CONTENT_TYPE};
|
||||
use http::{Request, Response};
|
||||
@@ -29,8 +32,9 @@ use serial_test::serial;
|
||||
use std::collections::BTreeSet;
|
||||
use std::convert::Infallible;
|
||||
use std::error::Error;
|
||||
use std::sync::Arc;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::sync::{Notify, mpsc};
|
||||
use tokio::task::{JoinHandle, JoinSet};
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
@@ -39,7 +43,22 @@ type TestResult = Result<(), BoxError>;
|
||||
const OPA_AUTH_TOKEN: &str = "sts-opa-token";
|
||||
|
||||
fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
|
||||
build_test_sts_client(url, access_key, secret_key, session_token, "e2e-sts-query-compat")
|
||||
let mut config = Config::builder()
|
||||
.credentials_provider(Credentials::new(
|
||||
access_key,
|
||||
secret_key,
|
||||
session_token.map(str::to_owned),
|
||||
None,
|
||||
"e2e-sts-query-compat",
|
||||
))
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(url)
|
||||
.retry_config(RetryConfig::standard().with_max_attempts(1))
|
||||
.behavior_version_latest();
|
||||
if url.starts_with("http://") {
|
||||
config = config.http_client(SmithyHttpClientBuilder::new().build_http());
|
||||
}
|
||||
Client::from_conf(config.build())
|
||||
}
|
||||
|
||||
async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(String, String), BoxError> {
|
||||
@@ -126,52 +145,6 @@ async fn assert_access_denied(client: &Client, context: &str) -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_list_buckets_access_denied(
|
||||
env: &RustFSTestEnvironment,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
context: &str,
|
||||
) -> TestResult {
|
||||
let error = aws_sdk_s3::Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
access_key,
|
||||
secret_key,
|
||||
None,
|
||||
"e2e-list-buckets-opa-unavailable",
|
||||
))
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("ListBuckets must be denied while OPA is unavailable");
|
||||
let service_error = error
|
||||
.as_service_error()
|
||||
.ok_or_else(|| format!("{context} should deserialize as an S3 service error: {error:?}"))?;
|
||||
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(403));
|
||||
assert_eq!(service_error.code(), Some("AccessDenied"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_opa_unavailable_denies_sts_and_list_buckets(env: &RustFSTestEnvironment, context: &str) -> TestResult {
|
||||
let user = "opaunavailable";
|
||||
let secret = "stsOpaUnavailableSecret123";
|
||||
create_user_with_policy(
|
||||
env,
|
||||
user,
|
||||
secret,
|
||||
"sts-opa-unavailable-local-policy",
|
||||
serde_json::json!([{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListAllMyBuckets"],
|
||||
"Resource": ["arn:aws:s3:::*"],
|
||||
}]),
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_access_denied(&sts_client(&env.url, user, secret, None), context).await?;
|
||||
assert_list_buckets_access_denied(env, user, secret, context).await
|
||||
}
|
||||
|
||||
async fn handle_opa_request(
|
||||
request: Request<Incoming>,
|
||||
requests: mpsc::UnboundedSender<Value>,
|
||||
@@ -213,15 +186,12 @@ async fn handle_opa_request(
|
||||
};
|
||||
if payload.is_none() {
|
||||
let _ = validation_started.send(());
|
||||
match validation_mode {
|
||||
OpaValidationMode::Blocked => std::future::pending::<()>().await,
|
||||
OpaValidationMode::Unavailable => {
|
||||
return Ok(Response::builder()
|
||||
.status(503)
|
||||
.body(Full::new(Bytes::new()))
|
||||
.expect("static OPA unavailable response must be valid"));
|
||||
}
|
||||
OpaValidationMode::Ready => {}
|
||||
if let OpaValidationMode::DelayedUnavailable(release) = validation_mode {
|
||||
release.notified().await;
|
||||
return Ok(Response::builder()
|
||||
.status(503)
|
||||
.body(Full::new(Bytes::new()))
|
||||
.expect("static OPA unavailable response must be valid"));
|
||||
}
|
||||
}
|
||||
let allow = match payload.as_ref().and_then(|value| value.pointer("/input/identity/account")) {
|
||||
@@ -231,25 +201,6 @@ async fn handle_opa_request(
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false),
|
||||
Some(Value::String(account)) if account == "opadeny" => false,
|
||||
Some(Value::String(account))
|
||||
if account == "opaunavailable" && matches!(validation_mode, OpaValidationMode::Unavailable) =>
|
||||
{
|
||||
true
|
||||
}
|
||||
Some(Value::String(account)) if account == "opalistbuckets" => {
|
||||
let action = payload
|
||||
.as_ref()
|
||||
.and_then(|value| value.pointer("/input/action"))
|
||||
.and_then(Value::as_str);
|
||||
let bucket = payload
|
||||
.as_ref()
|
||||
.and_then(|value| value.pointer("/input/resource/bucket"))
|
||||
.and_then(Value::as_str);
|
||||
matches!(
|
||||
(action, bucket),
|
||||
(Some("s3:ListBucket"), Some("opa-list-visible")) | (Some("s3:GetBucketLocation"), Some("opa-list-location"))
|
||||
)
|
||||
}
|
||||
None => true,
|
||||
_ => false,
|
||||
};
|
||||
@@ -264,17 +215,17 @@ async fn handle_opa_request(
|
||||
.expect("static OPA response must be valid"))
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
#[derive(Clone)]
|
||||
enum OpaValidationMode {
|
||||
Ready,
|
||||
Blocked,
|
||||
Unavailable,
|
||||
DelayedUnavailable(Arc<Notify>),
|
||||
}
|
||||
|
||||
struct OpaMock {
|
||||
url: String,
|
||||
requests: mpsc::UnboundedReceiver<Value>,
|
||||
validation_started: mpsc::UnboundedReceiver<()>,
|
||||
validation_release: Option<Arc<Notify>>,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
|
||||
@@ -283,8 +234,9 @@ impl OpaMock {
|
||||
Self::start_with_mode(OpaValidationMode::Ready, Some(OPA_AUTH_TOKEN)).await
|
||||
}
|
||||
|
||||
async fn start_blocked() -> Result<Self, BoxError> {
|
||||
Self::start_with_mode(OpaValidationMode::Blocked, None).await
|
||||
async fn start_delayed_unavailable() -> Result<Self, BoxError> {
|
||||
let release = Arc::new(Notify::new());
|
||||
Self::start_with_mode(OpaValidationMode::DelayedUnavailable(release), None).await
|
||||
}
|
||||
|
||||
async fn start_with_mode(validation_mode: OpaValidationMode, auth_token: Option<&str>) -> Result<Self, BoxError> {
|
||||
@@ -293,6 +245,10 @@ impl OpaMock {
|
||||
let (requests_tx, requests) = mpsc::unbounded_channel();
|
||||
let (validation_started_tx, validation_started) = mpsc::unbounded_channel();
|
||||
let expected_authorization = auth_token.map(|token| format!("Bearer {token}"));
|
||||
let validation_release = match &validation_mode {
|
||||
OpaValidationMode::Ready => None,
|
||||
OpaValidationMode::DelayedUnavailable(release) => Some(Arc::clone(release)),
|
||||
};
|
||||
let task = tokio::spawn(async move {
|
||||
let mut connections = JoinSet::new();
|
||||
loop {
|
||||
@@ -301,7 +257,7 @@ impl OpaMock {
|
||||
let Ok((stream, _)) = accepted else { break };
|
||||
let requests = requests_tx.clone();
|
||||
let validation_started = validation_started_tx.clone();
|
||||
let validation_mode = validation_mode;
|
||||
let validation_mode = validation_mode.clone();
|
||||
let expected_authorization = expected_authorization.clone();
|
||||
connections.spawn(async move {
|
||||
let handler = service_fn(move |request| {
|
||||
@@ -309,7 +265,7 @@ impl OpaMock {
|
||||
request,
|
||||
requests.clone(),
|
||||
validation_started.clone(),
|
||||
validation_mode,
|
||||
validation_mode.clone(),
|
||||
expected_authorization.clone(),
|
||||
)
|
||||
});
|
||||
@@ -326,6 +282,7 @@ impl OpaMock {
|
||||
url,
|
||||
requests,
|
||||
validation_started,
|
||||
validation_release,
|
||||
task,
|
||||
})
|
||||
}
|
||||
@@ -341,6 +298,12 @@ impl OpaMock {
|
||||
.await?
|
||||
.ok_or_else(|| "OPA validation channel closed".into())
|
||||
}
|
||||
|
||||
fn release_validation(&self) {
|
||||
if let Some(release) = &self.validation_release {
|
||||
release.notify_one();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for OpaMock {
|
||||
@@ -560,119 +523,35 @@ async fn test_sts_assume_role_opa_contract() -> TestResult {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_list_buckets_opa_contract() -> TestResult {
|
||||
async fn test_sts_assume_role_fails_closed_while_opa_is_unavailable() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut opa = OpaMock::start().await?;
|
||||
let mut opa = OpaMock::start_delayed_unavailable().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
|
||||
],
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
|
||||
.await?;
|
||||
opa.wait_for_validation().await?;
|
||||
|
||||
let user = "opaunavailable";
|
||||
let secret = "stsOpaUnavailableSecret123";
|
||||
create_user_with_policy(
|
||||
&env,
|
||||
user,
|
||||
secret,
|
||||
"sts-opa-unavailable-local-policy",
|
||||
serde_json::json!([{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListAllMyBuckets"],
|
||||
"Resource": ["arn:aws:s3:::*"],
|
||||
}]),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
for bucket in ["opa-list-hidden", "opa-list-location", "opa-list-visible"] {
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
}
|
||||
assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA initialization").await?;
|
||||
|
||||
let user = "opalistbuckets";
|
||||
let secret = "opaListBucketsSecret123";
|
||||
create_user(&env, user, secret).await?;
|
||||
|
||||
let output = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, user, secret, None, "e2e-list-buckets-opa"))
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?;
|
||||
let mut names = output
|
||||
.buckets()
|
||||
.iter()
|
||||
.filter_map(|bucket| bucket.name().map(str::to_owned))
|
||||
.collect::<Vec<_>>();
|
||||
names.sort();
|
||||
assert_eq!(names, ["opa-list-location", "opa-list-visible"]);
|
||||
|
||||
let mut evaluations = BTreeSet::new();
|
||||
for _ in 0..6 {
|
||||
let request = opa.next_request().await?;
|
||||
assert_eq!(request.pointer("/input/identity/account").and_then(Value::as_str), Some(user));
|
||||
assert_eq!(request.pointer("/input/context/deny_only").and_then(Value::as_bool), Some(false));
|
||||
|
||||
let action = request
|
||||
.pointer("/input/action")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("OPA ListBuckets input should include action")?;
|
||||
let bucket = request
|
||||
.pointer("/input/resource/bucket")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("OPA ListBuckets input should include resource.bucket")?;
|
||||
if bucket.is_empty() {
|
||||
assert_eq!(action, "s3:ListAllMyBuckets");
|
||||
assert!(request.pointer("/input/context/conditions/prefix").is_none());
|
||||
assert!(request.pointer("/input/context/conditions/delimiter").is_none());
|
||||
} else {
|
||||
let expected_arn = format!("arn:aws:s3:::{bucket}");
|
||||
assert_eq!(request.pointer("/input/context/conditions/prefix"), Some(&serde_json::json!([""])));
|
||||
assert_eq!(request.pointer("/input/context/conditions/delimiter"), Some(&serde_json::json!(["/"])));
|
||||
assert_eq!(
|
||||
request.pointer("/input/resource/arn").and_then(Value::as_str),
|
||||
Some(expected_arn.as_str())
|
||||
);
|
||||
}
|
||||
evaluations.insert((action.to_owned(), bucket.to_owned()));
|
||||
}
|
||||
assert_eq!(
|
||||
evaluations,
|
||||
BTreeSet::from([
|
||||
("s3:GetBucketLocation".to_owned(), "opa-list-hidden".to_owned()),
|
||||
("s3:GetBucketLocation".to_owned(), "opa-list-location".to_owned()),
|
||||
("s3:ListAllMyBuckets".to_owned(), String::new()),
|
||||
("s3:ListBucket".to_owned(), "opa-list-hidden".to_owned()),
|
||||
("s3:ListBucket".to_owned(), "opa-list-location".to_owned()),
|
||||
("s3:ListBucket".to_owned(), "opa-list-visible".to_owned()),
|
||||
])
|
||||
);
|
||||
assert!(
|
||||
matches!(opa.requests.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
|
||||
"ListBuckets should not make redundant OPA evaluations"
|
||||
);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_sts_and_list_buckets_fail_closed_while_opa_is_initializing() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut opa = OpaMock::start_blocked().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
|
||||
.await?;
|
||||
opa.wait_for_validation().await?;
|
||||
|
||||
assert_opa_unavailable_denies_sts_and_list_buckets(&env, "configured OPA initialization").await?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_sts_and_list_buckets_fail_closed_after_opa_validation_failure() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut opa = OpaMock::start_with_mode(OpaValidationMode::Unavailable, None).await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
|
||||
.await?;
|
||||
opa.wait_for_validation().await?;
|
||||
|
||||
assert_opa_unavailable_denies_sts_and_list_buckets(&env, "configured OPA validation failure").await?;
|
||||
opa.release_validation();
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA validation failure").await?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
|
||||
@@ -1,172 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression tests for Tier/ILM transition operations.
|
||||
//!
|
||||
//! Covers the recurring pattern where tier transition fails silently, the
|
||||
//! free-version recovery task loops forever, or transitioned objects cannot
|
||||
//! be read back. This has regressed 6+ times.
|
||||
//!
|
||||
//! ## Regression Issues
|
||||
//!
|
||||
//! - rustfs#5218: Remote tier mutation commit failed
|
||||
//! - rustfs#5130: tier_free_version_recovery task loops forever
|
||||
//! - rustfs#5011: Idle tier free-version recovery rescans every 60 seconds
|
||||
//! - rustfs#4826: Full GET of multipart transitioned object fails
|
||||
//! - rustfs#5024: Some files succeeded in tier offloading, others failed
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
|
||||
/// RT-13: Verify lifecycle rule with transition persists and is retrievable.
|
||||
///
|
||||
/// Note: Actual transition requires a configured remote tier. This test
|
||||
/// validates that an expiration-only rule (the persistence path) survives
|
||||
/// a server restart.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_lifecycle_rule_persists_after_restart() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-13: lifecycle rule persists after restart");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "rt13-tier-persist";
|
||||
|
||||
client.create_bucket().bucket(bucket).send().await.expect("create bucket");
|
||||
|
||||
// Apply a lifecycle rule with expiration (transition needs a real tier)
|
||||
let rule = aws_sdk_s3::types::LifecycleRule::builder()
|
||||
.id("expire-after-90d")
|
||||
.status(aws_sdk_s3::types::ExpirationStatus::Enabled)
|
||||
.filter(aws_sdk_s3::types::LifecycleRuleFilter::builder().prefix("archive/").build())
|
||||
.expiration(aws_sdk_s3::types::LifecycleExpiration::builder().days(90).build())
|
||||
.build()
|
||||
.expect("build rule");
|
||||
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.lifecycle_configuration(
|
||||
aws_sdk_s3::types::BucketLifecycleConfiguration::builder()
|
||||
.rules(rule)
|
||||
.build()
|
||||
.expect("build config"),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("put lifecycle");
|
||||
|
||||
// Restart server
|
||||
env.restart_server_preserving_data(vec![], &[]).await.expect("restart RustFS");
|
||||
|
||||
// Verify the rule survived restart
|
||||
let resp = client
|
||||
.get_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("get lifecycle after restart");
|
||||
|
||||
let rules = resp.rules();
|
||||
assert_eq!(rules.len(), 1, "RT-13 FAIL: expected 1 rule after restart");
|
||||
|
||||
let exp = rules[0].expiration().expect("expiration should be set");
|
||||
assert_eq!(exp.days(), Some(90), "RT-13 FAIL: expiration days corrupted after restart");
|
||||
|
||||
info!("RT-13 PASS: lifecycle rule persists after restart");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-13b: Verify admin tier configuration API is functional.
|
||||
///
|
||||
/// Regression pattern: tier add/verify/delete API fails or the tier
|
||||
/// configuration is not persisted (rustfs#5218).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_admin_tier_list_endpoint_returns_json() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-13b: admin tier list endpoint returns JSON");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
// Query the tier list endpoint
|
||||
let body = admin_ok(&env, http::Method::GET, "/rustfs/admin/v3/tier", None)
|
||||
.await
|
||||
.expect("list remote tiers");
|
||||
|
||||
let json: Value = serde_json::from_str(&body).expect("tier list response should be valid JSON");
|
||||
|
||||
// Should return an array (possibly empty)
|
||||
assert!(json.is_array(), "RT-13b FAIL: tier list response is not an array: {json}");
|
||||
|
||||
info!("RT-13b PASS: admin tier list endpoint returns valid JSON array");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// RT-13c: Verify scanner configuration persistence.
|
||||
///
|
||||
/// Regression pattern: scanner admin config update reports success but
|
||||
/// is not persisted (rustfs#5013), causing the scanner to not run or
|
||||
/// use stale settings.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scanner_config_persists_after_restart() -> TestResult {
|
||||
init_logging();
|
||||
info!("RT-13c: scanner config persists after restart");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_CONSOLE_ENABLE", "false")])
|
||||
.await
|
||||
.expect("start RustFS");
|
||||
|
||||
// Get current scanner status
|
||||
let body = admin_ok(&env, http::Method::GET, "/rustfs/admin/v3/scanner/status", None)
|
||||
.await
|
||||
.expect("get scanner status");
|
||||
|
||||
let json: Value = serde_json::from_str(&body).expect("scanner status should be valid JSON");
|
||||
|
||||
info!(" scanner status: {:?}", json.as_object().map(|o| o.keys().collect::<Vec<_>>()));
|
||||
|
||||
// Restart and verify config is still accessible
|
||||
env.restart_server_preserving_data(vec![], &[]).await.expect("restart RustFS");
|
||||
|
||||
let body2 = admin_ok(&env, http::Method::GET, "/rustfs/admin/v3/scanner/status", None)
|
||||
.await
|
||||
.expect("get scanner status after restart");
|
||||
|
||||
let json2: Value = serde_json::from_str(&body2).expect("scanner status after restart should be valid JSON");
|
||||
|
||||
// Both should be valid JSON objects
|
||||
assert!(json2.is_object(), "RT-13c FAIL: scanner status after restart is not a valid JSON object");
|
||||
|
||||
info!("RT-13c PASS: scanner/config persists across restart");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -144,13 +144,11 @@ rustfs-lifecycle.workspace = true
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-data-usage.workspace = true
|
||||
rustfs-object-capacity.workspace = true
|
||||
rustfs-object-data-cache = { workspace = true, features = ["runtime-memory"] }
|
||||
arc-swap.workspace = true
|
||||
async-trait.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
byteorder = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
glob = { workspace = true }
|
||||
thiserror.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
|
||||
@@ -130,15 +130,13 @@ pub mod bucket {
|
||||
|
||||
pub mod metadata_sys {
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy,
|
||||
BucketMetadataSys, acquire_bucket_metadata_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
|
||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
|
||||
get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config,
|
||||
get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
|
||||
set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
|
||||
update_under_transaction_lock,
|
||||
get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
|
||||
update, update_bucket_targets_under_transaction_lock, update_config_with, update_under_transaction_lock,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -180,24 +178,20 @@ pub mod bucket {
|
||||
mrf_backlog_observability_snapshot,
|
||||
};
|
||||
pub use crate::bucket::replication::{
|
||||
BucketReplicationResyncStatus, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot,
|
||||
DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry,
|
||||
MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
|
||||
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
||||
REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo,
|
||||
ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigStructureError, ReplicationConfigurationExt,
|
||||
BucketReplicationResyncStatus, BucketStats, DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo,
|
||||
DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts,
|
||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationConfig, ReplicationConfigurationExt,
|
||||
ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge,
|
||||
ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission,
|
||||
ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage,
|
||||
ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog,
|
||||
TargetReplicationResyncStatus, VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||
replication_statuses_map, replication_target_arns, resync_start_conflict_id, should_remove_replication_target,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
||||
validate_replication_config_structure, validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
TargetReplicationResyncStatus, VersionPurgeStatusType, delete_replication_state_from_config,
|
||||
delete_replication_version_id, get_global_replication_pool, get_global_replication_stats,
|
||||
init_background_replication, invalid_replication_config_status_field, read_durable_mrf_backlog,
|
||||
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
|
||||
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
unsupported_replication_config_field, validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -206,9 +200,7 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod target {
|
||||
pub use crate::bucket::target::{
|
||||
ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials, LatencyStat, duration_from_secs_or_nanos,
|
||||
};
|
||||
pub use crate::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials, LatencyStat};
|
||||
}
|
||||
|
||||
pub mod utils {
|
||||
@@ -310,8 +302,7 @@ pub mod config {
|
||||
pub mod data_usage {
|
||||
pub use crate::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
|
||||
init_compression_total_memory_from_backend, invalidate_data_usage_snapshot_cache, live_bucket_usage_computations,
|
||||
load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached,
|
||||
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
|
||||
record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
|
||||
@@ -408,11 +399,11 @@ pub mod notification {
|
||||
pub mod object {
|
||||
pub use crate::object_api::{
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
|
||||
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader,
|
||||
ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, ObjectEncryptionResolver, ObjectInfo,
|
||||
ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode,
|
||||
ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
|
||||
register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
|
||||
unregister_object_mutation_hook,
|
||||
};
|
||||
pub use crate::store::PreparedGetObjectReader;
|
||||
}
|
||||
@@ -436,14 +427,13 @@ pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
|
||||
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_put_file_auth_trailer, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
|
||||
tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -12,127 +12,20 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::bucket::metadata_sys::{self, ObjectLockConfigState};
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct LifecycleExpiryConfigs {
|
||||
pub(crate) lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
|
||||
pub(crate) object_lock: Option<Arc<ObjectLockConfiguration>>,
|
||||
pub(crate) bucket_incarnation_id: Uuid,
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result<LifecycleExpiryConfigs> {
|
||||
let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let sys = metadata_sys::bucket_metadata_sys_of(&api.ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
let metadata = sys.get_authoritative_metadata(bucket).await?;
|
||||
if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id != bucket_incarnation_id {
|
||||
return Err(Error::other(format!("bucket lifecycle metadata is not authoritative: {bucket}")));
|
||||
}
|
||||
|
||||
let lifecycle = if metadata.lifecycle_config.is_none() && !metadata.lifecycle_config_xml.is_empty() {
|
||||
return Err(Error::other("persisted bucket lifecycle configuration is invalid"));
|
||||
} else {
|
||||
metadata
|
||||
.lifecycle_config
|
||||
.clone()
|
||||
.filter(|config| !config.rules.is_empty())
|
||||
.map(Arc::new)
|
||||
};
|
||||
if lifecycle.is_none() {
|
||||
return Ok(LifecycleExpiryConfigs {
|
||||
lifecycle: None,
|
||||
object_lock: None,
|
||||
bucket_incarnation_id,
|
||||
});
|
||||
}
|
||||
let object_lock = match metadata_sys::object_lock_config_state_from_authoritative_metadata(&metadata)? {
|
||||
ObjectLockConfigState::Configured { config, .. } => Some(Arc::new(config)),
|
||||
ObjectLockConfigState::ConfirmedAbsent => None,
|
||||
ObjectLockConfigState::Fabricated => {
|
||||
return Err(Error::other(format!("bucket Object Lock metadata is not authoritative: {bucket}")));
|
||||
}
|
||||
};
|
||||
|
||||
Ok(LifecycleExpiryConfigs {
|
||||
lifecycle,
|
||||
object_lock,
|
||||
bucket_incarnation_id,
|
||||
})
|
||||
}
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::error::Result;
|
||||
|
||||
pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
|
||||
metadata_sys::get_lifecycle_config(bucket).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::{self, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
use s3s::dto::{ExpirationStatus, LifecycleExpiration, LifecycleRule};
|
||||
use serial_test::serial;
|
||||
|
||||
fn lifecycle_config() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some("expire".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn expiry_configs_are_resolved_from_the_owning_store() {
|
||||
let (_dirs_a, store_a) = isolated_store_over_temp_disks().await;
|
||||
let (_dirs_b, store_b) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "same-name-expiry-config";
|
||||
store_a
|
||||
.peer_sys
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
store_b
|
||||
.peer_sys
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_sys::init_bucket_metadata_sys(store_a.clone(), vec![bucket.to_string()]).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store_b.clone(), vec![bucket.to_string()]).await;
|
||||
|
||||
let mut metadata = BucketMetadata::new(bucket);
|
||||
let lifecycle = lifecycle_config();
|
||||
metadata.lifecycle_config_xml = crate::bucket::utils::serialize(&lifecycle).unwrap();
|
||||
metadata.lifecycle_config = Some(lifecycle);
|
||||
metadata_sys::set_new_bucket_metadata_in(&store_a.ctx, metadata)
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_sys::set_new_bucket_metadata_in(&store_b.ctx, BucketMetadata::new(bucket))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(get_expiry_configs(&store_a, bucket).await.unwrap().lifecycle.is_some());
|
||||
assert!(get_expiry_configs(&store_b, bucket).await.unwrap().lifecycle.is_none());
|
||||
}
|
||||
pub(crate) async fn get_object_lock_config(bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> {
|
||||
metadata_sys::get_object_lock_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
metadata_sys::get_replication_config(bucket).await
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ pub mod core;
|
||||
pub mod evaluator;
|
||||
pub mod manual_transition_job;
|
||||
mod metadata_boundary;
|
||||
pub(crate) use metadata_boundary::get_expiry_configs;
|
||||
mod object_lock_boundary;
|
||||
pub use self::core as lifecycle;
|
||||
mod replication_sink;
|
||||
|
||||
@@ -21,12 +21,12 @@ pub(crate) fn is_object_locked_by_metadata(user_defined: &HashMap<String, String
|
||||
rustfs_lifecycle::object_lock::is_object_locked_by_metadata(user_defined, is_delete_marker)
|
||||
}
|
||||
|
||||
pub(crate) fn check_object_lock_for_deletion_with_config(
|
||||
config: Option<&s3s::dto::ObjectLockConfiguration>,
|
||||
pub(crate) async fn check_object_lock_for_deletion(
|
||||
bucket: &str,
|
||||
obj_info: &ObjectInfo,
|
||||
bypass_governance: bool,
|
||||
) -> crate::error::Result<Option<ObjectLockBlockReason>> {
|
||||
objectlock_sys::check_object_lock_for_deletion_with_config(config, obj_info, bypass_governance)
|
||||
) -> Option<ObjectLockBlockReason> {
|
||||
objectlock_sys::check_object_lock_for_deletion(bucket, obj_info, bypass_governance).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -15,14 +15,15 @@
|
||||
use rustfs_common::metrics::IlmAction;
|
||||
|
||||
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
||||
pub(crate) use crate::bucket::replication::ReplicationStatusType;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
|
||||
pub(crate) use crate::bucket::replication::ReplicateTargetDecision;
|
||||
pub(crate) use crate::bucket::replication::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
|
||||
ReplicateDecision, ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta,
|
||||
replication_statuses_map, version_purge_statuses_map,
|
||||
};
|
||||
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||
use crate::storage_api_contracts::object::DeletedObject;
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::storage_api_contracts::object::{DeletedObject, ObjectToDelete};
|
||||
|
||||
pub(crate) type LifecycleReplicationConfig = ReplicationLifecycleConfig;
|
||||
|
||||
@@ -56,6 +57,15 @@ pub(crate) fn lifecycle_action_waits_for_replication(action: IlmAction) -> bool
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn check_delete_replication(
|
||||
bucket: &str,
|
||||
object: ObjectToDelete,
|
||||
source: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
) -> ReplicateDecision {
|
||||
ReplicationLifecycleBridge::check_delete_replication(bucket, &object, source, opts).await
|
||||
}
|
||||
|
||||
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
|
||||
ReplicationLifecycleBridge::schedule_delete(bucket, delete_object).await;
|
||||
}
|
||||
@@ -64,16 +74,7 @@ pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
use rustfs_common::metrics::IlmAction;
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
|
||||
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -138,97 +139,4 @@ mod tests {
|
||||
assert!(lifecycle_action_waits_for_replication(IlmAction::TransitionVersionAction));
|
||||
assert!(!lifecycle_action_waits_for_replication(IlmAction::NoneAction));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_delete_admission_uses_marker_and_version_switches_for_all_purges() {
|
||||
for marker_enabled in [false, true] {
|
||||
for purge_enabled in [false, true] {
|
||||
let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: Some(DeleteMarkerReplication {
|
||||
status: Some(if marker_enabled {
|
||||
DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)
|
||||
} else {
|
||||
DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)
|
||||
}),
|
||||
}),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: if purge_enabled {
|
||||
DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED)
|
||||
} else {
|
||||
DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED)
|
||||
},
|
||||
}),
|
||||
destination: Destination {
|
||||
bucket: "arn:rustfs:replication:target".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: None,
|
||||
filter: None,
|
||||
id: Some("lifecycle-delete-switches".to_string()),
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
);
|
||||
let source = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "logs/object".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let marker = ObjectToDelete {
|
||||
object_name: source.name.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let marker_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
ReplicationObjectBridge::check_delete_with_snapshot(&marker, &source, &marker_opts, false, &snapshot)
|
||||
.replicate_any(),
|
||||
marker_enabled
|
||||
);
|
||||
|
||||
for delete_marker in [false, true] {
|
||||
for version_id in [Uuid::new_v4(), Uuid::nil()] {
|
||||
let purge = ObjectToDelete {
|
||||
object_name: source.name.clone(),
|
||||
version_id: Some(version_id),
|
||||
..Default::default()
|
||||
};
|
||||
let purge_source = ObjectInfo {
|
||||
delete_marker,
|
||||
..source.clone()
|
||||
};
|
||||
let purge_opts = ObjectOptions {
|
||||
version_id: Some(version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
ReplicationObjectBridge::check_delete_with_snapshot(
|
||||
&purge,
|
||||
&purge_source,
|
||||
&purge_opts,
|
||||
false,
|
||||
&snapshot,
|
||||
)
|
||||
.replicate_any(),
|
||||
purge_enabled,
|
||||
"delete marker={delete_marker}, version_id={version_id}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,10 +20,8 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::runtime_boundary;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
Jentry, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
Jentry, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
@@ -32,7 +30,7 @@ use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader
|
||||
use crate::services::tier::tier::tier_destination_id_from_metadata;
|
||||
use crate::storage_api_contracts::{
|
||||
list::ListOperations as _,
|
||||
object::{DeletedObject, HTTPPreconditions, ObjectIO, ObjectOperations, ObjectToDelete},
|
||||
object::{DeletedObject, ObjectIO, ObjectOperations, ObjectToDelete},
|
||||
range::HTTPRangeSpec,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
@@ -48,7 +46,6 @@ const TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
|
||||
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
|
||||
const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
|
||||
const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
|
||||
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -64,22 +61,13 @@ struct PersistedTierDeleteJournalEntry {
|
||||
version_id_exact: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
version_state: Option<rustfs_filemeta::TransitionVersionState>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
state: Option<TierDeleteJournalState>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
source: Option<TierDeleteSourceIdentity>,
|
||||
}
|
||||
|
||||
impl PersistedTierDeleteJournalEntry {
|
||||
fn from_jentry(je: &Jentry) -> Result<Self> {
|
||||
validate_version_state(je.version_state, &je.version_id, je.version_id_exact)?;
|
||||
let legacy_unknown = je.version_state == rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
let version = if je.source.is_some() || je.state == TierDeleteJournalState::Prepared {
|
||||
if je.backend_identity.is_none() {
|
||||
return Err(Error::other("tier delete transaction is missing its backend identity"));
|
||||
}
|
||||
TIER_DELETE_JOURNAL_TRANSACTION_VERSION
|
||||
} else if legacy_unknown {
|
||||
let version = if legacy_unknown {
|
||||
if je.backend_identity.is_some() {
|
||||
TIER_DELETE_JOURNAL_VERSION
|
||||
} else {
|
||||
@@ -99,10 +87,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
backend_identity: je.backend_identity,
|
||||
version_id_exact: je.version_id_exact.then_some(true),
|
||||
version_state: (!legacy_unknown).then_some(je.version_state),
|
||||
state: (version == TIER_DELETE_JOURNAL_TRANSACTION_VERSION).then_some(je.state),
|
||||
source: (version == TIER_DELETE_JOURNAL_TRANSACTION_VERSION)
|
||||
.then(|| je.source.clone())
|
||||
.flatten(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -117,21 +101,14 @@ impl PersistedTierDeleteJournalEntry {
|
||||
}
|
||||
if self.version != TIER_DELETE_JOURNAL_EXACT_VERSION
|
||||
&& self.version != TIER_DELETE_JOURNAL_STATE_VERSION
|
||||
&& self.version != TIER_DELETE_JOURNAL_TRANSACTION_VERSION
|
||||
&& self.version_id_exact.unwrap_or(false)
|
||||
{
|
||||
return Err(Error::other(
|
||||
"legacy tier delete journal entry has an unsupported exact version constraint",
|
||||
));
|
||||
}
|
||||
let (backend_identity, version_id_exact, version_state, state, source) = match self.version {
|
||||
1 => (
|
||||
None,
|
||||
false,
|
||||
rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
TierDeleteJournalState::Committed,
|
||||
None,
|
||||
),
|
||||
let (backend_identity, version_id_exact, version_state) = match self.version {
|
||||
1 => (None, false, rustfs_filemeta::TransitionVersionState::Unknown),
|
||||
TIER_DELETE_JOURNAL_VERSION => (
|
||||
Some(
|
||||
self.backend_identity
|
||||
@@ -139,8 +116,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
),
|
||||
false,
|
||||
rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
TierDeleteJournalState::Committed,
|
||||
None,
|
||||
),
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION => {
|
||||
if self.version_id.is_empty() || self.version_id_exact != Some(true) {
|
||||
@@ -153,8 +128,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
),
|
||||
true,
|
||||
rustfs_filemeta::TransitionVersionState::Exact,
|
||||
TierDeleteJournalState::Committed,
|
||||
None,
|
||||
)
|
||||
}
|
||||
TIER_DELETE_JOURNAL_STATE_VERSION => {
|
||||
@@ -170,31 +143,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
),
|
||||
exact,
|
||||
state,
|
||||
TierDeleteJournalState::Committed,
|
||||
None,
|
||||
)
|
||||
}
|
||||
TIER_DELETE_JOURNAL_TRANSACTION_VERSION => {
|
||||
let state = self
|
||||
.state
|
||||
.ok_or_else(|| Error::other("tier delete journal v5 entry is missing its state"))?;
|
||||
let source = self
|
||||
.source
|
||||
.ok_or_else(|| Error::other("tier delete journal v5 entry is missing its source identity"))?;
|
||||
let exact = self.version_id_exact.unwrap_or(false);
|
||||
let version_state = self
|
||||
.version_state
|
||||
.ok_or_else(|| Error::other("tier delete journal v5 entry is missing its version state"))?;
|
||||
validate_version_state(version_state, &self.version_id, exact)?;
|
||||
(
|
||||
Some(
|
||||
self.backend_identity
|
||||
.ok_or_else(|| Error::other("tier delete journal v5 entry is missing its backend identity"))?,
|
||||
),
|
||||
exact,
|
||||
version_state,
|
||||
state,
|
||||
Some(source),
|
||||
)
|
||||
}
|
||||
version => return Err(Error::other(format!("unsupported tier delete journal version {version}"))),
|
||||
@@ -206,8 +154,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
backend_identity,
|
||||
version_id_exact,
|
||||
version_state,
|
||||
state,
|
||||
source,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -255,20 +201,6 @@ pub(crate) fn tier_delete_journal_object_name(je: &Jentry) -> String {
|
||||
hasher.update([0]);
|
||||
hasher.update(b"exact-version-id");
|
||||
}
|
||||
if let Some(source) = &je.source {
|
||||
hasher.update([0]);
|
||||
hasher.update(source.bucket.as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(source.object.as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(source.version_id.as_deref().unwrap_or_default().as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(source.data_dir.as_deref().unwrap_or_default().as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(source.etag.as_deref().unwrap_or_default().as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(source.mod_time.as_deref().unwrap_or_default().as_bytes());
|
||||
}
|
||||
format!(
|
||||
"{TIER_DELETE_JOURNAL_PREFIX}{}.json",
|
||||
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
|
||||
@@ -314,66 +246,6 @@ where
|
||||
.map_err(std::io::Error::other)
|
||||
}
|
||||
|
||||
pub async fn commit_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = http::HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let mut committed = je.clone();
|
||||
committed.state = TierDeleteJournalState::Committed;
|
||||
persist_tier_delete_journal_entry(api, &committed).await
|
||||
}
|
||||
|
||||
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
pub async fn abort_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
let name = tier_delete_journal_object_name(je);
|
||||
let (data, metadata) = match config_boundary::read_config_with_metadata(api.clone(), &name, &ObjectOptions::default()).await {
|
||||
Ok(result) => result,
|
||||
Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => return Ok(()),
|
||||
Err(err) => return Err(std::io::Error::other(err)),
|
||||
};
|
||||
let current = decode_tier_delete_journal_entry(&data).map_err(std::io::Error::other)?;
|
||||
if current.state != TierDeleteJournalState::Prepared {
|
||||
return Ok(());
|
||||
}
|
||||
let etag = metadata
|
||||
.etag
|
||||
.ok_or_else(|| std::io::Error::other("prepared tier delete journal has no entity tag"))?;
|
||||
match config_boundary::delete_config_if_match(api, &name, &etag).await {
|
||||
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
|
||||
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal changed before abort",
|
||||
)),
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn enqueue_committed_tier_delete_journal_entry(je: &Jentry) -> std::io::Result<()> {
|
||||
let expiry_state = runtime_boundary::expiry_state_handle();
|
||||
expiry_state.write().await.enqueue_tier_journal_entry(je)
|
||||
}
|
||||
|
||||
pub async fn remove_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
@@ -392,13 +264,6 @@ where
|
||||
}
|
||||
|
||||
pub async fn process_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
if je.state == TierDeleteJournalState::Prepared {
|
||||
return reconcile_prepared_tier_delete_journal_entry(api, je).await;
|
||||
}
|
||||
process_committed_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
if je.version_state == rustfs_filemeta::TransitionVersionState::Unknown {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
@@ -431,87 +296,6 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
let (data, metadata) =
|
||||
config_boundary::read_config_with_metadata(api.clone(), &tier_delete_journal_object_name(je), &ObjectOptions::default())
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let current = decode_tier_delete_journal_entry(&data).map_err(std::io::Error::other)?;
|
||||
if current.state != TierDeleteJournalState::Prepared {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal changed before reconciliation",
|
||||
));
|
||||
}
|
||||
let Some(etag) = metadata.etag else {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"prepared tier delete journal has no entity tag",
|
||||
));
|
||||
};
|
||||
let source = je
|
||||
.source
|
||||
.as_ref()
|
||||
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "prepared tier delete journal has no source"))?;
|
||||
match api
|
||||
.get_object_info(&source.bucket, &source.object, &source.lookup_options())
|
||||
.await
|
||||
{
|
||||
Ok(info) if source.matches(&info) => {
|
||||
match config_boundary::delete_config_if_match(api, &tier_delete_journal_object_name(¤t), &etag).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal changed before abort",
|
||||
)),
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
Ok(_info) if source.has_stable_identity() => {
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
|
||||
}
|
||||
Ok(_) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal source identity is not sufficient to confirm deletion",
|
||||
)),
|
||||
Err(Error::ObjectNotFound(_, _)) | Err(Error::FileNotFound) | Err(Error::FileVersionNotFound) => {
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
|
||||
}
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn commit_prepared_tier_delete_journal_entry_if_current(
|
||||
api: Arc<ECStore>,
|
||||
mut committed: Jentry,
|
||||
etag: String,
|
||||
) -> std::io::Result<()> {
|
||||
committed.state = TierDeleteJournalState::Committed;
|
||||
let data = encode_tier_delete_journal_entry(&committed).map_err(std::io::Error::other)?;
|
||||
match config_boundary::save_config_with_opts(
|
||||
api.clone(),
|
||||
&tier_delete_journal_object_name(&committed),
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => process_committed_tier_delete_journal_entry(api, &committed).await,
|
||||
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal changed before commit",
|
||||
)),
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn recover_tier_delete_journal_entries(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
@@ -698,13 +482,10 @@ mod tests {
|
||||
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
|
||||
tier_delete_journal_object_name,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, TierDeleteJournalState, TierDeleteSourceIdentity};
|
||||
use crate::bucket::lifecycle::tier_sweeper::Jentry;
|
||||
use crate::error::Result;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn journal_entry() -> Jentry {
|
||||
Jentry {
|
||||
@@ -714,8 +495,6 @@ mod tests {
|
||||
backend_identity: Some([7; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
state: TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -734,55 +513,6 @@ mod tests {
|
||||
assert_eq!(decoded.version_state, je.version_state);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_transaction_roundtrips_prepared_source_identity() {
|
||||
let mut je = journal_entry();
|
||||
je.state = TierDeleteJournalState::Prepared;
|
||||
je.source = Some(TierDeleteSourceIdentity {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: Some("version".to_string()),
|
||||
versioned: true,
|
||||
version_suspended: false,
|
||||
data_dir: Some("data-dir".to_string()),
|
||||
etag: Some("etag".to_string()),
|
||||
mod_time: Some("mod-time".to_string()),
|
||||
});
|
||||
|
||||
let encoded = encode_tier_delete_journal_entry(&je).expect("prepared transaction should encode");
|
||||
let value: serde_json::Value = serde_json::from_slice(&encoded).expect("transaction should be JSON");
|
||||
assert_eq!(value["version"], serde_json::json!(5));
|
||||
assert_eq!(value["state"], serde_json::json!("Prepared"));
|
||||
assert!(value["source"].is_object());
|
||||
|
||||
let decoded = decode_tier_delete_journal_entry(&encoded).expect("prepared transaction should decode");
|
||||
assert_eq!(decoded.state, TierDeleteJournalState::Prepared);
|
||||
assert_eq!(decoded.source, je.source);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_source_identity_rejects_recreated_object() {
|
||||
let version_id = Uuid::from_u128(1);
|
||||
let data_dir = Uuid::from_u128(2);
|
||||
let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1);
|
||||
let info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(version_id),
|
||||
data_dir: Some(data_dir),
|
||||
mod_time: Some(mod_time),
|
||||
..Default::default()
|
||||
};
|
||||
let source = TierDeleteSourceIdentity::from_object_info("bucket", "object", &info, true, false);
|
||||
assert!(source.matches(&info));
|
||||
|
||||
let recreated = ObjectInfo {
|
||||
data_dir: Some(Uuid::from_u128(3)),
|
||||
..info
|
||||
};
|
||||
assert!(!source.matches(&recreated));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_roundtrips_exact_put_response_constraint() {
|
||||
let mut exact = journal_entry();
|
||||
|
||||
@@ -23,12 +23,10 @@ use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp;
|
||||
use crate::bucket::lifecycle::lifecycle::{self, ObjectOpts};
|
||||
use crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry;
|
||||
use crate::client::signer_error::error_chain_contains_signer_header_marker;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease};
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use crate::store::ECStore;
|
||||
use rustfs_utils::get_env_usize;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::any::Any;
|
||||
use std::collections::VecDeque;
|
||||
@@ -259,8 +257,6 @@ impl ObjSweeper {
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
|
||||
),
|
||||
version_state: self.transition_version_state,
|
||||
state: TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
});
|
||||
}
|
||||
None
|
||||
@@ -289,76 +285,6 @@ impl ObjSweeper {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub(crate) enum TierDeleteJournalState {
|
||||
Prepared,
|
||||
Committed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct TierDeleteSourceIdentity {
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) object: String,
|
||||
pub(crate) version_id: Option<String>,
|
||||
pub(crate) versioned: bool,
|
||||
pub(crate) version_suspended: bool,
|
||||
pub(crate) data_dir: Option<String>,
|
||||
pub(crate) etag: Option<String>,
|
||||
pub(crate) mod_time: Option<String>,
|
||||
}
|
||||
|
||||
impl TierDeleteSourceIdentity {
|
||||
pub(crate) fn from_object_info(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
info: &ObjectInfo,
|
||||
versioned: bool,
|
||||
version_suspended: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: info.version_id.map(|id| id.to_string()),
|
||||
versioned,
|
||||
version_suspended,
|
||||
data_dir: info.data_dir.map(|id| id.to_string()),
|
||||
etag: info.etag.clone(),
|
||||
mod_time: info.mod_time.map(|time| time.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn lookup_options(&self) -> crate::object_api::ObjectOptions {
|
||||
crate::object_api::ObjectOptions {
|
||||
version_id: self.version_id.clone(),
|
||||
versioned: self.versioned,
|
||||
version_suspended: self.version_suspended,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, info: &ObjectInfo) -> bool {
|
||||
if self.bucket != info.bucket {
|
||||
return false;
|
||||
}
|
||||
if let Some(version_id) = &self.version_id {
|
||||
return info.version_id.map(|id| id.to_string()).as_deref() == Some(version_id.as_str())
|
||||
&& self.data_dir == info.data_dir.map(|id| id.to_string());
|
||||
}
|
||||
if self.data_dir.is_some() {
|
||||
return self.data_dir == info.data_dir.map(|id| id.to_string());
|
||||
}
|
||||
self.etag.is_some()
|
||||
&& self.etag == info.etag
|
||||
&& self.mod_time.is_some()
|
||||
&& self.mod_time == info.mod_time.map(|time| time.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn has_stable_identity(&self) -> bool {
|
||||
self.version_id.is_some() || self.data_dir.is_some() || (self.etag.is_some() && self.mod_time.is_some())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(unused_assignments)]
|
||||
pub struct Jentry {
|
||||
@@ -368,8 +294,6 @@ pub struct Jentry {
|
||||
pub(crate) backend_identity: Option<TierDestinationId>,
|
||||
pub(crate) version_id_exact: bool,
|
||||
pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
|
||||
pub(crate) state: TierDeleteJournalState,
|
||||
pub(crate) source: Option<TierDeleteSourceIdentity>,
|
||||
}
|
||||
|
||||
impl ExpiryOp for Jentry {
|
||||
@@ -630,48 +554,9 @@ pub fn transitioned_force_delete_journal_entry(
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
|
||||
),
|
||||
version_state: transition_version_state,
|
||||
state: TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn attach_tier_delete_source(
|
||||
je: &mut Jentry,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
info: &ObjectInfo,
|
||||
versioned: bool,
|
||||
version_suspended: bool,
|
||||
) {
|
||||
je.state = TierDeleteJournalState::Prepared;
|
||||
je.source = Some(TierDeleteSourceIdentity::from_object_info(
|
||||
bucket,
|
||||
object,
|
||||
info,
|
||||
versioned,
|
||||
version_suspended,
|
||||
));
|
||||
}
|
||||
|
||||
pub(crate) fn transitioned_delete_journal_entry_for_source(
|
||||
version_id: Option<Uuid>,
|
||||
versioned: bool,
|
||||
suspended: bool,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
source: &ObjectInfo,
|
||||
) -> Option<Jentry> {
|
||||
let mut je = transitioned_delete_journal_entry(
|
||||
version_id,
|
||||
versioned,
|
||||
suspended,
|
||||
&source.transitioned_object,
|
||||
source.transition_version_state,
|
||||
)?;
|
||||
attach_tier_delete_source(&mut je, bucket, object, source, versioned, suspended);
|
||||
Some(je)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use crate::client::signer_error::invalid_utf8_header_error;
|
||||
|
||||
@@ -18,7 +18,7 @@ use super::versioning::VersioningApi;
|
||||
use super::{quota::BucketQuota, target::BucketTargets};
|
||||
use crate::bucket::replication::invalid_replication_config_status_field;
|
||||
use crate::bucket::utils::deserialize;
|
||||
use crate::config::com::{read_config, read_config_preserve_empty, save_config};
|
||||
use crate::config::com::{read_config, save_config};
|
||||
use crate::disk::BUCKET_META_PREFIX;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
@@ -37,7 +37,6 @@ use std::io::{Read, Write};
|
||||
use std::sync::Arc;
|
||||
use time::{Date, OffsetDateTime, PrimitiveDateTime, Time as CivilTime, UtcOffset};
|
||||
use tracing::error;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn read_msgp_str<R: Read>(rd: &mut R) -> Result<String> {
|
||||
let len = rmp::decode::read_str_len(rd)? as usize;
|
||||
@@ -227,7 +226,6 @@ fn write_bin_field<W: Write>(wr: &mut W, key: &str, val: &[u8]) -> Result<()> {
|
||||
}
|
||||
|
||||
pub const BUCKET_METADATA_FILE: &str = ".metadata.bin";
|
||||
pub const BUCKET_INCARNATION_FILE: &str = ".bucket-incarnation";
|
||||
pub const BUCKET_METADATA_FORMAT: u16 = 1;
|
||||
pub const BUCKET_METADATA_VERSION: u16 = 1;
|
||||
|
||||
@@ -279,8 +277,6 @@ pub struct BucketMetadata {
|
||||
pub name: String,
|
||||
pub created: OffsetDateTime,
|
||||
pub lock_enabled: bool, // While marked as unused, it may need to be retained
|
||||
pub bucket_incarnation_id: Uuid,
|
||||
pub(crate) bucket_incarnation_sidecar: bool,
|
||||
pub policy_config_json: Vec<u8>,
|
||||
pub notification_config_xml: Vec<u8>,
|
||||
pub lifecycle_config_xml: Vec<u8>,
|
||||
@@ -351,8 +347,6 @@ impl Default for BucketMetadata {
|
||||
name: Default::default(),
|
||||
created: OffsetDateTime::UNIX_EPOCH,
|
||||
lock_enabled: Default::default(),
|
||||
bucket_incarnation_id: Uuid::nil(),
|
||||
bucket_incarnation_sidecar: false,
|
||||
policy_config_json: Default::default(),
|
||||
notification_config_xml: Default::default(),
|
||||
lifecycle_config_xml: Default::default(),
|
||||
@@ -420,7 +414,6 @@ impl BucketMetadata {
|
||||
pub fn new(name: &str) -> Self {
|
||||
BucketMetadata {
|
||||
name: name.to_string(),
|
||||
bucket_incarnation_id: Uuid::new_v4(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -486,11 +479,6 @@ impl BucketMetadata {
|
||||
"Name" => self.name = read_msgp_str(rd)?,
|
||||
"Created" => self.created = read_msgp_time_value(rd)?,
|
||||
"LockEnabled" => self.lock_enabled = read_msgp_bool(rd)?,
|
||||
"BucketIncarnationID" => {
|
||||
let bytes = read_msgp_bin(rd)?;
|
||||
self.bucket_incarnation_id =
|
||||
Uuid::from_slice(&bytes).map_err(|err| Error::other(format!("invalid BucketIncarnationID: {err}")))?;
|
||||
}
|
||||
"PolicyConfigJSON" | "PolicyConfigJson" => self.policy_config_json = read_msgp_bin(rd)?,
|
||||
"NotificationConfigXML" | "NotificationConfigXml" => self.notification_config_xml = read_msgp_bin(rd)?,
|
||||
"LifecycleConfigXML" | "LifecycleConfigXml" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
|
||||
@@ -547,8 +535,8 @@ impl BucketMetadata {
|
||||
|
||||
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Map size: MinIO fields (25) + RustFS extensions (19)
|
||||
let map_len: u32 = 44;
|
||||
// Map size: MinIO fields (25) + RustFS extensions (18)
|
||||
let map_len: u32 = 43;
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -561,8 +549,6 @@ impl BucketMetadata {
|
||||
rmp::encode::write_str(wr, "LockEnabled")?;
|
||||
rmp::encode::write_bool(wr, self.lock_enabled)?;
|
||||
|
||||
write_bin_field(wr, "BucketIncarnationID", self.bucket_incarnation_id.as_bytes())?;
|
||||
|
||||
write_bin_field(wr, "PolicyConfigJSON", &self.policy_config_json)?;
|
||||
write_bin_field(wr, "NotificationConfigXML", &self.notification_config_xml)?;
|
||||
write_bin_field(wr, "LifecycleConfigXML", &self.lifecycle_config_xml)?;
|
||||
@@ -762,10 +748,6 @@ impl BucketMetadata {
|
||||
self.quota_config_updated_at = updated;
|
||||
}
|
||||
OBJECT_LOCK_CONFIG => {
|
||||
self.object_lock_config = None;
|
||||
if !data.is_empty() {
|
||||
self.lock_enabled = true;
|
||||
}
|
||||
self.object_lock_config_xml = data;
|
||||
self.object_lock_config_updated_at = updated;
|
||||
}
|
||||
@@ -1133,29 +1115,6 @@ impl BucketMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn load_bucket_incarnation(api: Arc<ECStore>, bucket: &str) -> Result<Option<Uuid>> {
|
||||
let path = format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}");
|
||||
let data = match read_config_preserve_empty(api, &path).await {
|
||||
Ok(data) => data,
|
||||
Err(Error::ConfigNotFound) => return Ok(None),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let incarnation =
|
||||
Uuid::from_slice(&data).map_err(|err| Error::other(format!("persisted bucket incarnation is invalid: {err}")))?;
|
||||
if incarnation.is_nil() {
|
||||
return Err(Error::other("persisted bucket incarnation is nil"));
|
||||
}
|
||||
Ok(Some(incarnation))
|
||||
}
|
||||
|
||||
pub(crate) async fn save_bucket_incarnation(api: Arc<ECStore>, bucket: &str, incarnation: Uuid) -> Result<()> {
|
||||
if incarnation.is_nil() {
|
||||
return Err(Error::other("cannot persist a nil bucket incarnation"));
|
||||
}
|
||||
let path = format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}");
|
||||
save_config(api, &path, incarnation.as_bytes().to_vec()).await
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<BucketMetadata> {
|
||||
load_bucket_metadata_parse(api, bucket, true).await
|
||||
}
|
||||
@@ -1183,23 +1142,6 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
|
||||
}
|
||||
};
|
||||
|
||||
let incarnation = load_bucket_incarnation(api, bucket).await?;
|
||||
if persisted {
|
||||
if let Some(incarnation) = incarnation {
|
||||
if !bm.bucket_incarnation_id.is_nil() && bm.bucket_incarnation_id != incarnation {
|
||||
return Err(Error::other("bucket incarnation sidecar does not match bucket metadata"));
|
||||
}
|
||||
bm.bucket_incarnation_id = incarnation;
|
||||
bm.bucket_incarnation_sidecar = true;
|
||||
} else if !bm.bucket_incarnation_id.is_nil() {
|
||||
return Err(Error::other(format!(
|
||||
"bucket incarnation sidecar is missing for new-format metadata: {bucket}"
|
||||
)));
|
||||
}
|
||||
} else if incarnation.is_some() {
|
||||
return Err(Error::other("bucket incarnation sidecar exists without bucket metadata"));
|
||||
}
|
||||
|
||||
bm.default_timestamps();
|
||||
|
||||
if parse {
|
||||
@@ -1267,10 +1209,6 @@ mod test {
|
||||
// Same 4-byte format|version header (1|1) and msgpack layout as MinIO.
|
||||
BucketMetadata::check_header(&blob).expect("valid .metadata.bin header");
|
||||
let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata");
|
||||
assert!(
|
||||
bm.bucket_incarnation_id.is_nil(),
|
||||
"legacy MinIO metadata has no RustFS bucket incarnation field"
|
||||
);
|
||||
|
||||
// Raw config fields survive the msgpack decode (PascalCase MinIO field names).
|
||||
assert_eq!(bm.name, "interop");
|
||||
@@ -1353,42 +1291,6 @@ mod test {
|
||||
let new = BucketMetadata::unmarshal(&buf).unwrap();
|
||||
|
||||
assert_eq!(bm.name, new.name);
|
||||
assert!(!bm.bucket_incarnation_id.is_nil());
|
||||
assert_eq!(bm.bucket_incarnation_id, new.bucket_incarnation_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_incarnation_msgpack_rejects_invalid_binary_length() {
|
||||
let mut fixture = Vec::new();
|
||||
rmp::encode::write_map_len(&mut fixture, 1).unwrap();
|
||||
rmp::encode::write_str(&mut fixture, "BucketIncarnationID").unwrap();
|
||||
rmp::encode::write_bin(&mut fixture, &[0_u8; 15]).unwrap();
|
||||
|
||||
let err = BucketMetadata::unmarshal(&fixture).expect_err("non-UUID incarnation bytes must fail closed");
|
||||
assert!(err.to_string().contains("invalid BucketIncarnationID"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_name_bucket_metadata_gets_a_new_incarnation() {
|
||||
let old = BucketMetadata::new("recreated");
|
||||
let new = BucketMetadata::new("recreated");
|
||||
|
||||
assert!(!old.bucket_incarnation_id.is_nil());
|
||||
assert!(!new.bucket_incarnation_id.is_nil());
|
||||
assert_ne!(old.bucket_incarnation_id, new.bucket_incarnation_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn site_replication_config_updates_cannot_replace_bucket_incarnation() {
|
||||
let mut metadata = BucketMetadata::new("site-replication-update");
|
||||
let incarnation = metadata.bucket_incarnation_id;
|
||||
|
||||
metadata
|
||||
.update_config(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec())
|
||||
.unwrap();
|
||||
metadata.update_config(OBJECT_LOCK_CONFIG, Vec::new()).unwrap();
|
||||
|
||||
assert_eq!(metadata.bucket_incarnation_id, incarnation);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,12 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::bucket::metadata_sys::{ObjectLockConfigState, get_object_lock_config, get_object_lock_config_state};
|
||||
use crate::bucket::metadata_sys::get_object_lock_config;
|
||||
use crate::bucket::object_lock::objectlock;
|
||||
use crate::error::{Error, Result, StorageError};
|
||||
use crate::object_api::ObjectInfo;
|
||||
use s3s::dto::{Date, DefaultRetention, ObjectLockConfiguration, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
|
||||
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
|
||||
use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
@@ -39,20 +37,6 @@ impl BucketObjectLockSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_recursive_force_delete_allowed_for_state(bucket: &str, state: &ObjectLockConfigState) -> Result<()> {
|
||||
match state {
|
||||
ObjectLockConfigState::ConfirmedAbsent => Ok(()),
|
||||
ObjectLockConfigState::Configured { .. } => Err(StorageError::InvalidArgument(
|
||||
bucket.to_string(),
|
||||
String::new(),
|
||||
"force-delete is forbidden on Object Locking enabled buckets".to_string(),
|
||||
)),
|
||||
ObjectLockConfigState::Fabricated => {
|
||||
Err(Error::other(format!("bucket Object Lock metadata is not authoritative: {bucket}")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a retention period is still active based on mode and retain_until_date
|
||||
pub fn is_retention_active(mode: &str, retain_until_date: Option<&s3s::dto::Date>) -> bool {
|
||||
if mode != ObjectLockRetentionMode::COMPLIANCE && mode != ObjectLockRetentionMode::GOVERNANCE {
|
||||
@@ -221,122 +205,71 @@ fn check_retention_blocks_deletion(
|
||||
None
|
||||
}
|
||||
|
||||
/// Check an object's lock metadata using an already resolved bucket Object
|
||||
/// Lock configuration. `None` means the configuration is confirmed absent.
|
||||
///
|
||||
/// # S3 Standard Behavior
|
||||
/// - COMPLIANCE mode: Cannot be deleted even with bypass header
|
||||
/// - GOVERNANCE mode: Can be deleted if bypass_governance is true (caller must verify s3:BypassGovernanceRetention permission)
|
||||
/// - Legal Hold: Cannot be bypassed regardless of mode
|
||||
pub(crate) fn check_object_lock_for_deletion_with_config(
|
||||
config: Option<&ObjectLockConfiguration>,
|
||||
obj_info: &ObjectInfo,
|
||||
bypass_governance: bool,
|
||||
) -> Result<Option<ObjectLockBlockReason>> {
|
||||
if obj_info.delete_marker {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if let Some(status) = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) {
|
||||
if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) {
|
||||
return Ok(Some(ObjectLockBlockReason::LegalHold));
|
||||
}
|
||||
if !status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::OFF) {
|
||||
return Err(Error::other("persisted object legal-hold metadata is invalid"));
|
||||
}
|
||||
}
|
||||
|
||||
let mode = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str());
|
||||
let retain_until = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
|
||||
let explicit_ret = match (mode, retain_until) {
|
||||
(None, None) => None,
|
||||
(Some(mode), Some(retain_until)) => {
|
||||
let mode =
|
||||
objectlock::parse_ret_mode(mode).ok_or_else(|| Error::other("persisted object retention mode is invalid"))?;
|
||||
let retain_until = OffsetDateTime::parse(retain_until, &time::format_description::well_known::Iso8601::DEFAULT)
|
||||
.map(Date::from)
|
||||
.map_err(|_| Error::other("persisted object retention date is invalid"))?;
|
||||
Some((mode, retain_until))
|
||||
}
|
||||
_ => return Err(Error::other("persisted object retention metadata is incomplete")),
|
||||
};
|
||||
|
||||
if let Some((mode, retain_until)) = &explicit_ret {
|
||||
let mode_str = mode.as_str();
|
||||
if is_retention_active(mode_str, Some(retain_until))
|
||||
&& let Some(reason) =
|
||||
check_retention_blocks_deletion(mode_str, Some(OffsetDateTime::from(retain_until.clone())), bypass_governance)
|
||||
{
|
||||
return Ok(Some(reason));
|
||||
}
|
||||
}
|
||||
|
||||
if explicit_ret.is_none()
|
||||
&& let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref())
|
||||
&& let Some(mode) = &default_retention.mode
|
||||
{
|
||||
let mode_str = mode.as_str();
|
||||
if mode_str == ObjectLockRetentionMode::COMPLIANCE || mode_str == ObjectLockRetentionMode::GOVERNANCE {
|
||||
// Calculate retention expiration date from object modification time
|
||||
let mod_time = obj_info
|
||||
.mod_time
|
||||
.ok_or_else(|| Error::other("persisted object modification time is missing"))?;
|
||||
let now = objectlock::utc_now_ntp();
|
||||
let retain_until = if let Some(days) = default_retention.days {
|
||||
mod_time.saturating_add(time::Duration::days(i64::from(days)))
|
||||
} else {
|
||||
let years = default_retention
|
||||
.years
|
||||
.ok_or_else(|| Error::other("persisted bucket Object Lock retention period is invalid"))?;
|
||||
add_years(mod_time, years)
|
||||
};
|
||||
|
||||
if retain_until.unix_timestamp() > now.unix_timestamp()
|
||||
&& let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance)
|
||||
{
|
||||
return Ok(Some(reason));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub(crate) fn check_object_lock_for_deletion_with_state(
|
||||
state: &ObjectLockConfigState,
|
||||
obj_info: &ObjectInfo,
|
||||
bypass_governance: bool,
|
||||
) -> Result<Option<ObjectLockBlockReason>> {
|
||||
match state {
|
||||
ObjectLockConfigState::Configured { config, .. } => {
|
||||
check_object_lock_for_deletion_with_config(Some(config), obj_info, bypass_governance)
|
||||
}
|
||||
ObjectLockConfigState::ConfirmedAbsent => check_object_lock_for_deletion_with_config(None, obj_info, bypass_governance),
|
||||
ObjectLockConfigState::Fabricated => Err(Error::other("bucket Object Lock metadata is not authoritative")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compatibility wrapper for callers that predate fallible metadata lookup.
|
||||
/// An authority/read/parse failure is represented as a blocking reason rather
|
||||
/// than the old fail-open `None` result.
|
||||
pub async fn check_object_lock_for_deletion(
|
||||
bucket: &str,
|
||||
obj_info: &ObjectInfo,
|
||||
bypass_governance: bool,
|
||||
) -> Option<ObjectLockBlockReason> {
|
||||
match get_object_lock_config_state(bucket)
|
||||
.await
|
||||
.and_then(|state| check_object_lock_for_deletion_with_state(&state, obj_info, bypass_governance))
|
||||
{
|
||||
Ok(reason) => reason,
|
||||
Err(_) => Some(ObjectLockBlockReason::LegalHold),
|
||||
if obj_info.delete_marker {
|
||||
return None;
|
||||
}
|
||||
|
||||
// 1. Check legal hold - cannot be bypassed (reuse has_legal_hold)
|
||||
if has_legal_hold(&obj_info.user_defined) {
|
||||
return Some(ObjectLockBlockReason::LegalHold);
|
||||
}
|
||||
|
||||
// 2. Check explicit retention
|
||||
let explicit_ret = objectlock::get_object_retention_meta(&obj_info.user_defined);
|
||||
if let Some(mode) = &explicit_ret.mode {
|
||||
let mode_str = mode.as_str();
|
||||
if is_retention_active(mode_str, explicit_ret.retain_until_date.as_ref())
|
||||
&& let Some(reason) = check_retention_blocks_deletion(
|
||||
mode_str,
|
||||
explicit_ret.retain_until_date.map(OffsetDateTime::from),
|
||||
bypass_governance,
|
||||
)
|
||||
{
|
||||
return Some(reason);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Check default retention only if no explicit retention is set
|
||||
if explicit_ret.mode.is_none()
|
||||
&& let Some(default_retention) = BucketObjectLockSys::get(bucket).await
|
||||
&& let Some(mode) = &default_retention.mode
|
||||
{
|
||||
let mode_str = mode.as_str();
|
||||
if mode_str == ObjectLockRetentionMode::COMPLIANCE || mode_str == ObjectLockRetentionMode::GOVERNANCE {
|
||||
// Calculate retention expiration date from object modification time
|
||||
if let Some(mod_time) = obj_info.mod_time {
|
||||
let now = objectlock::utc_now_ntp();
|
||||
let retain_until = if let Some(days) = default_retention.days {
|
||||
mod_time.saturating_add(time::Duration::days(days as i64))
|
||||
} else {
|
||||
let years = default_retention.years?;
|
||||
add_years(mod_time, years)
|
||||
};
|
||||
|
||||
if retain_until.unix_timestamp() > now.unix_timestamp()
|
||||
&& let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance)
|
||||
{
|
||||
return Some(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use s3s::dto::{ObjectLockEnabled, ObjectLockRule};
|
||||
use time::{Date, Month, PrimitiveDateTime, Time};
|
||||
|
||||
fn make_datetime(year: i32, month: u8, day: u8) -> OffsetDateTime {
|
||||
@@ -345,160 +278,6 @@ mod tests {
|
||||
PrimitiveDateTime::new(date, time).assume_utc()
|
||||
}
|
||||
|
||||
fn default_retention_config(mode: &'static str) -> ObjectLockConfiguration {
|
||||
ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(ObjectLockRule {
|
||||
default_retention: Some(DefaultRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static(mode)),
|
||||
days: Some(30),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_with_config_blocks_active_default_compliance_even_with_bypass() {
|
||||
let config = default_retention_config(ObjectLockRetentionMode::COMPLIANCE);
|
||||
let obj_info = ObjectInfo {
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = check_object_lock_for_deletion_with_config(Some(&config), &obj_info, true);
|
||||
|
||||
assert!(matches!(result, Ok(Some(ObjectLockBlockReason::Retention { .. }))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_with_config_allows_active_default_governance_with_bypass() {
|
||||
let config = default_retention_config(ObjectLockRetentionMode::GOVERNANCE);
|
||||
let obj_info = ObjectInfo {
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(matches!(
|
||||
check_object_lock_for_deletion_with_config(Some(&config), &obj_info, true),
|
||||
Ok(None)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_with_default_retention_rejects_missing_object_mod_time() {
|
||||
let config = default_retention_config(ObjectLockRetentionMode::COMPLIANCE);
|
||||
|
||||
let err = check_object_lock_for_deletion_with_config(Some(&config), &ObjectInfo::default(), false)
|
||||
.expect_err("default retention needs an authoritative object modification time");
|
||||
|
||||
assert!(err.to_string().contains("modification time"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_with_confirmed_absence_still_blocks_explicit_compliance() {
|
||||
let retain_until = OffsetDateTime::now_utc() + time::Duration::days(30);
|
||||
let mut user_defined = std::collections::HashMap::new();
|
||||
user_defined.insert("x-amz-object-lock-mode".to_string(), ObjectLockRetentionMode::COMPLIANCE.to_string());
|
||||
user_defined.insert(
|
||||
"x-amz-object-lock-retain-until-date".to_string(),
|
||||
retain_until
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.expect("retain-until date should format"),
|
||||
);
|
||||
let obj_info = ObjectInfo {
|
||||
user_defined: Arc::new(user_defined),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = check_object_lock_for_deletion_with_config(None, &obj_info, true);
|
||||
|
||||
assert!(matches!(result, Ok(Some(ObjectLockBlockReason::Retention { .. }))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_with_fabricated_bucket_metadata_fails_closed() {
|
||||
let err = check_object_lock_for_deletion_with_state(&ObjectLockConfigState::Fabricated, &ObjectInfo::default(), false)
|
||||
.expect_err("non-authoritative Object Lock metadata must block deletion");
|
||||
|
||||
assert!(err.to_string().contains("not authoritative"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recursive_force_delete_with_fabricated_bucket_metadata_fails_closed() {
|
||||
let err = ensure_recursive_force_delete_allowed_for_state("bucket", &ObjectLockConfigState::Fabricated)
|
||||
.expect_err("non-authoritative Object Lock metadata must block recursive deletion");
|
||||
|
||||
assert!(err.to_string().contains("not authoritative"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_rejects_incomplete_persisted_retention_metadata() {
|
||||
let mut user_defined = std::collections::HashMap::new();
|
||||
user_defined.insert(
|
||||
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
|
||||
ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
);
|
||||
let obj_info = ObjectInfo {
|
||||
user_defined: Arc::new(user_defined),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = check_object_lock_for_deletion_with_config(None, &obj_info, false)
|
||||
.expect_err("mode without retain-until date must fail closed");
|
||||
|
||||
assert!(err.to_string().contains("incomplete"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_rejects_each_malformed_persisted_retention_shape() {
|
||||
let valid_date = (OffsetDateTime::now_utc() + time::Duration::days(30))
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.expect("retain-until date should format");
|
||||
let cases = [
|
||||
("invalid mode", Some("INVALID"), Some(valid_date.as_str()), "retention mode"),
|
||||
(
|
||||
"invalid date",
|
||||
Some(ObjectLockRetentionMode::COMPLIANCE),
|
||||
Some("not-a-date"),
|
||||
"retention date",
|
||||
),
|
||||
("date only", None, Some(valid_date.as_str()), "incomplete"),
|
||||
];
|
||||
|
||||
for (case, mode, retain_until, expected) in cases {
|
||||
let mut user_defined = std::collections::HashMap::new();
|
||||
if let Some(mode) = mode {
|
||||
user_defined.insert(X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), mode.to_string());
|
||||
}
|
||||
if let Some(retain_until) = retain_until {
|
||||
user_defined.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), retain_until.to_string());
|
||||
}
|
||||
let obj_info = ObjectInfo {
|
||||
user_defined: Arc::new(user_defined),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = check_object_lock_for_deletion_with_config(None, &obj_info, false).expect_err(case);
|
||||
assert!(err.to_string().contains(expected), "unexpected {case} error: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deletion_rejects_invalid_persisted_legal_hold_metadata() {
|
||||
let mut user_defined = std::collections::HashMap::new();
|
||||
user_defined.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "INVALID".to_string());
|
||||
let obj_info = ObjectInfo {
|
||||
user_defined: Arc::new(user_defined),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = check_object_lock_for_deletion_with_config(None, &obj_info, false)
|
||||
.expect_err("invalid legal-hold value must fail closed");
|
||||
|
||||
assert!(err.to_string().contains("legal-hold"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_years_normal() {
|
||||
// Normal case: add 1 year to a regular date
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::{BucketQuota, QuotaCheckResult, QuotaError, QuotaOperation};
|
||||
use crate::bucket::metadata_sys::{BucketMetadataSys, update, update_if_incarnation};
|
||||
use crate::bucket::metadata_sys::{BucketMetadataSys, update};
|
||||
use crate::data_usage::get_bucket_usage_memory;
|
||||
use rustfs_common::metrics::Metric;
|
||||
use rustfs_config::QUOTA_CONFIG_FILE;
|
||||
@@ -145,35 +145,14 @@ impl QuotaChecker {
|
||||
}
|
||||
|
||||
pub async fn set_quota_config(&mut self, bucket: &str, quota: BucketQuota) -> Result<OffsetDateTime, QuotaError> {
|
||||
self.set_quota_config_for_incarnation(bucket, quota, None).await
|
||||
}
|
||||
|
||||
pub async fn set_quota_config_if_incarnation(
|
||||
&mut self,
|
||||
bucket: &str,
|
||||
quota: BucketQuota,
|
||||
expected_incarnation_id: uuid::Uuid,
|
||||
) -> Result<OffsetDateTime, QuotaError> {
|
||||
self.set_quota_config_for_incarnation(bucket, quota, Some(expected_incarnation_id))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn set_quota_config_for_incarnation(
|
||||
&mut self,
|
||||
bucket: &str,
|
||||
quota: BucketQuota,
|
||||
expected_incarnation_id: Option<uuid::Uuid>,
|
||||
) -> Result<OffsetDateTime, QuotaError> {
|
||||
let json_data = serde_json::to_vec("a).map_err(|e| QuotaError::InvalidConfig {
|
||||
reason: format!("Failed to serialize quota config: {}", e),
|
||||
})?;
|
||||
let start_time = Instant::now();
|
||||
|
||||
let updated_at = match expected_incarnation_id {
|
||||
Some(incarnation_id) => update_if_incarnation(bucket, QUOTA_CONFIG_FILE, json_data, incarnation_id).await,
|
||||
None => update(bucket, QUOTA_CONFIG_FILE, json_data).await,
|
||||
}
|
||||
.map_err(QuotaError::StorageError)?;
|
||||
let updated_at = update(bucket, QUOTA_CONFIG_FILE, json_data)
|
||||
.await
|
||||
.map_err(QuotaError::StorageError)?;
|
||||
|
||||
rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
|
||||
Ok(updated_at)
|
||||
@@ -198,36 +177,11 @@ impl QuotaChecker {
|
||||
}
|
||||
|
||||
pub async fn get_real_time_usage(&self, bucket: &str) -> Result<u64, QuotaError> {
|
||||
if let Some(usage) = get_bucket_usage_memory(bucket).await {
|
||||
return Ok(usage);
|
||||
}
|
||||
|
||||
// Degraded window (issue #5716): with no authoritative usage — most
|
||||
// prominently after upgrading from a pre-v2 release, whose legacy
|
||||
// `.usage.json` is demoted to non-authoritative until the scanner's
|
||||
// first complete cycle persists `.usage.v2.json` — failing closed
|
||||
// turned every write to a quota-enabled bucket into a retryable 503
|
||||
// for the whole window. Quota admission instead degrades to the last
|
||||
// persisted per-bucket size. That baseline is static between snapshot
|
||||
// loads (live writes do not advance it), so hard-quota enforcement is
|
||||
// advisory for the duration of the window: the overrun is bounded by
|
||||
// the writes issued before the next complete scanner cycle. Buckets
|
||||
// with no persisted baseline anywhere keep failing closed.
|
||||
let store = self.metadata_sys.read().await.object_store();
|
||||
// Box the fallback: it embeds the whole snapshot-load future, and every
|
||||
// object write nests a quota check several futures deep, so keeping it
|
||||
// inline would grow each write's state machine by the loader's full
|
||||
// size — the debug-build 2MiB worker-stack overflow class fixed for
|
||||
// bucket-config writes in #5648. The allocation only happens on the
|
||||
// degraded path; the authoritative fast path returns above.
|
||||
if let Some(baseline) = Box::pin(crate::data_usage::lookup_degraded_bucket_usage_baseline(store, bucket)).await {
|
||||
debug!(bucket, baseline, "Bucket quota admission using degraded persisted usage baseline");
|
||||
return Ok(baseline);
|
||||
}
|
||||
|
||||
Err(QuotaError::UsageUnavailable {
|
||||
bucket: bucket.to_string(),
|
||||
})
|
||||
get_bucket_usage_memory(bucket)
|
||||
.await
|
||||
.ok_or_else(|| QuotaError::UsageUnavailable {
|
||||
bucket: bucket.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,76 +211,6 @@ mod tests {
|
||||
assert_eq!(result.quota_limit, None);
|
||||
}
|
||||
|
||||
/// Regression (issue #5716): an upgrade from a pre-v2 release leaves only
|
||||
/// the legacy `.usage.json` snapshot, which has no completeness marker and
|
||||
/// is demoted to non-authoritative, and the scanner's first complete cycle
|
||||
/// can be a long way off. Quota admission must degrade to that persisted
|
||||
/// baseline instead of failing every write to a quota-enabled bucket with
|
||||
/// a retryable 503 for the whole window.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_falls_back_to_legacy_snapshot_baseline() {
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore.clone())));
|
||||
let checker = QuotaChecker::new(sys);
|
||||
let bucket = format!("quota-legacy-{}", Uuid::new_v4().simple());
|
||||
|
||||
let mut legacy = rustfs_data_usage::DataUsageInfo {
|
||||
last_update: Some(std::time::SystemTime::now()),
|
||||
buckets_count: 1,
|
||||
..Default::default()
|
||||
};
|
||||
legacy.buckets_usage.insert(
|
||||
bucket.clone(),
|
||||
rustfs_data_usage::BucketUsageInfo {
|
||||
size: 1_234,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
legacy.bucket_sizes.insert(bucket.clone(), 1_234);
|
||||
// usage_snapshot_complete stays false: pre-v2 snapshots do not carry
|
||||
// the field at all, so they always deserialize as incomplete.
|
||||
let legacy_path = format!("{}/{}", crate::disk::BUCKET_META_PREFIX, rustfs_data_usage::LEGACY_DATA_USAGE_OBJECT_NAME);
|
||||
crate::config::com::save_config(
|
||||
ecstore.clone(),
|
||||
&legacy_path,
|
||||
serde_json::to_vec(&legacy).expect("legacy snapshot should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("legacy snapshot fixture should be stored");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
|
||||
let usage = checker
|
||||
.get_real_time_usage(&bucket)
|
||||
.await
|
||||
.expect("quota admission must degrade to the persisted legacy baseline");
|
||||
assert_eq!(usage, 1_234);
|
||||
|
||||
// A bucket absent from every persisted snapshot still has no grounded
|
||||
// baseline and must keep failing closed.
|
||||
let unknown = format!("quota-unknown-{}", Uuid::new_v4().simple());
|
||||
assert!(matches!(
|
||||
checker.get_real_time_usage(&unknown).await,
|
||||
Err(QuotaError::UsageUnavailable { .. })
|
||||
));
|
||||
|
||||
// Deleting the bucket's usage from the backend must purge the
|
||||
// baseline: a recreated bucket may not inherit the dead incarnation's
|
||||
// size, so with no persisted trace left it fails closed again.
|
||||
crate::data_usage::remove_bucket_usage_from_backend(ecstore.clone(), &bucket)
|
||||
.await
|
||||
.expect("bucket usage removal should succeed");
|
||||
assert!(matches!(
|
||||
checker.get_real_time_usage(&bucket).await,
|
||||
Err(QuotaError::UsageUnavailable { .. })
|
||||
));
|
||||
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
|
||||
.await
|
||||
.expect("test usage cache cleanup should succeed");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_usage_rejects_an_unknown_mutation_baseline() {
|
||||
|
||||
@@ -100,41 +100,11 @@ paths.
|
||||
behind the ECStore replication facade; only `rustfs/src/app/storage_api.rs`
|
||||
may retain direct object/delete replication helper calls.
|
||||
|
||||
## Completion Criteria
|
||||
## First Code-Bearing Step
|
||||
|
||||
The split is complete when the "Current dependency to remove" column in the
|
||||
Required Contracts table above is empty: every row is either deleted because
|
||||
the dependency is gone, or reduced to "none". No other signal — file count,
|
||||
boundary count, line count — measures completion.
|
||||
|
||||
Target end state:
|
||||
|
||||
- `replication_pool.rs`, `replication_resyncer.rs`, and `replication_state.rs`
|
||||
move into `crates/replication` behind the contracts above;
|
||||
- the `*_boundary.rs` and `*_bridge.rs` micro-files dissolve naturally as the
|
||||
code they fence moves across the crate boundary. They are the mechanical
|
||||
seams of the migration ratchet — the architecture guard scripts anchor on
|
||||
their file names — so batch-merging them beforehand is explicitly rejected:
|
||||
it forces synchronized guard-script/mod/import churn with zero functional
|
||||
gain;
|
||||
- the only module that can retire early is `datatypes.rs`: delete it once its
|
||||
facade consumers import the resync status enums through `rustfs-replication`
|
||||
directly.
|
||||
|
||||
## Milestones
|
||||
|
||||
| Milestone | Scope | Status |
|
||||
|---|---|---|
|
||||
| M0 | Record the completion criteria and end state (this section). | Done |
|
||||
| M1 | Contract extraction: resync/queue/stats/object-decision/filemeta/storage wire contracts owned by `crates/replication`; ECStore imports concentrated in `*_boundary.rs`; event sink and runtime access behind local contracts. | Done — see Required Contracts |
|
||||
| M2 | Move resyncer pure decision logic (no IO) into `crates/replication`. | Pending; sequence after splitting the oversized resyncer/pool functions (`resync_bucket`, `replicate_all`, `start_mrf_processor`) so moves stay mechanical |
|
||||
| M3 | Move the worker runtime (`replication_pool.rs`, the IO paths of `replication_resyncer.rs`, `replication_state.rs`) once the contract traits are stable. Highest-risk step of the whole plan; do it last. | Pending |
|
||||
| M4 | Retire the boundary modules together with their guard-script entries; delete `datatypes.rs`. | Pending |
|
||||
|
||||
The original first code-bearing step (narrow `ReplicationEventSink` /
|
||||
`ReplicationRuntime` contracts) has landed — `replication_event_sink.rs`
|
||||
exists and runtime access goes through local boundary aliases — so new work
|
||||
starts from M2.
|
||||
Start with `ReplicationRuntime` or `ReplicationEventSink`. Both can be added as
|
||||
narrow internal contracts while keeping current queue, MRF, resync, and target
|
||||
behavior unchanged. Do not start with a crate move.
|
||||
|
||||
Current compatibility guard: `crates/ecstore/tests/replication_facade_compat_test.rs`
|
||||
keeps the ECStore replication facade types covered while architecture rules
|
||||
|
||||
@@ -45,12 +45,12 @@ mod runtime_boundary;
|
||||
|
||||
pub use datatypes::ResyncStatusType;
|
||||
pub use replication_config_boundary::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
|
||||
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, invalid_replication_config_status_field,
|
||||
replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
|
||||
validate_replication_config_target_arns,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use replication_filemeta_boundary::ReplicateTargetDecision;
|
||||
pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
|
||||
pub use replication_filemeta_boundary::{
|
||||
MrfOpKind, MrfReplicateEntry, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState,
|
||||
@@ -70,17 +70,16 @@ pub use replication_object_decision_boundary::{
|
||||
should_use_existing_delete_replication_source,
|
||||
};
|
||||
pub use replication_pool::{
|
||||
DurableMrfBacklog, DynReplicationPool, ReplicationPoolTrait, commit_force_delete_intent, complete_force_delete_intent,
|
||||
get_global_replication_pool, get_global_replication_stats, init_background_replication, persist_force_delete_intent,
|
||||
read_durable_mrf_backlog, resync_start_conflict_id,
|
||||
DurableMrfBacklog, DynReplicationPool, ReplicationPoolTrait, get_global_replication_pool, get_global_replication_stats,
|
||||
init_background_replication, read_durable_mrf_backlog, resync_start_conflict_id,
|
||||
};
|
||||
pub use replication_queue_boundary::{
|
||||
DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation,
|
||||
ReplicationPriority, ReplicationQueueAdmission,
|
||||
DeletedObjectReplicationInfo, ReplicationHealQueueResult, ReplicationOperation, ReplicationPriority,
|
||||
ReplicationQueueAdmission,
|
||||
};
|
||||
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
||||
pub use replication_scanner_bridge::ReplicationScannerBridge;
|
||||
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
||||
pub use replication_stats_boundary::{BucketReplicationStats, BucketStats};
|
||||
pub use replication_stats_boundary::BucketStats;
|
||||
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||
|
||||
@@ -13,9 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
|
||||
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
unsupported_replication_config_field, validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::replication_error_boundary::Result;
|
||||
use super::replication_storage_boundary::{HTTPPreconditions, ObjectInfo, ObjectOptions, ReplicationObjectIO};
|
||||
use super::replication_storage_boundary::ReplicationObjectIO;
|
||||
use crate::config::{com, storageclass};
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -30,98 +30,10 @@ impl ReplicationConfigStore {
|
||||
com::read_config(api, file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_limited<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::read_config_limited(api, file, max_bytes).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::read_config_no_lock(api, file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_no_lock_with_metadata<S>(api: Arc<S>, file: &str) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::read_config_with_metadata(
|
||||
api,
|
||||
file,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_no_lock_with_metadata_preserve_empty<S>(api: Arc<S>, file: &str) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::read_config_no_lock_preserve_empty_with_metadata(api, file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::save_config(api, file, data).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_no_lock<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::save_config_no_lock(api, file, data).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_conditional<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
data: Vec<u8>,
|
||||
http_preconditions: HTTPPreconditions,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::save_config_with_opts_quiet(
|
||||
api,
|
||||
file,
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(http_preconditions),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_conditional_no_lock<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
data: Vec<u8>,
|
||||
http_preconditions: HTTPPreconditions,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::save_config_with_opts_quiet(
|
||||
api,
|
||||
file,
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
http_preconditions: Some(http_preconditions),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
||||
pub(crate) use rustfs_replication::{
|
||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos,
|
||||
ReplicatedTargetInfo, ReplicationAction, ReplicationWorkerOperation, ResyncDecision, get_replication_state,
|
||||
parse_replicate_decision, replicate_decision_for_admitted_targets, target_reset_header, version_purge_statuses_map,
|
||||
parse_replicate_decision, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
pub use rustfs_replication::{
|
||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState, ReplicationStatusType, ReplicationType,
|
||||
@@ -52,8 +52,6 @@ pub(crate) fn replication_state_from_filemeta(state: &rustfs_filemeta::Replicati
|
||||
.map(|(arn, status)| (arn.clone(), version_purge_status_from_filemeta(status.clone())))
|
||||
.collect(),
|
||||
reset_statuses_map: state.reset_statuses_map.clone(),
|
||||
target_delete_marker_version_ids: state.target_delete_marker_version_ids.clone(),
|
||||
target_delete_marker_version_ids_corrupt: state.target_delete_marker_version_ids_corrupt,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,120 +83,5 @@ pub fn replication_state_to_filemeta(state: &ReplicationState) -> rustfs_filemet
|
||||
.map(|(arn, status)| (arn.clone(), version_purge_status_to_filemeta(status.clone())))
|
||||
.collect(),
|
||||
reset_statuses_map: state.reset_statuses_map.clone(),
|
||||
target_delete_marker_version_ids: state.target_delete_marker_version_ids.clone(),
|
||||
target_delete_marker_version_ids_corrupt: state.target_delete_marker_version_ids_corrupt,
|
||||
}
|
||||
}
|
||||
|
||||
// Reconciliation tests for the deliberately duplicated wire types.
|
||||
//
|
||||
// `rustfs-filemeta` (xl.meta disk format) and `rustfs-replication` (MRF/resync
|
||||
// persistence format) each own a copy of `ReplicationStatusType`,
|
||||
// `VersionPurgeStatusType` and `ReplicationState`; the conversions above hop
|
||||
// between them via `as_str()`, whose `From<&str>` impls fall back to `Empty`
|
||||
// on any unknown token. That fallback silently degrades data the moment one
|
||||
// side gains a variant the other lacks, so these tests pin the two sides
|
||||
// together:
|
||||
//
|
||||
// - the `match` statements are exhaustive with no `_` arm — adding a variant
|
||||
// on either side fails compilation here until the mapping is reconsidered;
|
||||
// - the round-trips assert the string token survives both directions — a
|
||||
// variant whose token the other side does not recognize fails the assert
|
||||
// instead of quietly becoming `Empty`.
|
||||
//
|
||||
// Struct-shaped drift on `ReplicationState` is already compile-guarded by the
|
||||
// exhaustive struct literals in the two conversion functions above; the
|
||||
// round-trip test below additionally pins value fidelity for every field.
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[test]
|
||||
fn replication_status_variants_round_trip_across_boundary() {
|
||||
use rustfs_replication::ReplicationStatusType as Repl;
|
||||
|
||||
let all = [
|
||||
Repl::Pending,
|
||||
Repl::Completed,
|
||||
Repl::CompletedLegacy,
|
||||
Repl::Failed,
|
||||
Repl::Replica,
|
||||
Repl::Empty,
|
||||
];
|
||||
for status in all {
|
||||
// Exhaustive on the replication side: a new variant breaks this match.
|
||||
match status {
|
||||
Repl::Pending | Repl::Completed | Repl::CompletedLegacy | Repl::Failed | Repl::Replica | Repl::Empty => {}
|
||||
}
|
||||
let filemeta = replication_status_to_filemeta(status.clone());
|
||||
assert_eq!(
|
||||
filemeta.as_str(),
|
||||
status.as_str(),
|
||||
"replication->filemeta conversion must not degrade {status:?} (unknown tokens fall back to Empty)"
|
||||
);
|
||||
assert_eq!(replication_status_from_filemeta(filemeta), status);
|
||||
}
|
||||
|
||||
// Exhaustive on the filemeta side: a new variant breaks this match.
|
||||
fn _filemeta_side_is_covered(status: rustfs_filemeta::ReplicationStatusType) {
|
||||
use rustfs_filemeta::ReplicationStatusType as Meta;
|
||||
match status {
|
||||
Meta::Pending | Meta::Completed | Meta::CompletedLegacy | Meta::Failed | Meta::Replica | Meta::Empty => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_purge_status_variants_round_trip_across_boundary() {
|
||||
use rustfs_replication::VersionPurgeStatusType as Repl;
|
||||
|
||||
let all = [Repl::Pending, Repl::Complete, Repl::Failed, Repl::Empty];
|
||||
for status in all {
|
||||
// Exhaustive on the replication side: a new variant breaks this match.
|
||||
match status {
|
||||
Repl::Pending | Repl::Complete | Repl::Failed | Repl::Empty => {}
|
||||
}
|
||||
let filemeta = version_purge_status_to_filemeta(status.clone());
|
||||
assert_eq!(
|
||||
filemeta.as_str(),
|
||||
status.as_str(),
|
||||
"replication->filemeta conversion must not degrade {status:?} (unknown tokens fall back to Empty)"
|
||||
);
|
||||
assert_eq!(version_purge_status_from_filemeta(filemeta), status);
|
||||
}
|
||||
|
||||
// Exhaustive on the filemeta side: a new variant breaks this match.
|
||||
fn _filemeta_side_is_covered(status: rustfs_filemeta::VersionPurgeStatusType) {
|
||||
use rustfs_filemeta::VersionPurgeStatusType as Meta;
|
||||
match status {
|
||||
Meta::Pending | Meta::Complete | Meta::Failed | Meta::Empty => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_state_round_trips_every_field_across_boundary() {
|
||||
let timestamp = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid timestamp");
|
||||
let state = ReplicationState {
|
||||
replica_timestamp: Some(timestamp),
|
||||
replica_status: ReplicationStatusType::Replica,
|
||||
delete_marker: true,
|
||||
replication_timestamp: Some(timestamp),
|
||||
replication_status_internal: Some("arn:a=PENDING;".to_string()),
|
||||
version_purge_status_internal: Some("arn:a=FAILED;".to_string()),
|
||||
replicate_decision_str: "arn:a=true;false;;".to_string(),
|
||||
targets: HashMap::from([
|
||||
("arn:a".to_string(), ReplicationStatusType::Completed),
|
||||
("arn:b".to_string(), ReplicationStatusType::Failed),
|
||||
]),
|
||||
purge_targets: HashMap::from([("arn:a".to_string(), VersionPurgeStatusType::Pending)]),
|
||||
reset_statuses_map: HashMap::from([("reset-arn:a".to_string(), "reset-id;ts".to_string())]),
|
||||
target_delete_marker_version_ids: HashMap::from([("arn:a".to_string(), "version-1".to_string())]),
|
||||
target_delete_marker_version_ids_corrupt: true,
|
||||
};
|
||||
|
||||
let round_tripped = replication_state_from_filemeta(&replication_state_to_filemeta(&state));
|
||||
assert_eq!(round_tripped, state);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,9 +32,6 @@ pub(crate) struct ReplicationMetadataStore;
|
||||
|
||||
impl ReplicationMetadataStore {
|
||||
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
|
||||
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
|
||||
|
||||
pub(crate) async fn replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
metadata_sys::get_replication_config(bucket).await
|
||||
@@ -112,17 +109,5 @@ mod tests {
|
||||
"buckets/bucket-a/.replication/resync.bin"
|
||||
);
|
||||
assert_eq!(ReplicationMetadataStore::MRF_REPLICATION_FILE, "config/replication/mrf.bin");
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
"config/replication/mrf.bin.recovery"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
|
||||
"config/replication/force-delete.bin"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_TRANSACTION_LOCK,
|
||||
"config/replication/force-delete.bin.transaction"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use super::replication_error_boundary::Result;
|
||||
use super::replication_filemeta_boundary::{ReplicateDecision, ReplicatedTargetInfo, ReplicationStatusType, ReplicationType};
|
||||
use super::replication_filemeta_boundary::{ReplicateDecision, ReplicationStatusType, ReplicationType};
|
||||
use super::replication_metadata_boundary::ReplicationInstanceContext;
|
||||
use super::replication_object_config::{
|
||||
DeleteReplicationConfigSnapshot, check_replicate_delete, check_replicate_delete_strict, check_replicate_delete_with_snapshot,
|
||||
@@ -89,13 +89,6 @@ impl ReplicationObjectBridge {
|
||||
snapshot.has_active_rule(object)
|
||||
}
|
||||
|
||||
pub fn force_delete_target_set(
|
||||
snapshot: &DeleteReplicationConfigSnapshot,
|
||||
prefix: &str,
|
||||
) -> Option<(Vec<String>, time::OffsetDateTime)> {
|
||||
snapshot.force_delete_target_set(prefix)
|
||||
}
|
||||
|
||||
pub fn check_delete_with_snapshot(
|
||||
object: &ObjectToDelete,
|
||||
source: &ObjectInfo,
|
||||
@@ -119,31 +112,6 @@ impl ReplicationObjectBridge {
|
||||
schedule_replication_delete(delete_object).await;
|
||||
}
|
||||
|
||||
pub async fn schedule_deletes(delete_objects: &[DeletedObjectReplicationInfo]) {
|
||||
if let Some(pool) = super::runtime_boundary::replication_pool() {
|
||||
let _ = pool.queue_replica_delete_batch(delete_objects).await;
|
||||
}
|
||||
|
||||
if let Some(stats) = super::runtime_boundary::replication_stats() {
|
||||
for delete_object in delete_objects {
|
||||
if let Some(rs) = &delete_object.delete_object.replication_state {
|
||||
for k in rs.targets.keys() {
|
||||
let ri = ReplicatedTargetInfo {
|
||||
arn: k.clone(),
|
||||
size: 0,
|
||||
duration: std::time::Duration::default(),
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
stats
|
||||
.update(&delete_object.bucket, &ri, ReplicationStatusType::Pending, ReplicationStatusType::Empty)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn schedule_storage_delete(delete_object: DeletedObject, bucket: String, event_type: String) {
|
||||
Self::schedule_delete(DeletedObjectReplicationInfo {
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
@@ -153,19 +121,6 @@ impl ReplicationObjectBridge {
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
pub async fn schedule_storage_deletes(delete_objects: Vec<DeletedObject>, bucket: String, event_type: String) {
|
||||
let delete_objects = delete_objects
|
||||
.into_iter()
|
||||
.map(|delete_object| DeletedObjectReplicationInfo {
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
bucket: bucket.clone(),
|
||||
event_type: event_type.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
Self::schedule_deletes(&delete_objects).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -18,7 +18,6 @@ use crate::bucket::metadata::BucketMetadata;
|
||||
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
|
||||
use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::error;
|
||||
|
||||
use super::replication_config_boundary::{
|
||||
@@ -84,15 +83,6 @@ impl DeleteReplicationConfigSnapshot {
|
||||
.and_then(|metadata| metadata.replication_config.as_ref())
|
||||
}
|
||||
|
||||
pub(crate) fn force_delete_target_set(&self, prefix: &str) -> Option<(Vec<String>, OffsetDateTime)> {
|
||||
self.metadata.as_ref().and_then(|metadata| {
|
||||
metadata
|
||||
.replication_config
|
||||
.as_ref()
|
||||
.map(|config| (config.filter_force_delete_target_arns(prefix), metadata.replication_config_updated_at))
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn has_active_rule(&self, object: &str) -> bool {
|
||||
self.replication_config()
|
||||
.is_some_and(|config| config.has_active_rules(object, true))
|
||||
@@ -566,7 +556,7 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic
|
||||
let mut sopts = opts.clone();
|
||||
sopts.target_arn = arn.clone();
|
||||
|
||||
let replicate = cfg.replicate(&sopts) && mopts.metadata_target_is_eligible(&arn);
|
||||
let replicate = cfg.replicate(&sopts);
|
||||
let synchronous = if let Some(cli) = cli { cli.replicate_sync } else { false };
|
||||
|
||||
dsc.set(ReplicateTargetDecision::new(arn, replicate, synchronous));
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::{
|
||||
DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation,
|
||||
ReplicationPriority, ReplicationQueueAdmission,
|
||||
DeletedObjectReplicationInfo, ReplicationHealQueueResult, ReplicationOperation, ReplicationPriority,
|
||||
ReplicationQueueAdmission,
|
||||
};
|
||||
pub(crate) use rustfs_replication::{
|
||||
LARGE_WORKER_COUNT, ReplicationBackpressureRecommendation, ReplicationBackpressureState, ReplicationHealQueueAction,
|
||||
|
||||
@@ -23,7 +23,6 @@ pub(crate) use rustfs_replication::{
|
||||
|
||||
pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT;
|
||||
pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION;
|
||||
pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES;
|
||||
pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX;
|
||||
pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT;
|
||||
pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION;
|
||||
|
||||
@@ -19,7 +19,7 @@ use super::replication_error_boundary::{Result, is_err_object_not_found, is_err_
|
||||
use super::replication_event_sink::{EventArgs, send_event, send_local_event};
|
||||
use super::replication_filemeta_boundary::{
|
||||
NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos,
|
||||
ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType, VersionPurgeStatusType,
|
||||
ReplicatedTargetInfo, ReplicationAction, ReplicationStatusType, ReplicationType, VersionPurgeStatusType,
|
||||
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_lock_boundary::ReplicationLockTiming;
|
||||
@@ -96,6 +96,7 @@ const EVENT_REPLICATION_FORCE_DELETE_SKIPPED: &str = "replication_force_delete_s
|
||||
const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
|
||||
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
|
||||
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
|
||||
const ERR_REPLICATION_METADATA_COPY_UNSUPPORTED: &str = "metadata-only replication is not implemented";
|
||||
const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
|
||||
"dispatch failure",
|
||||
"timeouterror",
|
||||
@@ -209,7 +210,7 @@ fn is_replication_target_offline_error(err: &(impl Display + ?Sized)) -> bool {
|
||||
.any(|marker| message.contains(marker))
|
||||
}
|
||||
|
||||
async fn mark_replication_target_offline_if_needed(target_client: &Arc<TargetClient>, err: &(impl Display + ?Sized)) {
|
||||
async fn mark_replication_target_offline_if_needed(target_client: &TargetClient, err: &(impl Display + ?Sized)) {
|
||||
if is_replication_target_offline_error(err) {
|
||||
ReplicationTargetStore::mark_target_offline(target_client).await;
|
||||
}
|
||||
@@ -792,7 +793,6 @@ impl ReplicationResyncer {
|
||||
let storage = storage.clone();
|
||||
let results_tx = results_tx.clone();
|
||||
let bucket_name = opts.bucket.clone();
|
||||
let target_arn = opts.arn.clone();
|
||||
|
||||
let f = tokio::spawn(async move {
|
||||
while let Some(mut roi) = rx.recv().await {
|
||||
@@ -820,7 +820,6 @@ impl ReplicationResyncer {
|
||||
bucket: roi.bucket.clone(),
|
||||
event_type: REPLICATE_EXISTING_DELETE.to_string(),
|
||||
op_type: ReplicationType::ExistingObject,
|
||||
target_arn: target_arn.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
replicate_delete(doi, storage.clone()).await;
|
||||
@@ -1203,19 +1202,12 @@ pub(crate) async fn save_resync_status<S: ReplicationObjectIO>(
|
||||
}
|
||||
|
||||
pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicationInfo, storage: Arc<S>) {
|
||||
let _ = replicate_delete_with_outcome(dobj, storage).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
dobj: DeletedObjectReplicationInfo,
|
||||
storage: Arc<S>,
|
||||
) -> bool {
|
||||
if dobj.delete_object.force_delete {
|
||||
return replicate_force_delete_to_targets(&dobj, storage).await;
|
||||
replicate_force_delete_to_targets(&dobj, storage).await;
|
||||
return;
|
||||
}
|
||||
|
||||
let bucket = dobj.bucket.clone();
|
||||
let mut source_state_verified = true;
|
||||
let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
|
||||
Some(version_id.to_owned())
|
||||
} else {
|
||||
@@ -1252,7 +1244,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
reason = "source_not_delete_marker",
|
||||
"Skipping stale delete-marker replication"
|
||||
);
|
||||
return true;
|
||||
return;
|
||||
}
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
|
||||
debug!(
|
||||
@@ -1265,10 +1257,9 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
reason = "source_version_missing",
|
||||
"Skipping stale delete-marker replication"
|
||||
);
|
||||
return true;
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
source_state_verified = false;
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -1318,7 +1309,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
};
|
||||
let ns_lock = match storage
|
||||
@@ -1350,7 +1341,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1380,7 +1371,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1393,12 +1384,6 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
let mut join_set = JoinSet::new();
|
||||
|
||||
// Process each target
|
||||
let target_arns = dobj.admitted_target_arns();
|
||||
let expected_targets = dsc
|
||||
.targets_map
|
||||
.values()
|
||||
.filter(|target| target.replicate && (target_arns.is_empty() || target_arns.iter().any(|arn| arn == &target.arn)))
|
||||
.count();
|
||||
for tgt_entry in dsc.targets_map.values() {
|
||||
// Skip targets that should not be replicated
|
||||
if !tgt_entry.replicate {
|
||||
@@ -1406,7 +1391,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
}
|
||||
|
||||
// If dobj.TargetArn is not empty string, this is a case of specific target being re-synced.
|
||||
if !target_arns.is_empty() && !target_arns.iter().any(|arn| arn == &tgt_entry.arn) {
|
||||
if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1478,8 +1463,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
|
||||
let is_version_purge = is_version_delete_replication(&dobj.delete_object);
|
||||
|
||||
let requires_delayed_purge = should_retry_delete_marker_purge(&dobj.delete_object);
|
||||
if requires_delayed_purge {
|
||||
if should_retry_delete_marker_purge(&dobj.delete_object) {
|
||||
let bucket_clone = bucket.clone();
|
||||
let dobj_clone = dobj.clone();
|
||||
let dsc_clone = dsc.clone();
|
||||
@@ -1550,7 +1534,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
EventName::ObjectReplicationFailed.to_string()
|
||||
};
|
||||
|
||||
let state_persisted = match storage
|
||||
match storage
|
||||
.delete_object(
|
||||
&bucket,
|
||||
&dobj.delete_object.object_name,
|
||||
@@ -1572,7 +1556,6 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
object,
|
||||
..Default::default()
|
||||
});
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
@@ -1598,16 +1581,8 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
expected_targets > 0
|
||||
&& rinfos.targets.len() == expected_targets
|
||||
&& state_persisted
|
||||
&& source_state_verified
|
||||
&& !requires_delayed_purge
|
||||
&& replication_status == ReplicationStatusType::Completed
|
||||
}
|
||||
}
|
||||
|
||||
async fn source_delete_marker_missing<S: EcstoreObjectOperations>(
|
||||
@@ -1634,29 +1609,6 @@ async fn source_delete_marker_missing<S: EcstoreObjectOperations>(
|
||||
}
|
||||
}
|
||||
|
||||
/// Which version a delete-marker purge should address on one target.
|
||||
///
|
||||
/// `None` means do not purge at all: the recorded mapping disagreed across the
|
||||
/// dual internal prefixes, and guessing an id could destroy a live version on
|
||||
/// the target. `Some(id)` is the exact version the target reported when it
|
||||
/// accepted the marker; falling back to a source-derived id is only correct
|
||||
/// when the target mirrors source version ids, which a generic S3 target does
|
||||
/// not.
|
||||
fn delete_marker_purge_version_id(
|
||||
state: Option<&ReplicationState>,
|
||||
arn: &str,
|
||||
delete_marker_version_id: Uuid,
|
||||
) -> Option<Option<String>> {
|
||||
if state.is_some_and(|state| state.target_delete_marker_version_ids_corrupt) {
|
||||
return None;
|
||||
}
|
||||
let recorded = state.and_then(|state| state.target_delete_marker_version_ids.get(arn).cloned());
|
||||
Some(match recorded {
|
||||
Some(version_id) => Some(version_id),
|
||||
None => target_delete_version_id(delete_marker_version_id, true),
|
||||
})
|
||||
}
|
||||
|
||||
async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo, dsc: &ReplicateDecision) {
|
||||
let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
|
||||
return;
|
||||
@@ -1666,100 +1618,75 @@ async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedOb
|
||||
if !tgt_entry.replicate {
|
||||
continue;
|
||||
}
|
||||
let target_arns = dobj.admitted_target_arns();
|
||||
if !target_arns.is_empty() && !target_arns.iter().any(|arn| arn == &tgt_entry.arn) {
|
||||
if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn {
|
||||
continue;
|
||||
}
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &tgt_entry.arn).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let Some(purge_version_id) = delete_marker_purge_version_id(
|
||||
dobj.delete_object.replication_state.as_ref(),
|
||||
&tgt_entry.arn,
|
||||
delete_marker_version_id,
|
||||
) else {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
arn = tgt_entry.arn,
|
||||
"Skipping delete-marker purge: recorded target version metadata is inconsistent"
|
||||
);
|
||||
continue;
|
||||
};
|
||||
|
||||
let _ = tgt_client
|
||||
.remove_object(
|
||||
&tgt_client.bucket,
|
||||
&dobj.delete_object.object_name,
|
||||
purge_version_id,
|
||||
target_delete_version_id(delete_marker_version_id, true),
|
||||
replication_delete_marker_purge_remove_options(dobj.delete_object.delete_marker_mtime),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) -> bool {
|
||||
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
|
||||
let bucket = &dobj.bucket;
|
||||
let object_name = &dobj.delete_object.object_name;
|
||||
let admitted_target_arns = dobj.admitted_target_arns();
|
||||
|
||||
let legacy_target_arns = if admitted_target_arns.is_empty() {
|
||||
match get_replication_config(bucket).await {
|
||||
Ok(Some(config)) => config.filter_target_arns(&ObjectOpts {
|
||||
name: object_name.clone(),
|
||||
let rcfg = match get_replication_config(bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
reason = "replication_config_missing",
|
||||
"Skipping replication force-delete because replication config is missing"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
Ok(None) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
reason = "replication_config_missing",
|
||||
"Skipping replication force-delete because replication config is missing"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
Vec::new()
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
reason = "replication_config_lookup_failed",
|
||||
"Skipping replication force-delete because replication config lookup failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
Vec::new()
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
reason = "replication_config_lookup_failed",
|
||||
"Skipping replication force-delete because replication config lookup failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
let ns_lock = match storage
|
||||
@@ -1789,7 +1716,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1817,25 +1744,23 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let tgt_arns = if admitted_target_arns.is_empty() {
|
||||
legacy_target_arns
|
||||
let tgt_arns = if !dobj.target_arn.is_empty() {
|
||||
vec![dobj.target_arn.clone()]
|
||||
} else {
|
||||
admitted_target_arns
|
||||
rcfg.filter_target_arns(&ObjectOpts {
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
};
|
||||
if tgt_arns.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut join_set = JoinSet::new();
|
||||
let mut all_succeeded = true;
|
||||
|
||||
for arn in tgt_arns {
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
|
||||
all_succeeded = false;
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -1885,7 +1810,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) = tgt_client
|
||||
@@ -1914,49 +1839,24 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
true
|
||||
});
|
||||
}
|
||||
|
||||
while let Some(result) = join_set.join_next().await {
|
||||
match result {
|
||||
Ok(success) => all_succeeded &= success,
|
||||
Err(error) => {
|
||||
all_succeeded = false;
|
||||
error!(
|
||||
event = EVENT_RESYNC_TASK_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
object = %object_name,
|
||||
operation = "force_delete",
|
||||
error = %error,
|
||||
"Replication resync task failed"
|
||||
);
|
||||
}
|
||||
if let Err(e) = result {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TASK_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
object = %object_name,
|
||||
operation = "force_delete",
|
||||
error = %e,
|
||||
"Replication resync task failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if all_succeeded
|
||||
&& let Some(operation_id) = dobj.delete_object.force_delete_id
|
||||
&& let Err(error) = super::replication_pool::complete_force_delete_intent(storage, operation_id).await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
object = %object_name,
|
||||
operation_id = %operation_id,
|
||||
error = %error,
|
||||
"Force-delete replication completed but durable intent cleanup failed"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
all_succeeded
|
||||
}
|
||||
|
||||
fn target_delete_version_id(version_id: Uuid, version_purge: bool) -> Option<String> {
|
||||
@@ -2046,24 +1946,16 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(assigned_version_id) => {
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
bucket = tgt_client.bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = ?version_id,
|
||||
assigned_version_id = ?assigned_version_id,
|
||||
delete_marker = dobj.delete_object.delete_marker,
|
||||
is_version_purge,
|
||||
"replicate_delete_to_target succeeded"
|
||||
);
|
||||
if !is_version_purge {
|
||||
// Record the version the target actually assigned to the marker it
|
||||
// just created. A later purge addresses that id directly instead of
|
||||
// deriving one from the source uuid, which only holds when the
|
||||
// target mirrors source version ids.
|
||||
if dobj.delete_object.delete_marker {
|
||||
rinfo.target_delete_marker_version_id = assigned_version_id.filter(|version_id| !version_id.is_empty());
|
||||
}
|
||||
rinfo.replication_status = ReplicationStatusType::Completed;
|
||||
} else {
|
||||
rinfo.version_purge_status = VersionPurgeStatusType::Complete;
|
||||
@@ -2109,18 +2001,61 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
rinfo
|
||||
}
|
||||
|
||||
pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, storage: Arc<S>) -> ReplicationState {
|
||||
replicate_object_with_outcome(roi, storage).await.0
|
||||
}
|
||||
|
||||
pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
roi: ReplicateObjectInfo,
|
||||
storage: Arc<S>,
|
||||
) -> (ReplicationState, bool) {
|
||||
pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, storage: Arc<S>) {
|
||||
let bucket = roi.bucket.clone();
|
||||
let object = roi.name.clone();
|
||||
|
||||
let tgt_arns = roi.admitted_target_arns();
|
||||
let cfg = match get_replication_config(&bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => {
|
||||
debug!(
|
||||
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
reason = "replication_config_missing",
|
||||
"Skipping replication object because replication config is missing"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: roi.to_object_info(),
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
reason = "replication_config_lookup_failed",
|
||||
error = %err,
|
||||
"Failed to look up replication config for object replication"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: roi.to_object_info(),
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let tgt_arns = cfg.filter_target_arns(&ObjectOpts {
|
||||
name: object.clone(),
|
||||
user_tags: roi.user_tags.clone(),
|
||||
ssec: roi.ssec,
|
||||
op_type: roi.op_type,
|
||||
// ExistingObject ops must respect per-rule ExistingObjectReplicationStatus.
|
||||
// Heal ops intentionally bypass it (repairing a past failure is not an initial sync).
|
||||
existing_object: roi.op_type == ReplicationType::ExistingObject,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
// Acquire a per-object namespace lock so that at most one worker (across all cluster
|
||||
// nodes and MRF retry goroutines) replicates this object version at a time.
|
||||
@@ -2145,7 +2080,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return (roi.replication_state.unwrap_or_default(), false);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let _obj_lock_guard = match obj_ns_lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await {
|
||||
@@ -2168,7 +2103,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return (roi.replication_state.unwrap_or_default(), false);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2244,12 +2179,9 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
}
|
||||
}
|
||||
|
||||
let previous_state = roi.replication_state.clone().unwrap_or_default();
|
||||
let merged_state = get_replication_state(&rinfos, &previous_state, roi.version_id.map(|v| v.to_string()));
|
||||
let replication_status = merged_state.composite_replication_status();
|
||||
let new_replication_internal = merged_state.replication_status_internal.clone();
|
||||
let replication_status = rinfos.replication_status();
|
||||
let new_replication_internal = rinfos.replication_status_internal();
|
||||
let mut object_info = roi.to_object_info();
|
||||
let mut state_persisted = true;
|
||||
|
||||
if roi.replication_status_internal != new_replication_internal || rinfos.replication_resynced() {
|
||||
let mut eval_metadata = HashMap::new();
|
||||
@@ -2265,7 +2197,6 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
match storage.put_object_metadata(&bucket, &object, &popts).await {
|
||||
Ok(u) => object_info = u,
|
||||
Err(e) => {
|
||||
state_persisted = false;
|
||||
// Persisting the resynced replication status failed. Don't swallow
|
||||
// it silently — the object's on-disk status now disagrees with the
|
||||
// resync result and needs operator visibility (backlog#799 B23).
|
||||
@@ -2318,8 +2249,6 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(merged_state, state_persisted)
|
||||
}
|
||||
|
||||
trait ReplicateObjectInfoExt {
|
||||
@@ -2550,12 +2479,6 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
|
||||
Ok((put_opts, is_mp)) => (put_opts, is_mp),
|
||||
Err(e) => {
|
||||
// Unsupported source metadata (e.g. managed SSE) is a fail-closed
|
||||
// condition: report FAILED so the composite status and the
|
||||
// OperationFailedReplication event reflect that nothing reached
|
||||
// the target, instead of leaking the optimistic Completed above.
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -2937,7 +2860,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
// The target already holds a matching object (reached here only via
|
||||
// the version-id fallback ETag match above) — there is nothing to
|
||||
// copy. Record it as synced and return, instead of falling into the
|
||||
// metadata propagation path below, which previously left
|
||||
// metadata-unsupported failure branch below, which previously left
|
||||
// AWS-style targets permanently FAILED and never converging
|
||||
// (backlog#860 / #799 B11).
|
||||
if self.op_type == ReplicationType::ExistingObject && !tgt_client.reset_id.is_empty() {
|
||||
@@ -2954,76 +2877,10 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
// The target client has no metadata-only operation. Reuse the existing
|
||||
// object transport so metadata changes carry tags and object-lock state
|
||||
// atomically with the source version.
|
||||
let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
|
||||
Ok((put_opts, is_mp)) => (put_opts, is_mp),
|
||||
Err(e) => {
|
||||
// Unsupported source metadata (e.g. managed SSE) is a fail-closed
|
||||
// condition: report FAILED so the composite status and the
|
||||
// OperationFailedReplication event reflect that nothing reached
|
||||
// the target, instead of leaking the optimistic Completed above.
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %tgt_client.arn,
|
||||
operation = "build_put_options",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
return rinfo;
|
||||
}
|
||||
};
|
||||
|
||||
let has_tagging_replication = !put_opts.user_tags.is_empty();
|
||||
if let Some(err) = if is_multipart {
|
||||
drop(gr);
|
||||
let result = replicate_object_with_multipart(MultipartReplicationContext {
|
||||
storage: storage.clone(),
|
||||
cli: tgt_client.clone(),
|
||||
src_bucket: &bucket,
|
||||
dst_bucket: &tgt_client.bucket,
|
||||
object: &object,
|
||||
object_info: &object_info,
|
||||
obj_opts: &obj_opts,
|
||||
arn: &rinfo.arn,
|
||||
put_opts,
|
||||
})
|
||||
.await;
|
||||
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
|
||||
if has_tagging_replication {
|
||||
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
|
||||
}
|
||||
result.err()
|
||||
} else {
|
||||
gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
|
||||
let byte_stream = async_read_to_bytestream(gr.stream);
|
||||
let result = tgt_client
|
||||
.put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()));
|
||||
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
|
||||
if has_tagging_replication {
|
||||
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
|
||||
}
|
||||
result.err()
|
||||
} {
|
||||
// action == Metadata: metadata-only replication is not implemented.
|
||||
if replication_action != ReplicationAction::All {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(err.to_string());
|
||||
rinfo.error = Some(ERR_REPLICATION_METADATA_COPY_UNSUPPORTED.to_string());
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -3031,14 +2888,98 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
bucket = %bucket,
|
||||
arn = %tgt_client.arn,
|
||||
object = %object,
|
||||
operation = "put_object",
|
||||
error = ?err,
|
||||
operation = "copy_object_metadata",
|
||||
error = ERR_REPLICATION_METADATA_COPY_UNSUPPORTED,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
|
||||
mark_replication_target_offline_if_needed(&tgt_client, &err).await;
|
||||
return rinfo;
|
||||
} else {
|
||||
let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
|
||||
Ok((put_opts, is_mp)) => (put_opts, is_mp),
|
||||
Err(e) => {
|
||||
rinfo.error = Some(e.to_string());
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %tgt_client.arn,
|
||||
operation = "build_put_options",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
return rinfo;
|
||||
}
|
||||
};
|
||||
|
||||
let has_tagging_replication = !put_opts.user_tags.is_empty();
|
||||
if let Some(err) = if is_multipart {
|
||||
drop(gr);
|
||||
let result = replicate_object_with_multipart(MultipartReplicationContext {
|
||||
storage: storage.clone(),
|
||||
cli: tgt_client.clone(),
|
||||
src_bucket: &bucket,
|
||||
dst_bucket: &tgt_client.bucket,
|
||||
object: &object,
|
||||
object_info: &object_info,
|
||||
obj_opts: &obj_opts,
|
||||
arn: &rinfo.arn,
|
||||
put_opts,
|
||||
})
|
||||
.await;
|
||||
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
|
||||
if has_tagging_replication {
|
||||
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
|
||||
}
|
||||
result.err()
|
||||
} else {
|
||||
gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
|
||||
let byte_stream = async_read_to_bytestream(gr.stream);
|
||||
let result = tgt_client
|
||||
.put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()));
|
||||
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
|
||||
if has_tagging_replication {
|
||||
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
|
||||
}
|
||||
result.err()
|
||||
} {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(err.to_string());
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %tgt_client.arn,
|
||||
object = %object,
|
||||
operation = "put_object",
|
||||
error = ?err,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
|
||||
mark_replication_target_offline_if_needed(&tgt_client, &err).await;
|
||||
return rinfo;
|
||||
}
|
||||
}
|
||||
|
||||
rinfo
|
||||
@@ -3224,7 +3165,7 @@ mod tests {
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn test_target_client(endpoint: String) -> Arc<TargetClient> {
|
||||
fn test_target_client(endpoint: String) -> TargetClient {
|
||||
let config = aws_sdk_s3::Config::builder()
|
||||
.endpoint_url(endpoint.clone())
|
||||
.region(aws_sdk_s3::config::Region::new("us-east-1"))
|
||||
@@ -3234,7 +3175,7 @@ mod tests {
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.build();
|
||||
|
||||
Arc::new(TargetClient {
|
||||
TargetClient {
|
||||
endpoint,
|
||||
credentials: None,
|
||||
bucket: "target-bucket".to_string(),
|
||||
@@ -3246,11 +3187,7 @@ mod tests {
|
||||
health_check_duration: std::time::Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
|
||||
})
|
||||
}
|
||||
|
||||
async fn register_test_target(target: &Arc<TargetClient>) {
|
||||
ReplicationTargetStore::register_test_target(target).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3266,7 +3203,6 @@ mod tests {
|
||||
async fn replication_target_network_failure_marks_target_offline() {
|
||||
let endpoint = format!("http://network-failure-{}.example:9000", Uuid::new_v4());
|
||||
let target_client = test_target_client(endpoint);
|
||||
register_test_target(&target_client).await;
|
||||
|
||||
assert!(!ReplicationTargetStore::target_is_offline(&target_client).await);
|
||||
|
||||
@@ -3280,7 +3216,6 @@ mod tests {
|
||||
async fn replication_target_service_failure_keeps_target_online() {
|
||||
let endpoint = format!("http://service-failure-{}.example:9000", Uuid::new_v4());
|
||||
let target_client = test_target_client(endpoint);
|
||||
register_test_target(&target_client).await;
|
||||
|
||||
assert!(!ReplicationTargetStore::target_is_offline(&target_client).await);
|
||||
|
||||
@@ -4067,35 +4002,4 @@ mod tests {
|
||||
assert_eq!(target_delete_version_id(Uuid::nil(), true).as_deref(), Some(NULL_VERSION_ID));
|
||||
assert_eq!(target_delete_version_id(Uuid::nil(), false), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_marker_purge_prefers_the_recorded_target_version() {
|
||||
let source = Uuid::new_v4();
|
||||
let arn = "arn:rustfs:replication::target:bucket";
|
||||
|
||||
// No recorded mapping: fall back to deriving from the source uuid.
|
||||
assert_eq!(delete_marker_purge_version_id(None, arn, source), Some(Some(source.to_string())));
|
||||
|
||||
// Recorded mapping wins — a generic S3 target assigns its own id, so the
|
||||
// derived one would purge the wrong version or nothing at all.
|
||||
let mut state = ReplicationState::default();
|
||||
state
|
||||
.target_delete_marker_version_ids
|
||||
.insert(arn.to_string(), "target-assigned-id".to_string());
|
||||
assert_eq!(
|
||||
delete_marker_purge_version_id(Some(&state), arn, source),
|
||||
Some(Some("target-assigned-id".to_string()))
|
||||
);
|
||||
|
||||
// A mapping recorded for a different ARN must not be reused.
|
||||
assert_eq!(
|
||||
delete_marker_purge_version_id(Some(&state), "arn:rustfs:replication::other:bucket", source),
|
||||
Some(Some(source.to_string()))
|
||||
);
|
||||
|
||||
// Inconsistent persisted metadata: refuse to purge rather than guess.
|
||||
let mut corrupt = state.clone();
|
||||
corrupt.target_delete_marker_version_ids_corrupt = true;
|
||||
assert_eq!(delete_marker_purge_version_id(Some(&corrupt), arn, source), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,10 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::BucketStats;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_replication::FailStats;
|
||||
pub(crate) use rustfs_replication::{
|
||||
ActiveWorkerStat, BucketReplicationStat, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope,
|
||||
SRMetricsSummary, XferStats,
|
||||
ActiveWorkerStat, BucketReplicationStat, BucketReplicationStats, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache,
|
||||
ReplicationMetricScope, SRMetricsSummary, XferStats,
|
||||
};
|
||||
pub use rustfs_replication::{BucketReplicationStats, BucketStats};
|
||||
|
||||
@@ -25,7 +25,7 @@ pub(crate) use crate::storage_api_contracts::list::{
|
||||
};
|
||||
pub(crate) use crate::storage_api_contracts::namespace::NamespaceLocking as StorageNamespaceLocking;
|
||||
pub(crate) use crate::storage_api_contracts::object::{
|
||||
DeletedObject, EcstoreObjectOperations, HTTPPreconditions, ObjectIO, ObjectOperations, ObjectToDelete,
|
||||
DeletedObject, EcstoreObjectOperations, ObjectIO, ObjectOperations, ObjectToDelete,
|
||||
};
|
||||
pub(crate) use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
pub(crate) use rustfs_replication::{DeletedObject as ReplicationDeletedObject, ObjectToDelete as ReplicationObjectToDelete};
|
||||
@@ -105,9 +105,6 @@ pub(crate) fn deleted_object_for_replication(delete_object: DeletedObject) -> Re
|
||||
replication_state: delete_object.replication_state.as_ref().map(replication_state_from_filemeta),
|
||||
found: delete_object.found,
|
||||
force_delete: delete_object.force_delete,
|
||||
force_delete_id: delete_object.force_delete_id,
|
||||
force_delete_target_arns: delete_object.force_delete_target_arns,
|
||||
force_delete_generation: delete_object.force_delete_generation,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,11 +24,10 @@ use rustfs_replication::{
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
|
||||
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
|
||||
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE,
|
||||
HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
||||
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_TAGGING_TIMESTAMP, get_str, insert_header_map,
|
||||
is_internal_key,
|
||||
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT,
|
||||
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, HeaderExt as _,
|
||||
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE,
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_TAGGING_TIMESTAMP, get_str, insert_header_map, is_internal_key,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
@@ -80,48 +79,6 @@ static VALID_SSE_REPLICATION_HEADERS: &[(&str, &str)] = &[
|
||||
];
|
||||
|
||||
const ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED: &str = "managed SSE replication requires target encryption support";
|
||||
const ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED: &str = "replication source contains unsupported encryption metadata";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ReplicationSourceEncryption {
|
||||
Plaintext,
|
||||
SseS3,
|
||||
SseKms,
|
||||
SseC,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
fn metadata_value<'a>(metadata: &'a HashMap<String, String>, name: &str) -> Option<&'a str> {
|
||||
metadata
|
||||
.iter()
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.as_str())
|
||||
}
|
||||
|
||||
fn classify_replication_source_encryption(metadata: &HashMap<String, String>) -> ReplicationSourceEncryption {
|
||||
let is_ssec = replication_object_is_ssec_encrypted(metadata);
|
||||
let sse = metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION);
|
||||
let kms_key_id = metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID);
|
||||
let kms_context = metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT);
|
||||
|
||||
if is_ssec {
|
||||
return if sse.is_some() || kms_key_id.is_some() || kms_context.is_some() {
|
||||
ReplicationSourceEncryption::Unsupported
|
||||
} else {
|
||||
ReplicationSourceEncryption::SseC
|
||||
};
|
||||
}
|
||||
|
||||
match sse.map(str::trim) {
|
||||
None if kms_key_id.is_none() && kms_context.is_none() => ReplicationSourceEncryption::Plaintext,
|
||||
Some(value) if value.eq_ignore_ascii_case("AES256") && kms_key_id.is_none() && kms_context.is_none() => {
|
||||
ReplicationSourceEncryption::SseS3
|
||||
}
|
||||
Some(value) if value.eq_ignore_ascii_case("aws:kms") => ReplicationSourceEncryption::SseKms,
|
||||
_ if kms_key_id.is_some() => ReplicationSourceEncryption::SseKms,
|
||||
_ => ReplicationSourceEncryption::Unsupported,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
|
||||
rustfs_replication::is_ssec_encrypted(user_defined)
|
||||
@@ -138,20 +95,12 @@ impl ReplicationTargetStore {
|
||||
BucketTargetSys::get().get_remote_target_client(bucket, arn).await
|
||||
}
|
||||
|
||||
pub(crate) async fn target_is_offline(target_client: &Arc<TargetClient>) -> bool {
|
||||
BucketTargetSys::get().is_target_offline(target_client).await
|
||||
pub(crate) async fn target_is_offline(target_client: &TargetClient) -> bool {
|
||||
BucketTargetSys::get().is_offline(&target_client.to_url()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn mark_target_offline(target_client: &Arc<TargetClient>) {
|
||||
BucketTargetSys::get().mark_target_offline(target_client).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
|
||||
BucketTargetSys::get().arn_remotes_map.write().await.insert(
|
||||
target_client.arn.clone(),
|
||||
crate::bucket::bucket_target_sys::ArnTarget::with_client(target_client.clone()),
|
||||
);
|
||||
pub(crate) async fn mark_target_offline(target_client: &TargetClient) {
|
||||
BucketTargetSys::get().mark_offline(&target_client.to_url()).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,18 +109,7 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
use rustfs_utils::http::{AMZ_CHECKSUM_TYPE, AMZ_CHECKSUM_TYPE_FULL_OBJECT};
|
||||
|
||||
let mut meta = HashMap::new();
|
||||
let source_encryption = classify_replication_source_encryption(&object_info.user_defined);
|
||||
let is_ssec = matches!(source_encryption, ReplicationSourceEncryption::SseC);
|
||||
|
||||
match source_encryption {
|
||||
ReplicationSourceEncryption::Plaintext | ReplicationSourceEncryption::SseC => {}
|
||||
ReplicationSourceEncryption::SseS3 | ReplicationSourceEncryption::SseKms => {
|
||||
return Err(Error::other(ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED));
|
||||
}
|
||||
ReplicationSourceEncryption::Unsupported => {
|
||||
return Err(Error::other(ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED));
|
||||
}
|
||||
}
|
||||
let is_ssec = replication_object_is_ssec_encrypted(&object_info.user_defined);
|
||||
|
||||
for (key, value) in object_info.user_defined.iter() {
|
||||
let has_valid_sse_header = valid_sse_replication_header(key).is_some();
|
||||
@@ -297,6 +235,20 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
};
|
||||
}
|
||||
|
||||
let has_sse_s3 = object_info
|
||||
.user_defined
|
||||
.get(AMZ_SERVER_SIDE_ENCRYPTION)
|
||||
.is_some_and(|value| value.eq_ignore_ascii_case("AES256"));
|
||||
let has_sse_kms = object_info
|
||||
.user_defined
|
||||
.get(AMZ_SERVER_SIDE_ENCRYPTION)
|
||||
.is_some_and(|value| value.eq_ignore_ascii_case("aws:kms"))
|
||||
|| object_info.user_defined.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID);
|
||||
|
||||
if has_sse_s3 || has_sse_kms {
|
||||
return Err(Error::other(ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED));
|
||||
}
|
||||
|
||||
Ok((put_options, is_multipart))
|
||||
}
|
||||
|
||||
@@ -634,46 +586,6 @@ mod tests {
|
||||
assert!(get_header_map(&options.user_metadata, SUFFIX_REPLICATION_SSEC_CRC).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_source_encryption_classification_is_explicit_and_fail_closed() {
|
||||
assert_eq!(
|
||||
classify_replication_source_encryption(&HashMap::new()),
|
||||
ReplicationSourceEncryption::Plaintext
|
||||
);
|
||||
assert_eq!(
|
||||
classify_replication_source_encryption(&HashMap::from([(
|
||||
"x-amz-server-side-encryption".to_string(),
|
||||
"AES256".to_string()
|
||||
)])),
|
||||
ReplicationSourceEncryption::SseS3
|
||||
);
|
||||
assert_eq!(
|
||||
classify_replication_source_encryption(&HashMap::from([(
|
||||
"x-amz-server-side-encryption".to_string(),
|
||||
"AWS:KMS".to_string()
|
||||
)])),
|
||||
ReplicationSourceEncryption::SseKms
|
||||
);
|
||||
assert_eq!(
|
||||
classify_replication_source_encryption(&HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())])),
|
||||
ReplicationSourceEncryption::SseC
|
||||
);
|
||||
assert_eq!(
|
||||
classify_replication_source_encryption(&HashMap::from([(
|
||||
"x-amz-server-side-encryption".to_string(),
|
||||
"unsupported-algorithm".to_string(),
|
||||
)])),
|
||||
ReplicationSourceEncryption::Unsupported
|
||||
);
|
||||
assert_eq!(
|
||||
classify_replication_source_encryption(&HashMap::from([(
|
||||
AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT.to_string(),
|
||||
"opaque-context".to_string(),
|
||||
)])),
|
||||
ReplicationSourceEncryption::Unsupported
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_rejects_sse_s3_until_target_encryption_is_supported() {
|
||||
let object_info = ObjectInfo {
|
||||
@@ -707,25 +619,6 @@ mod tests {
|
||||
assert!(err.to_string().contains(ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_rejects_unknown_encryption_without_echoing_metadata() {
|
||||
let secret_like_value = "opaque-context-that-must-not-be-logged";
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(HashMap::from([
|
||||
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "unsupported-algorithm".to_string()),
|
||||
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT.to_string(), secret_like_value.to_string()),
|
||||
])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = match replication_put_object_options("", &object_info) {
|
||||
Ok(_) => panic!("unknown encryption must fail closed"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(err.to_string().contains(ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED));
|
||||
assert!(!err.to_string().contains(secret_like_value));
|
||||
}
|
||||
|
||||
// T3 (#1264): the outbound replication path forwards a stored object checksum into
|
||||
// user_metadata via decrypt_checksums, which is algorithm-agnostic. This locks that
|
||||
// the AWS 2026-04 additional algorithms (XXHash3/64/128, SHA-512, MD5) are forwarded
|
||||
|
||||
@@ -56,59 +56,11 @@ impl FromStr for ARN {
|
||||
if parts.len() != 6 {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
|
||||
}
|
||||
// Display emits `arn:rustfs:{type}:{region}:{id}:{bucket}`; read the
|
||||
// segments back in the same order so parse(display(a)) == a.
|
||||
Ok(ARN {
|
||||
arn_type: BucketTargetType::from_str(parts[2]).unwrap_or_default(),
|
||||
region: parts[3].to_string(),
|
||||
id: parts[4].to_string(),
|
||||
id: parts[3].to_string(),
|
||||
region: parts[4].to_string(),
|
||||
bucket: parts[5].to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Display emits `arn:rustfs:{type}:{region}:{id}:{bucket}` (madmin layout);
|
||||
/// FromStr must read the same positions back so parse(display(a)) == a.
|
||||
#[test]
|
||||
fn from_str_round_trips_display_with_region_and_id() {
|
||||
let arn = ARN::new(
|
||||
BucketTargetType::ReplicationService,
|
||||
"depl-123".to_string(),
|
||||
"us-east-1".to_string(),
|
||||
"bucket-a".to_string(),
|
||||
);
|
||||
|
||||
let parsed = ARN::from_str(&arn.to_string()).expect("display output must parse");
|
||||
|
||||
assert_eq!(parsed.arn_type, arn.arn_type);
|
||||
assert_eq!(parsed.region, arn.region, "region must survive display->parse round-trip");
|
||||
assert_eq!(parsed.id, arn.id, "id must survive display->parse round-trip");
|
||||
assert_eq!(parsed.bucket, arn.bucket);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_str_reads_region_then_id_in_display_order() {
|
||||
let parsed = ARN::from_str("arn:rustfs:replication:us-east-1:depl-123:bucket-a").expect("valid ARN must parse");
|
||||
|
||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||
assert_eq!(parsed.region, "us-east-1");
|
||||
assert_eq!(parsed.id, "depl-123");
|
||||
assert_eq!(parsed.bucket, "bucket-a");
|
||||
}
|
||||
|
||||
/// RustFS commonly generates ARNs with an empty region:
|
||||
/// `arn:rustfs:replication::<deployment_id>:<bucket>`.
|
||||
#[test]
|
||||
fn from_str_handles_empty_region_segment() {
|
||||
let parsed = ARN::from_str("arn:rustfs:replication::depl-123:bucket-a").expect("valid ARN must parse");
|
||||
|
||||
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
|
||||
assert_eq!(parsed.region, "", "region segment is empty in this form");
|
||||
assert_eq!(parsed.id, "depl-123");
|
||||
assert_eq!(parsed.bucket, "bucket-a");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use jiff::Timestamp;
|
||||
use rmp_serde::Serializer as rmpSerializer;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
@@ -33,7 +32,7 @@ pub struct Credentials {
|
||||
#[serde(rename = "secretKey")]
|
||||
pub secret_key: String,
|
||||
pub session_token: Option<String>,
|
||||
pub expiration: Option<Timestamp>,
|
||||
pub expiration: Option<chrono::DateTime<chrono::Utc>>,
|
||||
}
|
||||
|
||||
impl Credentials {
|
||||
@@ -94,21 +93,6 @@ mod duration_milliseconds {
|
||||
}
|
||||
}
|
||||
|
||||
/// Defensive decode for the two integer wire encodings of these duration
|
||||
/// fields: RustFS persists (and legacy RustFS clients sent) plain seconds,
|
||||
/// while Go `time.Duration` JSON — madmin/mc requests and MinIO-written
|
||||
/// bucket-targets metadata — is nanoseconds. No meaningful interval lies
|
||||
/// between 10^7 seconds (~115 days) and 10^7 nanoseconds (10ms), so the
|
||||
/// magnitude disambiguates the unit.
|
||||
pub fn duration_from_secs_or_nanos(value: u64) -> Duration {
|
||||
const NANOS_THRESHOLD: u64 = 10_000_000;
|
||||
if value < NANOS_THRESHOLD {
|
||||
Duration::from_secs(value)
|
||||
} else {
|
||||
Duration::from_nanos(value)
|
||||
}
|
||||
}
|
||||
|
||||
mod duration_seconds {
|
||||
use serde::{Deserialize, Deserializer, Serializer};
|
||||
use std::time::Duration;
|
||||
@@ -124,8 +108,8 @@ mod duration_seconds {
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let value = u64::deserialize(deserializer)?;
|
||||
Ok(super::duration_from_secs_or_nanos(value))
|
||||
let secs = u64::deserialize(deserializer)?;
|
||||
Ok(Duration::from_secs(secs))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -424,11 +408,7 @@ mod tests {
|
||||
assert_eq!(credentials.access_key, "test-access-key");
|
||||
assert_eq!(credentials.secret_key, "test-secret-key");
|
||||
assert_eq!(credentials.session_token, Some("test-session-token".to_string()));
|
||||
assert_eq!(
|
||||
serde_json::to_value(credentials.expiration.expect("expiration should parse"))
|
||||
.expect("expiration should serialize to JSON"),
|
||||
serde_json::json!("2024-12-31T23:59:59Z")
|
||||
);
|
||||
assert!(credentials.expiration.is_some());
|
||||
|
||||
// Verify latency statistics
|
||||
assert_eq!(target.latency.curr, Duration::from_millis(100));
|
||||
@@ -504,29 +484,6 @@ mod tests {
|
||||
assert_eq!(original.offline_count, deserialized.offline_count);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_target_reads_go_nanosecond_durations_defensively() {
|
||||
// MinIO-written bucket-targets metadata and madmin clients encode
|
||||
// these fields as Go `time.Duration` nanoseconds; RustFS has always
|
||||
// persisted seconds. Both encodings must decode to the same interval.
|
||||
let target: BucketTarget = serde_json::from_value(serde_json::json!({
|
||||
"endpoint": "localhost:9000",
|
||||
"targetbucket": "target",
|
||||
"type": "replication",
|
||||
"healthCheckDuration": 60_000_000_000u64,
|
||||
"totalDowntime": 90_000_000_000u64
|
||||
}))
|
||||
.expect("nanosecond durations should deserialize");
|
||||
|
||||
assert_eq!(target.health_check_duration, Duration::from_secs(60));
|
||||
assert_eq!(target.total_downtime, Duration::from_secs(90));
|
||||
|
||||
// The persisted wire format stays seconds for existing RustFS readers.
|
||||
let value = serde_json::to_value(&target).expect("target should serialize");
|
||||
assert_eq!(value["healthCheckDuration"], 60);
|
||||
assert_eq!(value["totalDowntime"], 90);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_target_debug_redacts_credentials() {
|
||||
let target = BucketTarget {
|
||||
@@ -573,50 +530,6 @@ mod tests {
|
||||
assert!(redacted_json.contains(r#""session_token":null"#));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn historical_bucket_target_options_remain_readable() {
|
||||
let target: BucketTarget = serde_json::from_value(serde_json::json!({
|
||||
"endpoint": "legacy.example:9000",
|
||||
"credentials": {
|
||||
"accessKey": "legacy-access",
|
||||
"secretKey": "legacy-secret",
|
||||
"session_token": "legacy-session-token",
|
||||
"expiration": "2024-12-31T23:59:59Z"
|
||||
},
|
||||
"targetbucket": "legacy-bucket",
|
||||
"api": "s3v2",
|
||||
"healthCheckDuration": 30,
|
||||
"disableProxy": true,
|
||||
"edge": true,
|
||||
"edgeSyncBeforeExpiry": true,
|
||||
"type": "replication"
|
||||
}))
|
||||
.expect("historical remote target should remain readable");
|
||||
|
||||
assert_eq!(target.api, "s3v2");
|
||||
assert_eq!(target.health_check_duration, Duration::from_secs(30));
|
||||
assert!(target.disable_proxy);
|
||||
assert!(target.edge);
|
||||
assert!(target.edge_sync_before_expiry);
|
||||
assert_eq!(
|
||||
target
|
||||
.credentials
|
||||
.as_ref()
|
||||
.and_then(|credentials| credentials.session_token.as_deref()),
|
||||
Some("legacy-session-token")
|
||||
);
|
||||
assert_eq!(
|
||||
target
|
||||
.credentials
|
||||
.as_ref()
|
||||
.and_then(|credentials| credentials.expiration)
|
||||
.map(serde_json::to_value)
|
||||
.transpose()
|
||||
.expect("expiration should serialize to JSON"),
|
||||
Some(serde_json::json!("2024-12-31T23:59:59Z"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_target_type_json_deserialize() {
|
||||
// Test BucketTargetType JSON deserialization
|
||||
@@ -655,11 +568,7 @@ mod tests {
|
||||
credentials.session_token,
|
||||
Some("AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(credentials.expiration.expect("expiration should parse"))
|
||||
.expect("expiration should serialize to JSON"),
|
||||
serde_json::json!("2024-12-31T23:59:59Z")
|
||||
);
|
||||
assert!(credentials.expiration.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -269,32 +269,6 @@ pub fn check_del_obj_args(bucket: &str, object: &str) -> Result<()> {
|
||||
check_bucket_and_object_names(bucket, object)
|
||||
}
|
||||
|
||||
/// Filesystem `NAME_MAX`: every object-key path segment becomes one on-disk
|
||||
/// directory entry, so a longer segment can never be stored and previously
|
||||
/// escaped as an `ENAMETOOLONG` io error → `InternalError` 500 (rustfs#5785).
|
||||
const MAX_OBJECT_KEY_SEGMENT_BYTES: usize = 255;
|
||||
|
||||
/// Reject object keys whose on-disk directory names would exceed `NAME_MAX`.
|
||||
///
|
||||
/// Middle segments map to their raw bytes; the final segment of a
|
||||
/// directory-object key (trailing `/`) is stored with the `__XLDIR__` suffix
|
||||
/// appended, shrinking its budget accordingly.
|
||||
fn object_key_segments_fit_on_disk(object: &str) -> bool {
|
||||
let trailing_dir = object.ends_with('/');
|
||||
let segments: Vec<&str> = object.split('/').collect();
|
||||
let last_nonempty = segments.iter().rposition(|s| !s.is_empty());
|
||||
for (index, segment) in segments.iter().enumerate() {
|
||||
let mut budget = MAX_OBJECT_KEY_SEGMENT_BYTES;
|
||||
if trailing_dir && Some(index) == last_nonempty {
|
||||
budget = budget.saturating_sub(rustfs_utils::path::GLOBAL_DIR_SUFFIX.len());
|
||||
}
|
||||
if segment.len() > budget {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub fn check_bucket_and_object_names(bucket: &str, object: &str) -> Result<()> {
|
||||
if !is_meta_bucketname(bucket) && check_valid_bucket_name_strict(bucket).is_err() {
|
||||
return Err(StorageError::BucketNameInvalid(bucket.to_string()));
|
||||
@@ -308,10 +282,6 @@ pub fn check_bucket_and_object_names(bucket: &str, object: &str) -> Result<()> {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
|
||||
if !object_key_segments_fit_on_disk(object) {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
|
||||
// if cfg!(target_os = "windows") && object.contains('\\') {
|
||||
// return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
|
||||
// }
|
||||
@@ -409,14 +379,6 @@ pub fn check_put_object_args(bucket: &str, object: &str) -> Result<()> {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
|
||||
// The write path validates arguments here rather than through
|
||||
// check_bucket_and_object_names, so the on-disk segment budget has to be
|
||||
// enforced in both places or an over-NAME_MAX key still reaches the disk
|
||||
// layer and escapes as ENAMETOOLONG → InternalError 500 (rustfs#5785).
|
||||
if !object_key_segments_fit_on_disk(object) {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -425,62 +387,6 @@ mod tests {
|
||||
use super::*;
|
||||
use proptest::prelude::*;
|
||||
|
||||
/// rustfs#5785: keys whose path segments exceed the on-disk NAME_MAX
|
||||
/// budget must be rejected up front as ObjectNameInvalid (4xx), not leak
|
||||
/// ENAMETOOLONG as InternalError 500 from the disk layer.
|
||||
#[test]
|
||||
fn object_key_segment_name_max_budget() {
|
||||
// 255-byte single segment: exactly at the on-disk limit.
|
||||
assert!(check_bucket_and_object_names("bucket", &"a".repeat(255)).is_ok());
|
||||
// 256 bytes: one over.
|
||||
assert!(matches!(
|
||||
check_bucket_and_object_names("bucket", &"a".repeat(256)),
|
||||
Err(StorageError::ObjectNameInvalid(_, _))
|
||||
));
|
||||
// Long keys are fine as long as every segment fits.
|
||||
let segmented = ["b".repeat(200), "c".repeat(200), "d".repeat(200)].join("/");
|
||||
assert!(check_bucket_and_object_names("bucket", &segmented).is_ok());
|
||||
// The budget counts bytes, not characters (100 CJK chars = 300 bytes).
|
||||
assert!(matches!(
|
||||
check_bucket_and_object_names("bucket", &"中".repeat(100)),
|
||||
Err(StorageError::ObjectNameInvalid(_, _))
|
||||
));
|
||||
assert!(check_bucket_and_object_names("bucket", &"中".repeat(85)).is_ok());
|
||||
// Directory-object keys spend GLOBAL_DIR_SUFFIX bytes of the final
|
||||
// segment's budget on the on-disk __XLDIR__ encoding.
|
||||
let dir_budget = 255 - rustfs_utils::path::GLOBAL_DIR_SUFFIX.len();
|
||||
assert!(check_bucket_and_object_names("bucket", &format!("{}/", "e".repeat(dir_budget))).is_ok());
|
||||
assert!(matches!(
|
||||
check_bucket_and_object_names("bucket", &format!("{}/", "e".repeat(dir_budget + 1))),
|
||||
Err(StorageError::ObjectNameInvalid(_, _))
|
||||
));
|
||||
}
|
||||
|
||||
/// rustfs#5785 follow-up: the write path validates through
|
||||
/// check_put_object_args, not check_bucket_and_object_names, so the same
|
||||
/// budget has to hold there — otherwise an over-NAME_MAX PUT still reached
|
||||
/// the disk layer and came back as InternalError 500.
|
||||
#[test]
|
||||
fn put_object_args_enforce_the_same_segment_budget() {
|
||||
assert!(check_put_object_args("bucket", &"a".repeat(255)).is_ok());
|
||||
assert!(matches!(
|
||||
check_put_object_args("bucket", &"a".repeat(256)),
|
||||
Err(StorageError::ObjectNameInvalid(_, _))
|
||||
));
|
||||
assert!(matches!(
|
||||
check_put_object_args("bucket", &"\u{4e2d}".repeat(100)),
|
||||
Err(StorageError::ObjectNameInvalid(_, _))
|
||||
));
|
||||
let segmented = ["b".repeat(200), "c".repeat(200), "d".repeat(200)].join("/");
|
||||
assert!(check_put_object_args("bucket", &segmented).is_ok());
|
||||
let dir_budget = 255 - rustfs_utils::path::GLOBAL_DIR_SUFFIX.len();
|
||||
assert!(check_put_object_args("bucket", &format!("{}/", "e".repeat(dir_budget))).is_ok());
|
||||
assert!(matches!(
|
||||
check_put_object_args("bucket", &format!("{}/", "e".repeat(dir_budget + 1))),
|
||||
Err(StorageError::ObjectNameInvalid(_, _))
|
||||
));
|
||||
}
|
||||
|
||||
// Test validation functions
|
||||
#[test]
|
||||
fn test_is_valid_object_name() {
|
||||
|
||||
@@ -21,22 +21,17 @@ const EVENT_LIFECYCLE_CLEANUP_SKIPPED: &str = "lifecycle_cleanup_skipped";
|
||||
const EVENT_LIFECYCLE_CLEANUP_FAILED: &str = "lifecycle_cleanup_failed";
|
||||
|
||||
use crate::bucket::lifecycle::lifecycle;
|
||||
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationObjectBridge};
|
||||
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationState, replication_state_to_filemeta};
|
||||
use crate::bucket::versioning::VersioningApi;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::storage_api_contracts::object::{ObjectOperations as _, ObjectToDelete};
|
||||
use crate::store::ECStore;
|
||||
use rustfs_lock::MAX_DELETE_LIST;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub async fn delete_object_versions(
|
||||
api: &Arc<ECStore>,
|
||||
bucket: &str,
|
||||
to_del: &[ObjectToDelete],
|
||||
_lc_event: lifecycle::Event,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) {
|
||||
let delete_config_snapshot = match ReplicationObjectBridge::delete_request_config(api, bucket).await {
|
||||
Ok(snapshot) => Arc::new(snapshot),
|
||||
pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
|
||||
let version_suspended = match BucketVersioningSys::get(bucket).await {
|
||||
Ok(vc) => vc.suspended(),
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_CLEANUP_SKIPPED,
|
||||
@@ -44,7 +39,7 @@ pub async fn delete_object_versions(
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket,
|
||||
error = ?err,
|
||||
reason = "delete_config_snapshot_unavailable",
|
||||
reason = "versioning_config_unavailable",
|
||||
"Skipped lifecycle noncurrent version cleanup"
|
||||
);
|
||||
return;
|
||||
@@ -60,13 +55,45 @@ pub async fn delete_object_versions(
|
||||
remaining = &[];
|
||||
}
|
||||
|
||||
let mut replication_candidates: Vec<Option<ReplicationState>> = Vec::with_capacity(to_del.len());
|
||||
for object in to_del.iter() {
|
||||
let version_id = object.version_id.map(|vid| vid.to_string());
|
||||
let opts = ObjectOptions {
|
||||
version_id: version_id.clone(),
|
||||
versioned: true,
|
||||
version_suspended,
|
||||
..Default::default()
|
||||
};
|
||||
let candidate = match api.get_object_info(bucket, &object.object_name, &opts).await {
|
||||
Ok(info) => {
|
||||
let dsc = ReplicationLifecycleBridge::check_delete_replication(bucket, object, &info, &opts).await;
|
||||
dsc.replicate_any()
|
||||
.then(|| ReplicationLifecycleBridge::version_delete_replication_state(&dsc))
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_CLEANUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket,
|
||||
object = %object.object_name,
|
||||
version_id = ?version_id,
|
||||
error = ?err,
|
||||
reason = "object_info_unavailable",
|
||||
"Skipped lifecycle delete replication scheduling"
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
replication_candidates.push(candidate);
|
||||
}
|
||||
|
||||
let (mut deleted_objs, errors) = api
|
||||
.delete_objects(
|
||||
bucket,
|
||||
to_del.to_vec(),
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&delete_config_snapshot)),
|
||||
expected_bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
version_suspended,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -81,9 +108,10 @@ pub async fn delete_object_versions(
|
||||
if let Some(target) = to_del.get(i) {
|
||||
crate::object_api::notify_object_mutation(bucket, &target.object_name).await;
|
||||
}
|
||||
if deleted_obj.replication_state.is_none() {
|
||||
let Some(replication_state) = replication_candidates.get(i).and_then(|c| c.clone()) else {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
deleted_obj.replication_state = Some(replication_state_to_filemeta(&replication_state));
|
||||
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_obj.clone()).await;
|
||||
}
|
||||
|
||||
|
||||
@@ -27,10 +27,7 @@
|
||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
|
||||
|
||||
use crate::cluster::rpc::context_propagation::{inject_request_id_into_http_headers, inject_trace_context_into_http_headers};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
NS_SCANNER_PROTOCOL_VERSION, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN,
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC,
|
||||
};
|
||||
use crate::storage_api_contracts::internode::NS_SCANNER_PROTOCOL_VERSION;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose;
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
@@ -39,20 +36,15 @@ use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
#[cfg(test)]
|
||||
use rustfs_credentials::{DEFAULT_SECRET_KEY, RPC_SECRET_REQUIRED_MESSAGE};
|
||||
use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_GRPC_OTHER, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics,
|
||||
};
|
||||
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use rustfs_utils::get_env_bool;
|
||||
use sha2::Digest as _;
|
||||
use sha2::Sha256;
|
||||
use std::collections::{HashSet, VecDeque};
|
||||
use std::sync::{LazyLock, Mutex, Once};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{error, info, warn};
|
||||
use tracing::error;
|
||||
use uuid::Uuid;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
@@ -74,16 +66,10 @@ const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
|
||||
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
||||
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
|
||||
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
|
||||
const HTTP_PUT_FILE_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-auth-v1\0";
|
||||
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
|
||||
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
||||
const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
|
||||
const REPLAY_CACHE_RETENTION_SECS: usize = 601;
|
||||
const REPLAY_CACHE_ENTRY_BYTES_ESTIMATE: u64 = 128;
|
||||
const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 8;
|
||||
const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 2048;
|
||||
const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 16_777_216;
|
||||
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
|
||||
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
|
||||
static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
@@ -105,211 +91,18 @@ static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
)
|
||||
});
|
||||
// Sized for peak legitimate authenticated RPC RPS x the retention window once replay scope is
|
||||
// active; overflow fails closed and increments the replay-cache overflow counter. Explicit operator
|
||||
// values and auto-sizing are both floored at the historical default so under-sizing cannot turn
|
||||
// legitimate high-throughput traffic into `No valid auth token` failures.
|
||||
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(resolve_replay_cache_capacity);
|
||||
// active; overflow fails closed and increments the replay-cache overflow counter. Clamped to at
|
||||
// least 1 so a misconfigured zero cannot disable replay protection by rejecting every request.
|
||||
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
)
|
||||
.max(1)
|
||||
});
|
||||
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
|
||||
static RPC_BOOT_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ReplayCacheCapacitySource {
|
||||
Env,
|
||||
EnvClampedToDefault,
|
||||
Auto,
|
||||
AutoClampedToDefault,
|
||||
AutoInvalidEnv,
|
||||
AutoInvalidEnvClampedToDefault,
|
||||
}
|
||||
|
||||
impl ReplayCacheCapacitySource {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Env => "env",
|
||||
Self::EnvClampedToDefault => "env_clamped_to_default",
|
||||
Self::Auto => "auto",
|
||||
Self::AutoClampedToDefault => "auto_clamped_to_default",
|
||||
Self::AutoInvalidEnv => "auto_invalid_env",
|
||||
Self::AutoInvalidEnvClampedToDefault => "auto_invalid_env_clamped_to_default",
|
||||
}
|
||||
}
|
||||
|
||||
fn is_env_clamped(self) -> bool {
|
||||
matches!(self, Self::EnvClampedToDefault)
|
||||
}
|
||||
|
||||
fn is_env(self) -> bool {
|
||||
matches!(self, Self::Env)
|
||||
}
|
||||
|
||||
fn is_invalid_env(self) -> bool {
|
||||
matches!(self, Self::AutoInvalidEnv | Self::AutoInvalidEnvClampedToDefault)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct ReplayCacheCapacityDecision {
|
||||
capacity: usize,
|
||||
source: ReplayCacheCapacitySource,
|
||||
cpu_count: usize,
|
||||
memory_limit_bytes: Option<u64>,
|
||||
memory_basis: Option<MemoryBasis>,
|
||||
memory_based_capacity: usize,
|
||||
cpu_based_capacity: usize,
|
||||
}
|
||||
|
||||
fn saturating_usize_from_u64(value: u64) -> usize {
|
||||
usize::try_from(value).unwrap_or(usize::MAX)
|
||||
}
|
||||
|
||||
fn replay_cache_capacity_from_resources(cpu_count: usize, memory_limit_bytes: Option<u64>) -> (usize, usize, usize) {
|
||||
let cpu_count = cpu_count.max(1);
|
||||
let cpu_based_capacity = cpu_count
|
||||
.saturating_mul(REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU)
|
||||
.saturating_mul(REPLAY_CACHE_RETENTION_SECS);
|
||||
let memory_based_capacity = memory_limit_bytes
|
||||
.map(|bytes| {
|
||||
let budget = bytes.saturating_mul(REPLAY_CACHE_AUTO_MEMORY_PERCENT) / 100;
|
||||
saturating_usize_from_u64(budget / REPLAY_CACHE_ENTRY_BYTES_ESTIMATE)
|
||||
})
|
||||
.unwrap_or(REPLAY_CACHE_AUTO_MAX_CAPACITY);
|
||||
let capacity = memory_based_capacity
|
||||
.min(cpu_based_capacity)
|
||||
.clamp(rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, REPLAY_CACHE_AUTO_MAX_CAPACITY);
|
||||
(capacity, memory_based_capacity, cpu_based_capacity)
|
||||
}
|
||||
|
||||
fn replay_cache_capacity_decision(
|
||||
env: rustfs_utils::EnvParseOutcome<usize>,
|
||||
cpu_count: usize,
|
||||
memory_limit_bytes: Option<u64>,
|
||||
memory_basis: Option<MemoryBasis>,
|
||||
) -> ReplayCacheCapacityDecision {
|
||||
let default = rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY;
|
||||
match env {
|
||||
rustfs_utils::EnvParseOutcome::Parsed(configured) => {
|
||||
let capacity = configured.max(default);
|
||||
let source = if configured < default {
|
||||
ReplayCacheCapacitySource::EnvClampedToDefault
|
||||
} else {
|
||||
ReplayCacheCapacitySource::Env
|
||||
};
|
||||
ReplayCacheCapacityDecision {
|
||||
capacity,
|
||||
source,
|
||||
cpu_count: cpu_count.max(1),
|
||||
memory_limit_bytes,
|
||||
memory_basis,
|
||||
memory_based_capacity: 0,
|
||||
cpu_based_capacity: 0,
|
||||
}
|
||||
}
|
||||
rustfs_utils::EnvParseOutcome::Absent | rustfs_utils::EnvParseOutcome::Invalid => {
|
||||
let (capacity, memory_based_capacity, cpu_based_capacity) =
|
||||
replay_cache_capacity_from_resources(cpu_count, memory_limit_bytes);
|
||||
let clamped_to_default = capacity == default && memory_based_capacity.min(cpu_based_capacity) < default;
|
||||
let invalid_env = matches!(env, rustfs_utils::EnvParseOutcome::Invalid);
|
||||
let source = match (invalid_env, clamped_to_default) {
|
||||
(true, true) => ReplayCacheCapacitySource::AutoInvalidEnvClampedToDefault,
|
||||
(true, false) => ReplayCacheCapacitySource::AutoInvalidEnv,
|
||||
(false, true) => ReplayCacheCapacitySource::AutoClampedToDefault,
|
||||
(false, false) => ReplayCacheCapacitySource::Auto,
|
||||
};
|
||||
ReplayCacheCapacityDecision {
|
||||
capacity,
|
||||
source,
|
||||
cpu_count: cpu_count.max(1),
|
||||
memory_limit_bytes,
|
||||
memory_basis,
|
||||
memory_based_capacity,
|
||||
cpu_based_capacity,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn detected_replay_cache_resources() -> (usize, Option<u64>, Option<MemoryBasis>) {
|
||||
let cpu_count = thread::available_parallelism().map(usize::from).unwrap_or(1).max(1);
|
||||
let memory = resolve_effective_memory();
|
||||
let memory_limit_bytes = (memory.total_bytes > 0).then_some(memory.total_bytes);
|
||||
(cpu_count, memory_limit_bytes, Some(memory.basis))
|
||||
}
|
||||
|
||||
fn log_replay_cache_capacity_decision(decision: ReplayCacheCapacityDecision) {
|
||||
let source = decision.source.as_str();
|
||||
if decision.source.is_env_clamped() {
|
||||
warn!(
|
||||
event = "internode_rpc_replay_cache_capacity_resolved",
|
||||
component = "ecstore",
|
||||
subsystem = "rpc_auth",
|
||||
capacity = decision.capacity,
|
||||
source,
|
||||
default_capacity = rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
env = rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
"internode rpc replay cache capacity clamped to default"
|
||||
);
|
||||
return;
|
||||
}
|
||||
if decision.source.is_env() {
|
||||
info!(
|
||||
event = "internode_rpc_replay_cache_capacity_resolved",
|
||||
component = "ecstore",
|
||||
subsystem = "rpc_auth",
|
||||
capacity = decision.capacity,
|
||||
source,
|
||||
default_capacity = rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
env = rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
"internode rpc replay cache capacity resolved from env"
|
||||
);
|
||||
return;
|
||||
}
|
||||
if decision.source.is_invalid_env() {
|
||||
warn!(
|
||||
event = "internode_rpc_replay_cache_capacity_resolved",
|
||||
component = "ecstore",
|
||||
subsystem = "rpc_auth",
|
||||
capacity = decision.capacity,
|
||||
source,
|
||||
cpu_count = decision.cpu_count,
|
||||
memory_limit_bytes = decision.memory_limit_bytes,
|
||||
memory_basis = decision.memory_basis.map(MemoryBasis::as_str),
|
||||
memory_based_capacity = decision.memory_based_capacity,
|
||||
cpu_based_capacity = decision.cpu_based_capacity,
|
||||
auto_max_capacity = REPLAY_CACHE_AUTO_MAX_CAPACITY,
|
||||
env = rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
|
||||
"internode rpc replay cache capacity auto-sized after invalid env"
|
||||
);
|
||||
return;
|
||||
}
|
||||
info!(
|
||||
event = "internode_rpc_replay_cache_capacity_resolved",
|
||||
component = "ecstore",
|
||||
subsystem = "rpc_auth",
|
||||
capacity = decision.capacity,
|
||||
source,
|
||||
cpu_count = decision.cpu_count,
|
||||
memory_limit_bytes = decision.memory_limit_bytes,
|
||||
memory_basis = decision.memory_basis.map(MemoryBasis::as_str),
|
||||
memory_based_capacity = decision.memory_based_capacity,
|
||||
cpu_based_capacity = decision.cpu_based_capacity,
|
||||
auto_max_capacity = REPLAY_CACHE_AUTO_MAX_CAPACITY,
|
||||
"internode rpc replay cache capacity resolved"
|
||||
);
|
||||
}
|
||||
|
||||
fn resolve_replay_cache_capacity() -> usize {
|
||||
let (cpu_count, memory_limit_bytes, memory_basis) = detected_replay_cache_resources();
|
||||
let decision = replay_cache_capacity_decision(
|
||||
rustfs_utils::get_env_parse_outcome(rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY),
|
||||
cpu_count,
|
||||
memory_limit_bytes,
|
||||
memory_basis,
|
||||
);
|
||||
global_internode_metrics().record_replay_cache_state(0, decision.capacity);
|
||||
log_replay_cache_capacity_decision(decision);
|
||||
decision.capacity
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RpcNonceCache {
|
||||
nonces: HashSet<Uuid>,
|
||||
@@ -317,50 +110,8 @@ struct RpcNonceCache {
|
||||
max_wall_time: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct RpcReplayCacheMetricScope<'a> {
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
rpc_path: &'a str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct RpcNonceRecord<'a> {
|
||||
nonce: Uuid,
|
||||
signed_at: i64,
|
||||
now: Instant,
|
||||
wall_time: i64,
|
||||
expires_at: Instant,
|
||||
capacity: usize,
|
||||
metric_scope: RpcReplayCacheMetricScope<'a>,
|
||||
}
|
||||
|
||||
struct RpcNonceCacheMetrics<'a> {
|
||||
expired: usize,
|
||||
entries: usize,
|
||||
capacity: usize,
|
||||
overflow_scope: Option<RpcReplayCacheMetricScope<'a>>,
|
||||
}
|
||||
|
||||
fn publish_nonce_cache_metrics(metrics: Option<RpcNonceCacheMetrics<'_>>) {
|
||||
let Some(metrics) = metrics else {
|
||||
return;
|
||||
};
|
||||
let internode_metrics = global_internode_metrics();
|
||||
internode_metrics.record_replay_cache_evictions("expired", metrics.expired);
|
||||
internode_metrics.record_replay_cache_state(metrics.entries, metrics.capacity);
|
||||
if let Some(scope) = metrics.overflow_scope {
|
||||
internode_metrics.record_replay_cache_overflow_for_operation_and_backend_path(
|
||||
scope.operation,
|
||||
scope.backend,
|
||||
scope.rpc_path,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl RpcNonceCache {
|
||||
fn remove_expired(&mut self, now: Instant, wall_time: i64) -> usize {
|
||||
let mut removed = 0;
|
||||
fn remove_expired(&mut self, now: Instant, wall_time: i64) {
|
||||
while matches!(
|
||||
self.expirations.front(),
|
||||
Some((expires_at, valid_until, _)) if *expires_at < now && *valid_until < wall_time
|
||||
@@ -369,48 +120,37 @@ impl RpcNonceCache {
|
||||
break;
|
||||
};
|
||||
self.nonces.remove(&nonce);
|
||||
removed += 1;
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
fn check_and_record<'a>(&mut self, record: RpcNonceRecord<'a>) -> (std::io::Result<()>, Option<RpcNonceCacheMetrics<'a>>) {
|
||||
self.max_wall_time = self.max_wall_time.max(record.wall_time);
|
||||
if self.max_wall_time.saturating_sub(record.signed_at) > SIGNATURE_VALID_DURATION {
|
||||
return (Err(std::io::Error::other("RPC request timestamp expired after clock regression")), None);
|
||||
fn check_and_record(
|
||||
&mut self,
|
||||
nonce: Uuid,
|
||||
signed_at: i64,
|
||||
now: Instant,
|
||||
wall_time: i64,
|
||||
expires_at: Instant,
|
||||
capacity: usize,
|
||||
) -> std::io::Result<()> {
|
||||
self.max_wall_time = self.max_wall_time.max(wall_time);
|
||||
if self.max_wall_time.saturating_sub(signed_at) > SIGNATURE_VALID_DURATION {
|
||||
return Err(std::io::Error::other("RPC request timestamp expired after clock regression"));
|
||||
}
|
||||
let expired = self.remove_expired(record.now, self.max_wall_time);
|
||||
let metrics = RpcNonceCacheMetrics {
|
||||
expired,
|
||||
entries: self.nonces.len(),
|
||||
capacity: record.capacity,
|
||||
overflow_scope: None,
|
||||
};
|
||||
if self.nonces.contains(&record.nonce) {
|
||||
return (Err(std::io::Error::other("RPC request replay detected")), Some(metrics));
|
||||
self.remove_expired(now, self.max_wall_time);
|
||||
if self.nonces.contains(&nonce) {
|
||||
return Err(std::io::Error::other("RPC request replay detected"));
|
||||
}
|
||||
if self.nonces.len() >= record.capacity {
|
||||
if self.nonces.len() >= capacity {
|
||||
// Fail closed and alert: only legitimately signed traffic can fill the cache, so a
|
||||
// sustained overflow means RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY is undersized
|
||||
// for this node's peak mutation rate and writes are being refused.
|
||||
return (
|
||||
Err(std::io::Error::other("RPC replay cache capacity exceeded")),
|
||||
Some(RpcNonceCacheMetrics {
|
||||
overflow_scope: Some(record.metric_scope),
|
||||
..metrics
|
||||
}),
|
||||
);
|
||||
global_internode_metrics().record_replay_cache_overflow();
|
||||
return Err(std::io::Error::other("RPC replay cache capacity exceeded"));
|
||||
}
|
||||
self.nonces.insert(record.nonce);
|
||||
self.nonces.insert(nonce);
|
||||
self.expirations
|
||||
.push_back((record.expires_at, record.signed_at.saturating_add(SIGNATURE_VALID_DURATION), record.nonce));
|
||||
(
|
||||
Ok(()),
|
||||
Some(RpcNonceCacheMetrics {
|
||||
entries: self.nonces.len(),
|
||||
..metrics
|
||||
}),
|
||||
)
|
||||
.push_back((expires_at, signed_at.saturating_add(SIGNATURE_VALID_DURATION), nonce));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -479,13 +219,6 @@ fn signature_payload(url: &str, method: &Method, timestamp: i64) -> String {
|
||||
format!("{url}|{method}|{timestamp}")
|
||||
}
|
||||
|
||||
fn canonical_path_and_query(url: &str) -> std::io::Result<String> {
|
||||
let uri: Uri = url.parse().map_err(|_| std::io::Error::other("Invalid RPC URL"))?;
|
||||
uri.path_and_query()
|
||||
.map(ToString::to_string)
|
||||
.ok_or_else(|| std::io::Error::other("Invalid RPC URL"))
|
||||
}
|
||||
|
||||
fn redacted_rpc_path(url: &str) -> String {
|
||||
url.parse::<Uri>()
|
||||
.ok()
|
||||
@@ -513,76 +246,6 @@ fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, si
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
|
||||
fn update_put_file_auth_mac(
|
||||
mac: &mut HmacSha256,
|
||||
url: &str,
|
||||
method: &Method,
|
||||
nonce: Uuid,
|
||||
body_sha256: &str,
|
||||
) -> std::io::Result<()> {
|
||||
if !valid_content_sha256(body_sha256) || body_sha256 == UNSIGNED_PAYLOAD {
|
||||
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
|
||||
}
|
||||
let path_and_query = canonical_path_and_query(url)?;
|
||||
mac.update(HTTP_PUT_FILE_AUTH_DOMAIN);
|
||||
for part in [
|
||||
path_and_query.as_bytes(),
|
||||
b"|",
|
||||
method.as_str().as_bytes(),
|
||||
b"|",
|
||||
nonce.as_bytes(),
|
||||
b"|",
|
||||
body_sha256.as_bytes(),
|
||||
] {
|
||||
mac.update(part);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn put_file_auth_mac(url: &str, method: &Method, nonce: Uuid, body_sha256: &str) -> std::io::Result<[u8; 32]> {
|
||||
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(get_shared_secret()?.as_bytes())
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_put_file_auth_mac(&mut mac, url, method, nonce, body_sha256)?;
|
||||
Ok(mac.finalize().into_bytes().into())
|
||||
}
|
||||
|
||||
fn verify_put_file_auth_mac(url: &str, method: &Method, nonce: Uuid, body_sha256: &str, signature: &[u8]) -> std::io::Result<()> {
|
||||
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(get_shared_secret()?.as_bytes())
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_put_file_auth_mac(&mut mac, url, method, nonce, body_sha256)?;
|
||||
mac.verify_slice(signature)
|
||||
.map_err(|_| std::io::Error::other("Invalid put_file auth trailer"))
|
||||
}
|
||||
|
||||
pub fn build_put_file_auth_trailer(url: &str, method: &Method, nonce: Uuid, body_sha256: &str) -> std::io::Result<Vec<u8>> {
|
||||
let mac = put_file_auth_mac(url, method, nonce, body_sha256)?;
|
||||
let mut trailer = Vec::with_capacity(PUT_FILE_AUTH_TRAILER_LEN);
|
||||
trailer.extend_from_slice(PUT_FILE_AUTH_TRAILER_MAGIC);
|
||||
trailer.extend_from_slice(body_sha256.as_bytes());
|
||||
trailer.extend_from_slice(&mac);
|
||||
Ok(trailer)
|
||||
}
|
||||
|
||||
pub fn verify_put_file_auth_trailer(url: &str, method: &Method, nonce: Uuid, trailer: &[u8]) -> std::io::Result<String> {
|
||||
if trailer.len() != PUT_FILE_AUTH_TRAILER_LEN {
|
||||
return Err(std::io::Error::other("Invalid put_file auth trailer length"));
|
||||
}
|
||||
if &trailer[..PUT_FILE_AUTH_TRAILER_MAGIC.len()] != PUT_FILE_AUTH_TRAILER_MAGIC {
|
||||
return Err(std::io::Error::other("Invalid put_file auth trailer"));
|
||||
}
|
||||
let digest_start = PUT_FILE_AUTH_TRAILER_MAGIC.len();
|
||||
let digest_end = digest_start + PUT_FILE_AUTH_TRAILER_DIGEST_LEN;
|
||||
let body_sha256 = std::str::from_utf8(&trailer[digest_start..digest_end])
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC content SHA-256"))?;
|
||||
if !valid_content_sha256(body_sha256) || body_sha256 == UNSIGNED_PAYLOAD {
|
||||
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
|
||||
}
|
||||
let mac_start = digest_end;
|
||||
let mac_end = mac_start + PUT_FILE_AUTH_TRAILER_MAC_LEN;
|
||||
verify_put_file_auth_mac(url, method, nonce, body_sha256, &trailer[mac_start..mac_end])?;
|
||||
Ok(body_sha256.to_string())
|
||||
}
|
||||
|
||||
fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) {
|
||||
mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
|
||||
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
|
||||
@@ -878,80 +541,18 @@ fn check_timestamp(timestamp: i64) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn tonic_rpc_metric_operation(path: &str) -> &'static str {
|
||||
match parse_tonic_rpc_path(path).ok().map(|(_, rpc_method)| rpc_method) {
|
||||
Some("ReadAll") => INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||
Some("ReadMultiple") => INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
Some("WriteAll") => INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
_ => INTERNODE_OPERATION_GRPC_OTHER,
|
||||
}
|
||||
}
|
||||
|
||||
fn check_and_record_nonce_with_scope(
|
||||
nonce: Uuid,
|
||||
signed_at: i64,
|
||||
rpc_path: &str,
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
) -> std::io::Result<()> {
|
||||
fn check_and_record_nonce(nonce: Uuid, signed_at: i64) -> std::io::Result<()> {
|
||||
let wall_time = OffsetDateTime::now_utc().unix_timestamp();
|
||||
let (result, metrics) = {
|
||||
let mut cache = LOCAL_RPC_NONCE_CACHE
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("RPC replay cache unavailable"))?;
|
||||
// Take the monotonic timestamp after acquiring the lock so expiration
|
||||
// entries remain ordered by the same serialization point as insertion.
|
||||
let now = Instant::now();
|
||||
let expires_at = now
|
||||
.checked_add(REPLAY_CACHE_RETENTION)
|
||||
.ok_or_else(|| std::io::Error::other("RPC replay expiry overflow"))?;
|
||||
cache.check_and_record(RpcNonceRecord {
|
||||
nonce,
|
||||
signed_at,
|
||||
now,
|
||||
wall_time,
|
||||
expires_at,
|
||||
capacity: *REPLAY_CACHE_CAPACITY,
|
||||
metric_scope: RpcReplayCacheMetricScope {
|
||||
operation,
|
||||
backend,
|
||||
rpc_path,
|
||||
},
|
||||
})
|
||||
};
|
||||
publish_nonce_cache_metrics(metrics);
|
||||
result
|
||||
}
|
||||
|
||||
fn check_and_record_tonic_nonce(nonce: Uuid, signed_at: i64, rpc_path: &str) -> std::io::Result<()> {
|
||||
check_and_record_nonce_with_scope(
|
||||
nonce,
|
||||
signed_at,
|
||||
rpc_path,
|
||||
tonic_rpc_metric_operation(rpc_path),
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn check_and_record_signed_rpc_nonce(
|
||||
headers: &HeaderMap,
|
||||
nonce: Uuid,
|
||||
rpc_path: &str,
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
) -> std::io::Result<()> {
|
||||
if nonce.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid RPC nonce"));
|
||||
}
|
||||
let timestamp_header = headers
|
||||
.get(TIMESTAMP_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
|
||||
let timestamp = timestamp_header
|
||||
.parse::<i64>()
|
||||
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
|
||||
check_timestamp(timestamp)?;
|
||||
check_and_record_nonce_with_scope(nonce, timestamp, rpc_path, operation, backend)
|
||||
let mut cache = LOCAL_RPC_NONCE_CACHE
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("RPC replay cache unavailable"))?;
|
||||
// Take the monotonic timestamp after acquiring the lock so expiration
|
||||
// entries remain ordered by the same serialization point as insertion.
|
||||
let now = Instant::now();
|
||||
let expires_at = now
|
||||
.checked_add(REPLAY_CACHE_RETENTION)
|
||||
.ok_or_else(|| std::io::Error::other("RPC replay expiry overflow"))?;
|
||||
cache.check_and_record(nonce, signed_at, now, wall_time, expires_at, *REPLAY_CACHE_CAPACITY)
|
||||
}
|
||||
|
||||
/// Build headers with authentication signature
|
||||
@@ -1213,7 +814,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
|
||||
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||
}
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)
|
||||
check_and_record_nonce(nonce, signed_at)
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
@@ -1246,48 +847,6 @@ pub fn verify_tonic_rpc_signature_with_bootstrap(
|
||||
)
|
||||
}
|
||||
|
||||
pub fn tonic_rpc_auth_failure_reason(error: &std::io::Error) -> &'static str {
|
||||
match error.to_string().as_str() {
|
||||
"Missing RPC audience" => "missing_audience",
|
||||
"Invalid RPC request path" => "invalid_request_path",
|
||||
"RPC replay-scoped authentication required" => "replay_scope_required",
|
||||
"Missing RPC replay scope version" => "missing_replay_scope_version",
|
||||
"Unsupported RPC replay scope version" => "unsupported_replay_scope_version",
|
||||
"Missing RPC replay scope signature" => "missing_replay_scope_signature",
|
||||
"Missing RPC replay scope nonce" => "missing_replay_scope_nonce",
|
||||
"Invalid RPC replay scope nonce" => "invalid_replay_scope_nonce",
|
||||
"Missing RPC boot epoch" => "missing_boot_epoch",
|
||||
"Invalid RPC boot epoch" => "invalid_boot_epoch",
|
||||
"Invalid RPC replay scope signature" => "invalid_replay_scope_signature",
|
||||
"RPC boot epoch is stale" => "stale_boot_epoch",
|
||||
"RPC request replay detected" => "replay_detected",
|
||||
"RPC replay cache capacity exceeded" => "replay_cache_capacity",
|
||||
"RPC replay cache unavailable" => "replay_cache_unavailable",
|
||||
"RPC replay expiry overflow" => "replay_expiry_overflow",
|
||||
"RPC request timestamp expired after clock regression" => "timestamp_expired_after_clock_regression",
|
||||
"RPC v2 authentication required" => "v2_required",
|
||||
"Missing RPC auth version" => "missing_v2_auth_version",
|
||||
"Unsupported RPC auth version" => "unsupported_v2_auth_version",
|
||||
"Missing RPC v2 signature" => "missing_v2_signature",
|
||||
"Invalid RPC v2 signature" => "invalid_v2_signature",
|
||||
"Missing timestamp header" => "missing_timestamp",
|
||||
"Invalid timestamp format" => "invalid_timestamp",
|
||||
"Request timestamp expired" => "timestamp_expired",
|
||||
"Missing RPC nonce" => "missing_v2_nonce",
|
||||
"Invalid RPC nonce" => "invalid_v2_nonce",
|
||||
"Invalid unsigned RPC nonce" => "invalid_unsigned_v2_nonce",
|
||||
"Missing RPC content SHA-256" => "missing_content_sha256",
|
||||
"Invalid RPC content SHA-256" => "invalid_content_sha256",
|
||||
"Invalid put_file auth trailer length" => "invalid_put_file_auth_trailer_length",
|
||||
"Invalid put_file auth trailer" => "invalid_put_file_auth_trailer",
|
||||
"Missing signature header" => "missing_v1_signature",
|
||||
"Invalid signature" => "invalid_v1_signature",
|
||||
"Invalid RPC HMAC key" => "invalid_hmac_key",
|
||||
message if message.contains(RPC_SECRET_REQUIRED_OPERATOR_MESSAGE) => "missing_rpc_secret",
|
||||
_ => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
fn verify_tonic_rpc_signature_with_policy(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
@@ -1406,7 +965,7 @@ fn verify_tonic_rpc_signature_with_strictness(
|
||||
return Err(std::io::Error::other("Invalid RPC v2 signature"));
|
||||
}
|
||||
if let Some(nonce) = parsed_nonce {
|
||||
check_and_record_tonic_nonce(nonce, timestamp, path)?;
|
||||
check_and_record_nonce(nonce, timestamp)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -2140,33 +1699,6 @@ mod tests {
|
||||
assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_rpc_auth_failure_reason_maps_security_relevant_errors() {
|
||||
for (message, reason) in [
|
||||
("Invalid RPC v2 signature", "invalid_v2_signature"),
|
||||
("RPC replay-scoped authentication required", "replay_scope_required"),
|
||||
("Missing RPC replay scope signature", "missing_replay_scope_signature"),
|
||||
("RPC boot epoch is stale", "stale_boot_epoch"),
|
||||
("RPC request replay detected", "replay_detected"),
|
||||
("Request timestamp expired", "timestamp_expired"),
|
||||
("Missing RPC content SHA-256", "missing_content_sha256"),
|
||||
("Invalid RPC content SHA-256", "invalid_content_sha256"),
|
||||
("Invalid put_file auth trailer length", "invalid_put_file_auth_trailer_length"),
|
||||
("Invalid put_file auth trailer", "invalid_put_file_auth_trailer"),
|
||||
] {
|
||||
assert_eq!(
|
||||
tonic_rpc_auth_failure_reason(&std::io::Error::other(message)),
|
||||
reason,
|
||||
"message {message:?} should map to a stable low-cardinality reason"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_rpc_auth_failure_reason_falls_back_for_unclassified_errors() {
|
||||
assert_eq!(tonic_rpc_auth_failure_reason(&std::io::Error::other("opaque failure")), "unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
|
||||
ensure_test_rpc_secret();
|
||||
@@ -2300,37 +1832,6 @@ mod tests {
|
||||
assert_eq!(tampered.to_string(), "RPC content SHA-256 mismatch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_file_auth_trailer_binds_url_nonce_and_body_digest() {
|
||||
ensure_test_rpc_secret();
|
||||
let url = concat!(
|
||||
"/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
|
||||
"&append=false&size=11&put_file_auth=digest-trailer-v1&put_file_nonce=11111111-2222-4333-8444-555555555555"
|
||||
);
|
||||
let nonce = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("nonce");
|
||||
let body_sha256 = hex_simd::encode_to_string(Sha256::digest(b"hello world"), hex_simd::AsciiCase::Lower);
|
||||
let trailer = build_put_file_auth_trailer(url, &Method::PUT, nonce, &body_sha256).expect("trailer should build");
|
||||
|
||||
assert_eq!(trailer.len(), PUT_FILE_AUTH_TRAILER_LEN);
|
||||
let verified = verify_put_file_auth_trailer(url, &Method::PUT, nonce, &trailer).expect("trailer should verify");
|
||||
assert_eq!(verified, body_sha256);
|
||||
|
||||
let different_url = url.replace("size=11", "size=12");
|
||||
let err = verify_put_file_auth_trailer(&different_url, &Method::PUT, nonce, &trailer)
|
||||
.expect_err("trailer must bind the signed URL");
|
||||
assert_eq!(err.to_string(), "Invalid put_file auth trailer");
|
||||
|
||||
let err =
|
||||
verify_put_file_auth_trailer(url, &Method::PUT, Uuid::new_v4(), &trailer).expect_err("trailer must bind the nonce");
|
||||
assert_eq!(err.to_string(), "Invalid put_file auth trailer");
|
||||
|
||||
let mut tampered = trailer;
|
||||
tampered[PUT_FILE_AUTH_TRAILER_MAGIC.len()] = b'0';
|
||||
let err =
|
||||
verify_put_file_auth_trailer(url, &Method::PUT, nonce, &tampered).expect_err("trailer must bind the digest bytes");
|
||||
assert_eq!(err.to_string(), "Invalid put_file auth trailer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_mutation_rpc_contract_requires_method_bound_v2_body_digest() {
|
||||
ensure_test_rpc_secret();
|
||||
@@ -2402,126 +1903,6 @@ mod tests {
|
||||
assert_eq!(error.to_string(), "RPC mutation requires v2 authentication");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_rpc_metric_operation_classifies_get_hot_path_methods() {
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/ReadAll"),
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/ReadMultiple"),
|
||||
INTERNODE_OPERATION_GRPC_READ_MULTIPLE
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/WriteAll"),
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/SignalService"),
|
||||
INTERNODE_OPERATION_GRPC_OTHER
|
||||
);
|
||||
assert_eq!(tonic_rpc_metric_operation("not-a-grpc-path"), INTERNODE_OPERATION_GRPC_OTHER);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_capacity_uses_env_with_default_floor() {
|
||||
let default = rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY;
|
||||
|
||||
let high = replay_cache_capacity_decision(
|
||||
rustfs_utils::EnvParseOutcome::Parsed(default * 16),
|
||||
2,
|
||||
Some(512 * 1024 * 1024),
|
||||
Some(MemoryBasis::Host),
|
||||
);
|
||||
assert_eq!(high.capacity, default * 16);
|
||||
assert_eq!(high.source, ReplayCacheCapacitySource::Env);
|
||||
|
||||
let low = replay_cache_capacity_decision(
|
||||
rustfs_utils::EnvParseOutcome::Parsed(1),
|
||||
64,
|
||||
Some(128 * 1024 * 1024 * 1024),
|
||||
Some(MemoryBasis::Host),
|
||||
);
|
||||
assert_eq!(low.capacity, default);
|
||||
assert_eq!(low.source, ReplayCacheCapacitySource::EnvClampedToDefault);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_capacity_auto_sizes_from_cpu_and_memory() {
|
||||
let gib = 1024_u64 * 1024 * 1024;
|
||||
let decision =
|
||||
replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Absent, 8, Some(16 * gib), Some(MemoryBasis::Host));
|
||||
|
||||
assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
|
||||
assert_eq!(decision.memory_basis, Some(MemoryBasis::Host));
|
||||
assert_eq!(decision.memory_based_capacity, 10_737_418);
|
||||
assert_eq!(decision.cpu_based_capacity, 9_846_784);
|
||||
assert_eq!(decision.capacity, 9_846_784);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_capacity_auto_reaches_hotpath_verified_capacity_on_larger_nodes() {
|
||||
let gib = 1024_u64 * 1024 * 1024;
|
||||
let decision =
|
||||
replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Absent, 16, Some(32 * gib), Some(MemoryBasis::Host));
|
||||
|
||||
assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
|
||||
assert_eq!(decision.memory_based_capacity, 21_474_836);
|
||||
assert_eq!(decision.cpu_based_capacity, 19_693_568);
|
||||
assert_eq!(decision.capacity, 16_777_216);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_capacity_auto_keeps_default_floor_for_small_nodes() {
|
||||
let decision = replay_cache_capacity_decision(
|
||||
rustfs_utils::EnvParseOutcome::Absent,
|
||||
1,
|
||||
Some(512 * 1024 * 1024),
|
||||
Some(MemoryBasis::Host),
|
||||
);
|
||||
|
||||
assert_eq!(decision.capacity, rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY);
|
||||
assert_eq!(decision.source, ReplayCacheCapacitySource::AutoClampedToDefault);
|
||||
assert!(decision.memory_based_capacity < rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_capacity_invalid_env_uses_auto_sizing() {
|
||||
let decision = replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Invalid, 8, None, None);
|
||||
|
||||
assert_eq!(decision.source, ReplayCacheCapacitySource::AutoInvalidEnv);
|
||||
assert_eq!(decision.capacity, 9_846_784);
|
||||
}
|
||||
|
||||
fn check_test_nonce_record(cache: &mut RpcNonceCache, record: RpcNonceRecord<'_>) -> std::io::Result<()> {
|
||||
let (result, metrics) = cache.check_and_record(record);
|
||||
publish_nonce_cache_metrics(metrics);
|
||||
result
|
||||
}
|
||||
|
||||
fn test_nonce_record(
|
||||
nonce: Uuid,
|
||||
signed_at: i64,
|
||||
now: Instant,
|
||||
wall_time: i64,
|
||||
expires_at: Instant,
|
||||
capacity: usize,
|
||||
) -> RpcNonceRecord<'static> {
|
||||
RpcNonceRecord {
|
||||
nonce,
|
||||
signed_at,
|
||||
now,
|
||||
wall_time,
|
||||
expires_at,
|
||||
capacity,
|
||||
metric_scope: RpcReplayCacheMetricScope {
|
||||
operation: INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||
backend: INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
rpc_path: "/node_service.NodeService/ReadAll",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_expires_by_monotonic_deadline_and_fails_closed_at_capacity() {
|
||||
let now = Instant::now();
|
||||
@@ -2531,12 +1912,15 @@ mod tests {
|
||||
let nonce_b = Uuid::new_v4();
|
||||
let mut cache = RpcNonceCache::default();
|
||||
|
||||
check_test_nonce_record(&mut cache, test_nonce_record(nonce_a, 100, now, 100, expiry, 1))
|
||||
cache
|
||||
.check_and_record(nonce_a, 100, now, 100, expiry, 1)
|
||||
.expect("first nonce should be recorded");
|
||||
let capacity = check_test_nonce_record(&mut cache, test_nonce_record(nonce_b, 100, now, 100, expiry, 1))
|
||||
let capacity = cache
|
||||
.check_and_record(nonce_b, 100, now, 100, expiry, 1)
|
||||
.expect_err("a full replay cache must fail closed");
|
||||
assert_eq!(capacity.to_string(), "RPC replay cache capacity exceeded");
|
||||
check_test_nonce_record(&mut cache, test_nonce_record(nonce_b, 702, after_expiry, 702, after_expiry, 1))
|
||||
cache
|
||||
.check_and_record(nonce_b, 702, after_expiry, 702, after_expiry, 1)
|
||||
.expect("expired nonce should release capacity");
|
||||
assert!(!cache.nonces.contains(&nonce_a));
|
||||
assert!(cache.nonces.contains(&nonce_b));
|
||||
@@ -2739,15 +2123,17 @@ mod tests {
|
||||
let nonce = Uuid::new_v4();
|
||||
let mut cache = RpcNonceCache::default();
|
||||
|
||||
check_test_nonce_record(&mut cache, test_nonce_record(nonce, 1_000, now, 1_000, expiry, 2))
|
||||
cache
|
||||
.check_and_record(nonce, 1_000, now, 1_000, expiry, 2)
|
||||
.expect("first nonce should be recorded");
|
||||
let replay = check_test_nonce_record(&mut cache, test_nonce_record(nonce, 1_000, after_expiry, 900, after_expiry, 2))
|
||||
let replay = cache
|
||||
.check_and_record(nonce, 1_000, after_expiry, 900, after_expiry, 2)
|
||||
.expect_err("wall clock regression must not make an old signature reusable");
|
||||
assert_eq!(replay.to_string(), "RPC request replay detected");
|
||||
|
||||
let stale =
|
||||
check_test_nonce_record(&mut cache, test_nonce_record(Uuid::new_v4(), 600, after_expiry, 900, after_expiry, 2))
|
||||
.expect_err("the monotonic wall-clock high-water mark must fail closed");
|
||||
let stale = cache
|
||||
.check_and_record(Uuid::new_v4(), 600, after_expiry, 900, after_expiry, 2)
|
||||
.expect_err("the monotonic wall-clock high-water mark must fail closed");
|
||||
assert_eq!(stale.to_string(), "RPC request timestamp expired after clock regression");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,15 +12,14 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::{build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability};
|
||||
use crate::cluster::rpc::{build_auth_headers, verify_ns_scanner_capability};
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::disk::{FileReader, FileWriter};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
|
||||
NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
|
||||
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY,
|
||||
PUT_FILE_AUTH_V1, PUT_FILE_NONCE_QUERY, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
|
||||
WALK_DIR_STREAM_COMPLETION_V1,
|
||||
NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY,
|
||||
WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
|
||||
@@ -30,11 +29,9 @@ use rustfs_config::{
|
||||
};
|
||||
use rustfs_rio::{HttpReader, HttpWriter};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWrite};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
static INTERNODE_DATA_TRANSPORT: OnceLock<std::result::Result<Arc<dyn InternodeDataTransport>, String>> = OnceLock::new();
|
||||
@@ -169,12 +166,10 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
}
|
||||
|
||||
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
|
||||
let nonce = Uuid::new_v4();
|
||||
let url = build_put_file_stream_url(&request, Some(nonce));
|
||||
let url = build_put_file_stream_url(&request);
|
||||
let mut headers = json_headers();
|
||||
build_auth_headers(&url, &Method::PUT, &mut headers)?;
|
||||
let writer = HttpWriter::new(url.clone(), Method::PUT, headers).await?;
|
||||
Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce)))
|
||||
Ok(Box::new(HttpWriter::new(url, Method::PUT, headers).await?))
|
||||
}
|
||||
|
||||
async fn open_walk_dir(&self, request: WalkDirStreamRequest) -> Result<FileReader> {
|
||||
@@ -241,8 +236,8 @@ fn build_read_file_stream_url(request: &ReadStreamRequest) -> String {
|
||||
)
|
||||
}
|
||||
|
||||
fn build_put_file_stream_url(request: &WriteStreamRequest, auth_nonce: Option<Uuid>) -> String {
|
||||
let mut url = format!(
|
||||
fn build_put_file_stream_url(request: &WriteStreamRequest) -> String {
|
||||
format!(
|
||||
"{}{}?disk={}&volume={}&path={}&append={}&size={}",
|
||||
request.endpoint,
|
||||
PUT_FILE_STREAM_PATH,
|
||||
@@ -251,104 +246,7 @@ fn build_put_file_stream_url(request: &WriteStreamRequest, auth_nonce: Option<Uu
|
||||
urlencoding::encode(&request.path),
|
||||
request.append,
|
||||
request.size
|
||||
);
|
||||
if let Some(nonce) = auth_nonce {
|
||||
url.push_str(&format!(
|
||||
"&{}={}&{}={}",
|
||||
PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1, PUT_FILE_NONCE_QUERY, nonce
|
||||
));
|
||||
}
|
||||
url
|
||||
}
|
||||
|
||||
struct PutFileAuthWriter<W> {
|
||||
inner: W,
|
||||
url: String,
|
||||
nonce: Uuid,
|
||||
hasher: Sha256,
|
||||
trailer: Option<Vec<u8>>,
|
||||
trailer_offset: usize,
|
||||
}
|
||||
|
||||
impl<W> PutFileAuthWriter<W> {
|
||||
fn new(inner: W, url: String, nonce: Uuid) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
url,
|
||||
nonce,
|
||||
hasher: Sha256::new(),
|
||||
trailer: None,
|
||||
trailer_offset: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_trailer(&mut self) -> std::io::Result<()> {
|
||||
if self.trailer.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
let digest = hex_simd::encode_to_string(self.hasher.clone().finalize(), hex_simd::AsciiCase::Lower);
|
||||
self.trailer = Some(build_put_file_auth_trailer(&self.url, &Method::PUT, self.nonce, &digest)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn poll_write_trailer(&mut self, cx: &mut Context<'_>) -> Poll<std::io::Result<()>>
|
||||
where
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
self.ensure_trailer()?;
|
||||
let Some(trailer) = self.trailer.as_ref() else {
|
||||
return Poll::Ready(Err(std::io::Error::other("put_file auth trailer missing")));
|
||||
};
|
||||
while self.trailer_offset < trailer.len() {
|
||||
let written = match Pin::new(&mut self.inner).poll_write(cx, &trailer[self.trailer_offset..]) {
|
||||
Poll::Ready(Ok(0)) => {
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WriteZero,
|
||||
"failed to write put_file auth trailer",
|
||||
)));
|
||||
}
|
||||
Poll::Ready(Ok(written)) => written,
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
};
|
||||
self.trailer_offset += written;
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl<W> AsyncWrite for PutFileAuthWriter<W>
|
||||
where
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
if self.trailer.is_some() {
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::BrokenPipe,
|
||||
"cannot write after put_file auth trailer",
|
||||
)));
|
||||
}
|
||||
match Pin::new(&mut self.inner).poll_write(cx, buf) {
|
||||
Poll::Ready(Ok(written)) => {
|
||||
self.hasher.update(&buf[..written]);
|
||||
Poll::Ready(Ok(written))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
match self.poll_write_trailer(cx) {
|
||||
Poll::Ready(Ok(())) => {}
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
|
||||
@@ -557,17 +455,14 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn put_file_stream_url_encodes_query_values() {
|
||||
let url = build_put_file_stream_url(
|
||||
&WriteStreamRequest {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
disk: "http://node1:9000/data/rustfs0".to_string(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object/part.1".to_string(),
|
||||
append: false,
|
||||
size: 4096,
|
||||
},
|
||||
None,
|
||||
);
|
||||
let url = build_put_file_stream_url(&WriteStreamRequest {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
disk: "http://node1:9000/data/rustfs0".to_string(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object/part.1".to_string(),
|
||||
append: false,
|
||||
size: 4096,
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
url,
|
||||
@@ -575,63 +470,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_file_stream_url_advertises_auth_nonce_when_enabled() {
|
||||
let nonce = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("nonce");
|
||||
let url = build_put_file_stream_url(
|
||||
&WriteStreamRequest {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
disk: "http://node1:9000/data/rustfs0".to_string(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object/part.1".to_string(),
|
||||
append: false,
|
||||
size: 4096,
|
||||
},
|
||||
Some(nonce),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
url,
|
||||
concat!(
|
||||
"http://node1:9000/rustfs/rpc/put_file_stream?disk=http%3A%2F%2Fnode1%3A9000%2Fdata%2Frustfs0",
|
||||
"&volume=bucket&path=object%2Fpart.1&append=false&size=4096",
|
||||
"&put_file_auth=digest-trailer-v1&put_file_nonce=11111111-2222-4333-8444-555555555555"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_file_auth_writer_appends_trailer_on_shutdown() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("put-file-auth-writer-test-secret".to_string());
|
||||
let nonce = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("nonce");
|
||||
let url = concat!(
|
||||
"http://node1:9000/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
|
||||
"&append=false&size=11&put_file_auth=digest-trailer-v1&put_file_nonce=11111111-2222-4333-8444-555555555555"
|
||||
)
|
||||
.to_string();
|
||||
let mut sink = Vec::new();
|
||||
|
||||
{
|
||||
let mut writer = PutFileAuthWriter::new(&mut sink, url.clone(), nonce);
|
||||
writer.write_all(b"hello world").await.expect("body write should succeed");
|
||||
writer.shutdown().await.expect("shutdown should append auth trailer");
|
||||
let err = writer
|
||||
.write_all(b"!")
|
||||
.await
|
||||
.expect_err("post-trailer writes must be rejected");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::BrokenPipe);
|
||||
}
|
||||
|
||||
assert_eq!(&sink[..11], b"hello world");
|
||||
let trailer = &sink[11..];
|
||||
let expected_digest = hex_simd::encode_to_string(Sha256::digest(b"hello world"), hex_simd::AsciiCase::Lower);
|
||||
let verified = crate::cluster::rpc::verify_put_file_auth_trailer(&url, &Method::PUT, nonce, trailer)
|
||||
.expect("emitted trailer should verify");
|
||||
assert_eq!(verified, expected_digest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn walk_dir_url_encodes_disk_ref() {
|
||||
let url = build_walk_dir_url(&WalkDirStreamRequest {
|
||||
|
||||
@@ -32,10 +32,9 @@ pub use client::{
|
||||
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
|
||||
#[allow(unused_imports)]
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
set_tonic_mutation_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
|
||||
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_put_file_auth_trailer,
|
||||
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
|
||||
@@ -99,69 +99,6 @@ impl DeleteBucketEmptyScanBarrier {
|
||||
#[cfg(test)]
|
||||
static DELETE_BUCKET_EMPTY_SCAN_BARRIER: StdMutex<Option<Arc<DeleteBucketEmptyScanBarrier>>> = StdMutex::new(None);
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
enum HealBucketOperation {
|
||||
Make,
|
||||
Delete,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct HealBucketOperationFailure {
|
||||
bucket: String,
|
||||
disk_index: usize,
|
||||
operation: HealBucketOperation,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type HealBucketOperationFailureKey = (String, usize, HealBucketOperation);
|
||||
|
||||
#[cfg(test)]
|
||||
fn heal_bucket_operation_failures() -> &'static StdMutex<HashMap<HealBucketOperationFailureKey, Error>> {
|
||||
static FAILURES: std::sync::OnceLock<StdMutex<HashMap<HealBucketOperationFailureKey, Error>>> = std::sync::OnceLock::new();
|
||||
FAILURES.get_or_init(|| StdMutex::new(HashMap::new()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl HealBucketOperationFailure {
|
||||
fn install(bucket: &str, disk_index: usize, operation: HealBucketOperation, error: Error) -> Self {
|
||||
let key = (bucket.to_string(), disk_index, operation);
|
||||
let previous = heal_bucket_operation_failures()
|
||||
.lock()
|
||||
.expect("heal bucket failure registry should not poison")
|
||||
.insert(key, error);
|
||||
assert!(previous.is_none(), "heal bucket operation failure already installed");
|
||||
Self {
|
||||
bucket: bucket.to_string(),
|
||||
disk_index,
|
||||
operation,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for HealBucketOperationFailure {
|
||||
fn drop(&mut self) {
|
||||
heal_bucket_operation_failures()
|
||||
.lock()
|
||||
.expect("heal bucket failure registry should not poison")
|
||||
.remove(&(self.bucket.clone(), self.disk_index, self.operation));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn injected_heal_bucket_operation_error(bucket: &str, disk_index: usize, operation: HealBucketOperation) -> Option<Error> {
|
||||
heal_bucket_operation_failures()
|
||||
.lock()
|
||||
.expect("heal bucket failure registry should not poison")
|
||||
.get(&(bucket.to_string(), disk_index, operation))
|
||||
.cloned()
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn injected_heal_bucket_operation_error(_bucket: &str, _disk_index: usize, _operation: HealBucketOperation) -> Option<Error> {
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn install_delete_bucket_empty_scan_barrier() -> Arc<DeleteBucketEmptyScanBarrier> {
|
||||
let barrier = Arc::new(DeleteBucketEmptyScanBarrier::default());
|
||||
@@ -1270,6 +1207,10 @@ pub(crate) async fn heal_bucket_local_on_disks(
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if opts.dry_run {
|
||||
return Ok(res);
|
||||
}
|
||||
|
||||
for (disk, state) in disks.iter().zip(before_state.read().await.iter()) {
|
||||
res.before.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
@@ -1278,68 +1219,35 @@ pub(crate) async fn heal_bucket_local_on_disks(
|
||||
});
|
||||
}
|
||||
|
||||
if opts.dry_run {
|
||||
for (disk, state) in disks.iter().zip(after_state.read().await.iter()) {
|
||||
res.after.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
endpoint: disk.clone().map(|s| s.to_string()).unwrap_or_default(),
|
||||
state: state.to_string(),
|
||||
});
|
||||
}
|
||||
return Ok(res);
|
||||
}
|
||||
|
||||
let mut operation_error = errs
|
||||
.iter()
|
||||
.filter_map(|err| match err {
|
||||
Some(Error::VolumeNotFound) | None => None,
|
||||
Some(err) => Some(err.clone()),
|
||||
})
|
||||
.next();
|
||||
|
||||
if opts.remove && !bucket.starts_with(disk::RUSTFS_META_BUCKET) && !is_all_buckets_not_found(&errs) {
|
||||
let mut futures = Vec::new();
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
if matches!(errs[index].as_ref(), Some(Error::DiskNotFound | Error::VolumeNotFound)) {
|
||||
continue;
|
||||
}
|
||||
let Some(disk) = disk.clone() else {
|
||||
continue;
|
||||
};
|
||||
for disk in disks.iter() {
|
||||
let disk = disk.clone();
|
||||
let bucket = bucket.to_string();
|
||||
info!("heal_bucket_local, errs: {:?}, opts: {:?}", errs, opts);
|
||||
futures.push(async move {
|
||||
if let Some(err) = injected_heal_bucket_operation_error(&bucket, index, HealBucketOperation::Delete) {
|
||||
return (index, Err(err));
|
||||
match disk {
|
||||
Some(disk) => {
|
||||
// Non-force: a bucket that still holds object data refuses
|
||||
// deletion (VolumeNotEmpty) instead of being recursively
|
||||
// wiped, so a misclassified "dangling" bucket cannot lose
|
||||
// data (backlog#799 B1). Surface that refusal instead of
|
||||
// discarding it — it signals the bucket is not dangling.
|
||||
match disk.delete_volume(&bucket, false).await {
|
||||
Ok(()) => None,
|
||||
Err(Error::VolumeNotEmpty) => {
|
||||
warn!("heal declined to remove non-empty bucket {bucket} (not dangling)");
|
||||
None
|
||||
}
|
||||
Err(e) => Some(e),
|
||||
}
|
||||
}
|
||||
None => Some(Error::DiskNotFound),
|
||||
}
|
||||
(index, disk.delete_volume(&bucket, false).await)
|
||||
});
|
||||
}
|
||||
|
||||
for (index, result) in join_all(futures).await {
|
||||
match result {
|
||||
Ok(()) | Err(Error::VolumeNotFound) => {
|
||||
after_state.write().await[index] = DriveState::Missing.to_string();
|
||||
}
|
||||
Err(Error::VolumeNotEmpty) => {
|
||||
warn!(
|
||||
bucket,
|
||||
operation = "heal_bucket_delete_volume",
|
||||
result = "preserved_non_empty_bucket",
|
||||
"heal declined to remove non-empty bucket"
|
||||
);
|
||||
after_state.write().await[index] = DriveState::Ok.to_string();
|
||||
}
|
||||
Err(err) => {
|
||||
after_state.write().await[index] = match &err {
|
||||
Error::DiskNotFound => DriveState::Offline.to_string(),
|
||||
_ => DriveState::Corrupt.to_string(),
|
||||
};
|
||||
if operation_error.is_none() {
|
||||
operation_error = Some(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let _ = join_all(futures).await;
|
||||
}
|
||||
|
||||
if !opts.remove {
|
||||
@@ -1348,56 +1256,41 @@ pub(crate) async fn heal_bucket_local_on_disks(
|
||||
let disk = disk.clone();
|
||||
let bucket = bucket.to_string();
|
||||
let bs_clone = before_state.clone();
|
||||
let as_clone = after_state.clone();
|
||||
let errs_clone = errs.to_vec();
|
||||
futures.push(async move {
|
||||
if bs_clone.read().await[idx] == DriveState::Missing.to_string() {
|
||||
let Some(disk) = disk.as_ref() else {
|
||||
return (idx, Some(Error::DiskNotFound));
|
||||
return Some(Error::DiskNotFound);
|
||||
};
|
||||
|
||||
if let Some(err) = injected_heal_bucket_operation_error(&bucket, idx, HealBucketOperation::Make) {
|
||||
return (idx, Some(err));
|
||||
}
|
||||
info!("bucket not find, will recreate");
|
||||
match disk.make_volume(&bucket).await {
|
||||
Ok(()) | Err(Error::VolumeExists) => return (idx, None),
|
||||
Err(err) => return (idx, Some(err)),
|
||||
Ok(_) => {
|
||||
as_clone.write().await[idx] = DriveState::Ok.to_string();
|
||||
return None;
|
||||
}
|
||||
Err(err) => {
|
||||
return Some(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
(idx, None)
|
||||
errs_clone[idx].clone()
|
||||
});
|
||||
}
|
||||
|
||||
for (index, result) in join_all(futures).await {
|
||||
match result {
|
||||
None => {
|
||||
if before_state.read().await[index] == DriveState::Missing.to_string() {
|
||||
after_state.write().await[index] = DriveState::Ok.to_string();
|
||||
}
|
||||
}
|
||||
Some(err) => {
|
||||
after_state.write().await[index] = match &err {
|
||||
Error::DiskNotFound => DriveState::Offline.to_string(),
|
||||
_ => DriveState::Corrupt.to_string(),
|
||||
};
|
||||
if operation_error.is_none() {
|
||||
operation_error = Some(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let _ = join_all(futures).await;
|
||||
}
|
||||
|
||||
for (disk, state) in disks.iter().zip(after_state.read().await.iter()) {
|
||||
res.after.drives.push(HealDriveInfo {
|
||||
res.before.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
endpoint: disk.clone().map(|s| s.to_string()).unwrap_or_default(),
|
||||
state: state.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
match operation_error {
|
||||
Some(err) => Err(err),
|
||||
None => Ok(res),
|
||||
}
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
async fn clone_drives() -> Vec<Option<DiskStore>> {
|
||||
@@ -1863,7 +1756,7 @@ mod tests {
|
||||
.await
|
||||
.expect_err("second disk should start missing the bucket");
|
||||
|
||||
let result = heal_bucket_local(
|
||||
heal_bucket_local(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
@@ -1873,25 +1766,6 @@ mod tests {
|
||||
.await
|
||||
.expect("bucket heal should recreate missing volumes");
|
||||
|
||||
assert_eq!(result.before.drives.len(), 2);
|
||||
assert_eq!(result.after.drives.len(), 2);
|
||||
assert!(
|
||||
result
|
||||
.before
|
||||
.drives
|
||||
.iter()
|
||||
.any(|drive| drive.state == DriveState::Missing.to_string()),
|
||||
"one bucket volume must be reported missing before heal"
|
||||
);
|
||||
assert!(
|
||||
result
|
||||
.after
|
||||
.drives
|
||||
.iter()
|
||||
.all(|drive| drive.state == DriveState::Ok.to_string()),
|
||||
"all bucket volumes must be reported healthy after heal"
|
||||
);
|
||||
|
||||
for disk in disks {
|
||||
disk.stat_volume(bucket).await.expect("bucket should exist after heal");
|
||||
}
|
||||
@@ -1899,166 +1773,6 @@ mod tests {
|
||||
reset_local_disk_test_state().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn heal_bucket_local_dry_run_reports_discovered_drive_states() {
|
||||
reset_local_disk_test_state().await;
|
||||
|
||||
let temp_dir = TempDir::new().expect("create temp dir for bucket heal dry-run regression");
|
||||
let disks = init_test_local_disks(&temp_dir, 2, "heal-bucket-local-dry-run-reports-state").await;
|
||||
let bucket = "dry-run-healed-bucket";
|
||||
disks[0]
|
||||
.make_volume(bucket)
|
||||
.await
|
||||
.expect("bucket should exist on the first disk");
|
||||
|
||||
let result = heal_bucket_local_on_disks(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
dry_run: true,
|
||||
..Default::default()
|
||||
},
|
||||
vec![Some(disks[0].clone()), Some(disks[1].clone()), None],
|
||||
)
|
||||
.await
|
||||
.expect("dry-run bucket heal should inspect disks");
|
||||
|
||||
assert_eq!(result.before.drives.len(), 3);
|
||||
assert_eq!(result.after.drives.len(), 3);
|
||||
assert_eq!(result.before.drives[0].state, DriveState::Ok.to_string());
|
||||
assert_eq!(result.before.drives[1].state, DriveState::Missing.to_string());
|
||||
assert_eq!(result.before.drives[2].state, DriveState::Offline.to_string());
|
||||
for (before, after) in result.before.drives.iter().zip(&result.after.drives) {
|
||||
assert_eq!(after.endpoint, before.endpoint);
|
||||
assert_eq!(after.state, before.state);
|
||||
}
|
||||
assert!(matches!(disks[1].stat_volume(bucket).await, Err(Error::VolumeNotFound)));
|
||||
|
||||
reset_local_disk_test_state().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn heal_bucket_local_propagates_recreate_failure() {
|
||||
reset_local_disk_test_state().await;
|
||||
|
||||
let temp_dir = TempDir::new().expect("create temp dir for bucket recreate failure regression");
|
||||
let disks = init_test_local_disks(&temp_dir, 2, "heal-bucket-local-propagates-recreate-failure").await;
|
||||
let bucket = "recreate-failure-bucket";
|
||||
disks[0]
|
||||
.make_volume(bucket)
|
||||
.await
|
||||
.expect("bucket should exist on the first disk");
|
||||
let _failure = HealBucketOperationFailure::install(bucket, 1, HealBucketOperation::Make, Error::DiskAccessDenied);
|
||||
|
||||
let error = heal_bucket_local_on_disks(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
disks.iter().cloned().map(Some).collect(),
|
||||
)
|
||||
.await
|
||||
.expect_err("failed volume recreation must fail bucket heal");
|
||||
|
||||
assert_eq!(error, Error::DiskAccessDenied);
|
||||
assert!(matches!(disks[1].stat_volume(bucket).await, Err(Error::VolumeNotFound)));
|
||||
|
||||
reset_local_disk_test_state().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn heal_bucket_local_propagates_delete_failure() {
|
||||
reset_local_disk_test_state().await;
|
||||
|
||||
let temp_dir = TempDir::new().expect("create temp dir for bucket delete failure regression");
|
||||
let disks = init_test_local_disks(&temp_dir, 2, "heal-bucket-local-propagates-delete-failure").await;
|
||||
let bucket = "delete-failure-bucket";
|
||||
disks[0]
|
||||
.make_volume(bucket)
|
||||
.await
|
||||
.expect("bucket should exist on the first disk");
|
||||
let _failure = HealBucketOperationFailure::install(bucket, 0, HealBucketOperation::Delete, Error::DiskAccessDenied);
|
||||
|
||||
let error = heal_bucket_local_on_disks(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
remove: true,
|
||||
..Default::default()
|
||||
},
|
||||
disks.iter().cloned().map(Some).collect(),
|
||||
)
|
||||
.await
|
||||
.expect_err("failed volume deletion must fail bucket heal");
|
||||
|
||||
assert_eq!(error, Error::DiskAccessDenied);
|
||||
disks[0]
|
||||
.stat_volume(bucket)
|
||||
.await
|
||||
.expect("failed deletion must leave the bucket volume present");
|
||||
|
||||
reset_local_disk_test_state().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn heal_bucket_local_preserves_non_empty_bucket() {
|
||||
reset_local_disk_test_state().await;
|
||||
|
||||
let temp_dir = TempDir::new().expect("create temp dir for non-empty bucket heal regression");
|
||||
let disks = init_test_local_disks(&temp_dir, 1, "heal-bucket-local-preserves-non-empty").await;
|
||||
let bucket = "non-empty-bucket";
|
||||
disks[0]
|
||||
.make_volume(bucket)
|
||||
.await
|
||||
.expect("bucket should exist on the first disk");
|
||||
let _failure = HealBucketOperationFailure::install(bucket, 0, HealBucketOperation::Delete, Error::VolumeNotEmpty);
|
||||
|
||||
let result = heal_bucket_local_on_disks(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
remove: true,
|
||||
..Default::default()
|
||||
},
|
||||
disks.iter().cloned().map(Some).collect(),
|
||||
)
|
||||
.await
|
||||
.expect("a non-empty bucket refusal is an expected safety result");
|
||||
|
||||
assert_eq!(result.after.drives.len(), 1);
|
||||
assert_eq!(result.after.drives[0].state, DriveState::Ok.to_string());
|
||||
disks[0]
|
||||
.stat_volume(bucket)
|
||||
.await
|
||||
.expect("the non-empty bucket must remain present");
|
||||
|
||||
reset_local_disk_test_state().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn heal_bucket_local_propagates_preexisting_offline_disk() {
|
||||
reset_local_disk_test_state().await;
|
||||
|
||||
let temp_dir = TempDir::new().expect("create temp dir for offline bucket heal regression");
|
||||
let disks = init_test_local_disks(&temp_dir, 1, "heal-bucket-local-preexisting-offline").await;
|
||||
let bucket = "offline-disk-bucket";
|
||||
disks[0]
|
||||
.make_volume(bucket)
|
||||
.await
|
||||
.expect("bucket should exist on the online disk");
|
||||
|
||||
let error = heal_bucket_local_on_disks(bucket, &HealOpts::default(), vec![Some(disks[0].clone()), None])
|
||||
.await
|
||||
.expect_err("a prepass offline disk must keep the bucket heal incomplete");
|
||||
|
||||
assert_eq!(error, Error::DiskNotFound);
|
||||
|
||||
reset_local_disk_test_state().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reduce_pool_write_quorum_uses_only_pool_participants() {
|
||||
let clients = vec![
|
||||
|
||||
@@ -1117,29 +1117,13 @@ impl RemoteDisk {
|
||||
}
|
||||
|
||||
/// Initial capacity hint (bytes) for msgpack encode buffers, sized to cover a typical single-
|
||||
/// request without repeated growth reallocations. Larger payloads still grow as needed.
|
||||
/// version `FileInfo` without repeated growth reallocations. Larger payloads still grow as needed.
|
||||
const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512;
|
||||
const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024;
|
||||
|
||||
fn encode_msgpack_with_capacity<T: Serialize>(value: &T, capacity: usize) -> Result<Vec<u8>> {
|
||||
let mut serializer = rmp_serde::Serializer::new(Vec::with_capacity(capacity));
|
||||
value.serialize(&mut serializer)?;
|
||||
Ok(serializer.into_inner())
|
||||
}
|
||||
|
||||
fn encode_msgpack<T: Serialize>(value: &T) -> Result<Vec<u8>> {
|
||||
encode_msgpack_with_capacity(value, MSGPACK_ENCODE_CAPACITY_HINT)
|
||||
}
|
||||
|
||||
fn encode_file_info_msgpack(value: &FileInfo) -> Result<Vec<u8>> {
|
||||
encode_msgpack_with_capacity(value, FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT)
|
||||
}
|
||||
|
||||
fn encode_file_info_versions_msgpack(value: &FileInfoVersions) -> Result<Vec<u8>> {
|
||||
let version_count = value.versions.len().saturating_add(value.free_versions.len());
|
||||
let capacity =
|
||||
MSGPACK_ENCODE_CAPACITY_HINT.saturating_add(FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT.saturating_mul(version_count));
|
||||
encode_msgpack_with_capacity(value, capacity)
|
||||
let mut serializer = rmp_serde::Serializer::new(Vec::with_capacity(MSGPACK_ENCODE_CAPACITY_HINT));
|
||||
value.serialize(&mut serializer)?;
|
||||
Ok(serializer.into_inner())
|
||||
}
|
||||
|
||||
/// JSON compatibility string for a dual-encoded (`_bin` + text) request field. Returns an empty
|
||||
@@ -1152,6 +1136,12 @@ fn compat_json<T: Serialize>(value: &T) -> Result<String> {
|
||||
Ok(serde_json::to_string(value)?)
|
||||
}
|
||||
|
||||
fn encode_msgpack_named<T: Serialize>(value: &T) -> Result<Vec<u8>> {
|
||||
let mut serializer = rmp_serde::Serializer::new(Vec::with_capacity(MSGPACK_ENCODE_CAPACITY_HINT)).with_struct_map();
|
||||
value.serialize(&mut serializer)?;
|
||||
Ok(serializer.into_inner())
|
||||
}
|
||||
|
||||
fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_name: &'static str) -> Result<T> {
|
||||
if !binary.is_empty() {
|
||||
let mut deserializer = rmp_serde::Deserializer::new(Cursor::new(binary));
|
||||
@@ -1590,7 +1580,7 @@ impl DiskAPI for RemoteDisk {
|
||||
|| async {
|
||||
// `_bin` support for DeleteVersion is new (grpc-optimization P2); always dual-write
|
||||
// JSON + msgpack until its fallback counter has read zero across a release window.
|
||||
let file_info_bin = encode_file_info_msgpack(&fi)?;
|
||||
let file_info_bin = encode_msgpack(&fi)?;
|
||||
let opts_bin = encode_msgpack(&opts)?;
|
||||
let file_info = serde_json::to_string(&fi)?;
|
||||
let opts = serde_json::to_string(&opts)?;
|
||||
@@ -1680,7 +1670,7 @@ impl DiskAPI for RemoteDisk {
|
||||
return errors;
|
||||
}
|
||||
});
|
||||
versions_bin.push(match encode_file_info_versions_msgpack(file_info_versions) {
|
||||
versions_bin.push(match encode_msgpack(file_info_versions) {
|
||||
Ok(versions_bin) => Bytes::from(versions_bin),
|
||||
Err(err) => {
|
||||
let mut errors = Vec::with_capacity(versions.len());
|
||||
@@ -1896,7 +1886,7 @@ impl DiskAPI for RemoteDisk {
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
let file_info = compat_json(&fi)?;
|
||||
let file_info_bin = encode_file_info_msgpack(&fi)?;
|
||||
let file_info_bin = encode_msgpack(&fi)?;
|
||||
|
||||
self.execute_with_timeout_for_op(
|
||||
"write_metadata",
|
||||
@@ -1975,7 +1965,7 @@ impl DiskAPI for RemoteDisk {
|
||||
);
|
||||
let file_info = compat_json(&fi)?;
|
||||
let opts_str = compat_json(&opts)?;
|
||||
let file_info_bin = encode_file_info_msgpack(&fi)?;
|
||||
let file_info_bin = encode_msgpack(&fi)?;
|
||||
let opts_bin = encode_msgpack(opts)?;
|
||||
|
||||
self.execute_with_timeout_for_op(
|
||||
@@ -2239,7 +2229,7 @@ impl DiskAPI for RemoteDisk {
|
||||
"rename_data",
|
||||
|| async {
|
||||
let file_info = compat_json(&fi)?;
|
||||
let file_info_bin = encode_file_info_msgpack(&fi)?;
|
||||
let file_info_bin = encode_msgpack_named(&fi)?;
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
@@ -3381,8 +3371,6 @@ mod tests {
|
||||
crate::cluster::rpc::runtime_sources::reset_internode_metrics_for_test();
|
||||
let response = RenameDataResp {
|
||||
old_data_dir: Some(Uuid::new_v4()),
|
||||
rollback_data_dir: Some(Uuid::new_v4()),
|
||||
cleanup_data_dir: Some(Uuid::new_v4()),
|
||||
sign: Some(vec![0x14, 0x35]),
|
||||
old_current_size: Some(crate::disk::OldCurrentSize::Present(64 * 1024)),
|
||||
};
|
||||
@@ -3396,8 +3384,6 @@ mod tests {
|
||||
let decode_errors_after = crate::cluster::rpc::runtime_sources::internode_msgpack_json_decode_error_total_for_test();
|
||||
|
||||
assert_eq!(decoded.old_data_dir, response.old_data_dir);
|
||||
assert_eq!(decoded.rollback_data_dir, response.rollback_data_dir);
|
||||
assert_eq!(decoded.cleanup_data_dir, response.cleanup_data_dir);
|
||||
assert_eq!(decoded.sign, response.sign);
|
||||
assert_eq!(decoded.old_current_size, response.old_current_size);
|
||||
assert!(
|
||||
@@ -3747,13 +3733,8 @@ mod tests {
|
||||
fn rename_data_file_info_named_msgpack_is_smaller_than_json() {
|
||||
let file_info = sample_rename_data_file_info();
|
||||
let json = serde_json::to_vec(&file_info).expect("file info json should encode");
|
||||
let named_msgpack = encode_file_info_msgpack(&file_info).expect("file info named msgpack should encode");
|
||||
let named_msgpack = encode_msgpack_named(&file_info).expect("file info named msgpack should encode");
|
||||
|
||||
assert!(
|
||||
named_msgpack.len() <= FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT,
|
||||
"typical FileInfo should fit the msgpack capacity hint (msgpack={}, hint={FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT})",
|
||||
named_msgpack.len()
|
||||
);
|
||||
assert!(
|
||||
named_msgpack.len() < json.len(),
|
||||
"expected named msgpack payload to be smaller than json (msgpack={}, json={})",
|
||||
@@ -3766,13 +3747,11 @@ mod tests {
|
||||
fn rename_data_resp_named_msgpack_is_smaller_than_json() {
|
||||
let response = RenameDataResp {
|
||||
old_data_dir: Some(Uuid::new_v4()),
|
||||
rollback_data_dir: Some(Uuid::new_v4()),
|
||||
cleanup_data_dir: Some(Uuid::new_v4()),
|
||||
sign: Some(vec![1_u8; 32]),
|
||||
old_current_size: Some(crate::disk::OldCurrentSize::Present(4096)),
|
||||
};
|
||||
let json = serde_json::to_vec(&response).expect("rename data response json should encode");
|
||||
let named_msgpack = rmp_serde::encode::to_vec_named(&response).expect("rename data response named msgpack should encode");
|
||||
let named_msgpack = encode_msgpack_named(&response).expect("rename data response named msgpack should encode");
|
||||
|
||||
assert!(
|
||||
named_msgpack.len() < json.len(),
|
||||
|
||||
@@ -51,7 +51,6 @@ use serde_json::{Map, Value};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::{OwnedRwLockWriteGuard, RwLock as AsyncRwLock};
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
use uuid::Uuid;
|
||||
@@ -401,14 +400,6 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), false, Some(max_bytes)).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Read an existing config object without treating an empty payload as absent.
|
||||
/// Callers that validate their own payload format need to distinguish corruption
|
||||
/// from `ConfigNotFound`.
|
||||
@@ -416,7 +407,7 @@ pub(crate) async fn read_config_preserve_empty<S>(api: Arc<S>, file: &str) -> Re
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, None).await?;
|
||||
let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
@@ -444,23 +435,6 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_no_lock_preserve_empty_with_metadata<S>(api: Arc<S>, file: &str) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(
|
||||
api,
|
||||
file,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn read_config_with_metadata<S>(api: Arc<S>, file: &str, opts: &ObjectOptions) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -473,7 +447,7 @@ where
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
read_config_with_metadata_inner(api, file, opts, false, None).await
|
||||
read_config_with_metadata_inner(api, file, opts, false).await
|
||||
}
|
||||
|
||||
async fn read_config_with_metadata_inner<S>(
|
||||
@@ -481,7 +455,6 @@ async fn read_config_with_metadata_inner<S>(
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
preserve_empty: bool,
|
||||
max_bytes: Option<usize>,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: ObjectIO<
|
||||
@@ -507,25 +480,7 @@ where
|
||||
}
|
||||
})?;
|
||||
|
||||
let data = if let Some(max_bytes) = max_bytes {
|
||||
let object_size = usize::try_from(rd.object_info.size).map_err(|_| Error::CorruptedFormat)?;
|
||||
if object_size > max_bytes {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
|
||||
let read_limit = max_bytes.checked_add(1).ok_or(Error::CorruptedFormat)?;
|
||||
let mut data = Vec::with_capacity(read_limit.min(64 * 1024));
|
||||
(&mut rd)
|
||||
.take(u64::try_from(read_limit).map_err(|_| Error::CorruptedFormat)?)
|
||||
.read_to_end(&mut data)
|
||||
.await?;
|
||||
if data.len() > max_bytes {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
data
|
||||
} else {
|
||||
rd.read_all().await?
|
||||
};
|
||||
let data = rd.read_all().await?;
|
||||
|
||||
if data.is_empty() && !preserve_empty {
|
||||
return Err(Error::ConfigNotFound);
|
||||
@@ -631,47 +586,10 @@ where
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
save_config_with_opts_inner(api, file, data, opts, true).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Saves a configuration object without logging an error for a retryable caller-owned failure.
|
||||
pub async fn save_config_with_opts_quiet<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
save_config_with_opts_inner(api, file, data, opts, false).await.map(|_| ())
|
||||
save_config_with_opts_and_metadata(api, file, data, opts).await.map(|_| ())
|
||||
}
|
||||
|
||||
async fn save_config_with_opts_and_metadata<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<ObjectInfo>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
save_config_with_opts_inner(api, file, data, opts, true).await
|
||||
}
|
||||
|
||||
async fn save_config_with_opts_inner<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
data: Vec<u8>,
|
||||
opts: &ObjectOptions,
|
||||
log_error: bool,
|
||||
) -> Result<ObjectInfo>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
@@ -687,9 +605,7 @@ where
|
||||
match api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
|
||||
Ok(object_info) => Ok(object_info),
|
||||
Err(err) => {
|
||||
if log_error {
|
||||
error!("save_config_with_opts: err: {:?}, file: {}", err, file);
|
||||
}
|
||||
error!("save_config_with_opts: err: {:?}, file: {}", err, file);
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
@@ -2383,7 +2299,7 @@ where
|
||||
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?;
|
||||
let guard = lock.get_write_lock(get_lock_acquire_timeout()).await?;
|
||||
let read_options = ObjectOptions::default();
|
||||
match read_config_with_metadata_inner(api, &config_file, &read_options, true, None).await {
|
||||
match read_config_with_metadata_inner(api, &config_file, &read_options, true).await {
|
||||
Ok((raw, object_info)) => {
|
||||
let (config, seed) = decode_persisted_server_config_with_seed(&raw)?;
|
||||
Ok(ServerConfigSnapshot {
|
||||
@@ -2639,10 +2555,9 @@ mod tests {
|
||||
use super::{
|
||||
SERVER_CONFIG_LOCK, ServerConfigSnapshot, apply_dynamic_config_for_sub_sys_with, config_task_join_error,
|
||||
configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config,
|
||||
lookup_configs, new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata,
|
||||
read_config_preserve_empty, read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot,
|
||||
save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
|
||||
server_config_transaction_lock_path, storage_class_kvs_mut,
|
||||
lookup_configs, new_and_save_server_config, read_config, read_config_preserve_empty, read_config_with_metadata,
|
||||
read_config_without_migrate, read_server_config_snapshot, save_server_config, save_server_config_snapshot,
|
||||
save_server_config_snapshot_with_generation, server_config_transaction_lock_path, storage_class_kvs_mut,
|
||||
};
|
||||
use crate::config::{audit, heal, notify, oidc, scanner};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
@@ -5073,15 +4988,10 @@ mod tests {
|
||||
.expect_err("the existing config contract treats empty objects as missing");
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
|
||||
let data = read_config_preserve_empty(store.clone(), "config/empty.json")
|
||||
let data = read_config_preserve_empty(store, "config/empty.json")
|
||||
.await
|
||||
.expect("payload-validating callers must observe the empty object");
|
||||
assert!(data.is_empty());
|
||||
|
||||
let (data, _) = read_config_no_lock_preserve_empty_with_metadata(store, "config/empty.json")
|
||||
.await
|
||||
.expect("no-lock payload-validating callers must observe the empty object");
|
||||
assert!(data.is_empty());
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
|
||||
+107
-292
@@ -18,13 +18,13 @@ use crate::bucket::{
|
||||
lifecycle::{
|
||||
bucket_lifecycle_audit::LcEventSrc,
|
||||
bucket_lifecycle_ops::{
|
||||
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle,
|
||||
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule, eval_action_from_lifecycle,
|
||||
lifecycle_delete_all_versions_blocked_by_replication,
|
||||
},
|
||||
get_expiry_configs,
|
||||
lifecycle::IlmAction,
|
||||
},
|
||||
metadata_sys,
|
||||
object_lock::objectlock_sys::BucketObjectLockSys,
|
||||
};
|
||||
use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
|
||||
use crate::config::com::{CONFIG_PREFIX, read_config, read_config_no_lock, save_config, save_config_with_opts};
|
||||
@@ -60,7 +60,7 @@ use rustfs_common::defer;
|
||||
use rustfs_common::heal_channel::HealOpts;
|
||||
use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
|
||||
use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path};
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration};
|
||||
use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
@@ -91,8 +91,6 @@ const DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD: usize = 1000;
|
||||
const DECOMMISSION_BUCKET_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_BUCKET_CONCURRENCY";
|
||||
const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
|
||||
const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30;
|
||||
const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
pub const POOL_META_NAME: &str = "pool.bin";
|
||||
pub const POOL_META_FORMAT: u16 = 1;
|
||||
@@ -887,149 +885,9 @@ fn ensure_pool_not_left_in_cmdline_after_decommission(position: usize, cmd_line:
|
||||
|
||||
fn resolve_decommission_listing_worker_result(
|
||||
set_idx: usize,
|
||||
worker_result: std::result::Result<Result<()>, tokio::task::JoinError>,
|
||||
worker_result: std::result::Result<(), tokio::task::JoinError>,
|
||||
) -> Result<()> {
|
||||
worker_result.map_err(|err| Error::other(format!("decommission listing worker {set_idx} task join error: {err}")))?
|
||||
}
|
||||
|
||||
fn should_retry_decommission_listing(err: &Error, attempt: usize, max_attempts: usize) -> bool {
|
||||
!is_err_bucket_not_found(err) && attempt + 1 < max_attempts
|
||||
}
|
||||
|
||||
async fn wait_decommission_listing_retry(rx: &CancellationToken, delay: std::time::Duration) -> bool {
|
||||
tokio::select! {
|
||||
_ = rx.cancelled() => true,
|
||||
_ = tokio::time::sleep(delay) => false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_decommission_listing_with_retry<List, ListFuture>(
|
||||
rx: CancellationToken,
|
||||
bucket: String,
|
||||
cb: ListCallback,
|
||||
pool_idx: usize,
|
||||
set_idx: usize,
|
||||
max_attempts: usize,
|
||||
mut list: List,
|
||||
) -> Result<()>
|
||||
where
|
||||
List: FnMut(ListCallback) -> ListFuture,
|
||||
ListFuture: std::future::Future<Output = Result<()>>,
|
||||
{
|
||||
let max_attempts = max_attempts.max(1);
|
||||
|
||||
for attempt in 0..max_attempts {
|
||||
if rx.is_cancelled() {
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
state = "listing_worker_cancelled",
|
||||
"Decommission listing worker cancelled"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
attempt = attempt + 1,
|
||||
max_attempts,
|
||||
state = "listing_started",
|
||||
"Decommission listing started"
|
||||
);
|
||||
|
||||
match list(cb.clone()).await {
|
||||
Ok(()) => {
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
attempt = attempt + 1,
|
||||
max_attempts,
|
||||
state = "listing_completed",
|
||||
"Decommission listing completed"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Err(err) if is_err_bucket_not_found(&err) => {
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
attempt = attempt + 1,
|
||||
max_attempts,
|
||||
state = "listing_bucket_missing",
|
||||
"Decommission listing bucket missing"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Err(err) if should_retry_decommission_listing(&err, attempt, max_attempts) => {
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
attempt = attempt + 1,
|
||||
max_attempts,
|
||||
retry_delay_ms = DECOMMISSION_LISTING_RETRY_DELAY.as_millis(),
|
||||
state = "listing_failed_retrying",
|
||||
error = ?err,
|
||||
"Decommission listing failed; retrying"
|
||||
);
|
||||
if wait_decommission_listing_retry(&rx, DECOMMISSION_LISTING_RETRY_DELAY).await {
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
state = "listing_worker_cancelled",
|
||||
"Decommission listing worker cancelled during retry wait"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = pool_idx,
|
||||
set_index = set_idx,
|
||||
bucket = %bucket,
|
||||
attempt = attempt + 1,
|
||||
max_attempts,
|
||||
state = "listing_failed",
|
||||
error = ?err,
|
||||
"Decommission listing failed"
|
||||
);
|
||||
return Err(Error::other(format!(
|
||||
"decommission listing failed for bucket {bucket} pool {pool_idx} set {set_idx} attempt {}/{}: {err}",
|
||||
attempt + 1,
|
||||
max_attempts
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
worker_result.map_err(|err| Error::other(format!("decommission listing worker {set_idx} task join error: {err}")))
|
||||
}
|
||||
|
||||
fn should_count_decommission_version_complete(ignore: bool, cleanup_ignored: bool, failure: bool) -> bool {
|
||||
@@ -2334,7 +2192,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
||||
bucket: &str,
|
||||
version: &rustfs_filemeta::FileInfo,
|
||||
lifecycle_config: Option<&BucketLifecycleConfiguration>,
|
||||
object_lock_config: Option<&ObjectLockConfiguration>,
|
||||
lock_retention: Option<DefaultRetention>,
|
||||
apply_actions: bool,
|
||||
event_source: &LcEventSrc,
|
||||
) -> Result<bool> {
|
||||
@@ -2344,16 +2202,12 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
||||
|
||||
let versioned = BucketVersioningSys::prefix_enabled(bucket, &version.name).await;
|
||||
let object_info = crate::object_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
|
||||
let event = eval_action_from_lifecycle(lifecycle_config, object_lock_config, &object_info).await;
|
||||
let event = eval_action_from_lifecycle(lifecycle_config, lock_retention, &object_info).await;
|
||||
|
||||
match event.action {
|
||||
IlmAction::DeleteRestoredAction | IlmAction::DeleteRestoredVersionAction => {
|
||||
if apply_actions && object_info.is_remote() {
|
||||
let Ok(bucket_incarnation_id) = store.bucket_incarnation_id_from_disk(bucket).await else {
|
||||
return Ok(false);
|
||||
};
|
||||
let _ =
|
||||
apply_expiry_on_transitioned_object(store, &object_info, &event, event_source, bucket_incarnation_id).await;
|
||||
let _ = apply_expiry_on_transitioned_object(store, &object_info, &event, event_source).await;
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
@@ -2361,7 +2215,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
||||
if lifecycle_delete_all_versions_blocked_by_replication(store.clone(), bucket, &object_info.name, action).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
let applied = !apply_actions || apply_expiry_rule_in(store, &event, event_source, &object_info).await;
|
||||
let applied = !apply_actions || apply_expiry_rule(&event, event_source, &object_info).await;
|
||||
resolve_data_movement_lifecycle_expiry_result(action, apply_actions, applied)
|
||||
}
|
||||
_ => Ok(false),
|
||||
@@ -2793,7 +2647,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
#[allow(unused_assignments, clippy::too_many_arguments)]
|
||||
#[tracing::instrument(skip(self, set, _worker_permit, lifecycle_config, object_lock_config, replication_config))]
|
||||
#[tracing::instrument(skip(self, set, _worker_permit, lifecycle_config, lock_retention, replication_config))]
|
||||
async fn decommission_entry(
|
||||
self: &Arc<Self>,
|
||||
rx: CancellationToken,
|
||||
@@ -2803,7 +2657,7 @@ impl ECStore {
|
||||
set: Arc<SetDisks>,
|
||||
_worker_permit: OwnedSemaphorePermit,
|
||||
lifecycle_config: Option<BucketLifecycleConfiguration>,
|
||||
object_lock_config: Option<ObjectLockConfiguration>,
|
||||
lock_retention: Option<DefaultRetention>,
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
) -> Result<()> {
|
||||
debug!(
|
||||
@@ -2854,7 +2708,7 @@ impl ECStore {
|
||||
&bucket,
|
||||
version,
|
||||
lifecycle_config.as_ref(),
|
||||
object_lock_config.as_ref(),
|
||||
lock_retention.clone(),
|
||||
true,
|
||||
&LcEventSrc::Decom,
|
||||
)
|
||||
@@ -3159,14 +3013,7 @@ impl ECStore {
|
||||
&cleanup_preflight_allowed_missing,
|
||||
"decommission",
|
||||
)
|
||||
.await
|
||||
.map_err(|err| match err {
|
||||
data_movement::SourceCleanupError::SourceChanged => Error::other(format!(
|
||||
"decommission: source cleanup preflight failed for {}/{}: source versions changed after migration started",
|
||||
bucket, entry.name
|
||||
)),
|
||||
data_movement::SourceCleanupError::Storage(err) => err,
|
||||
});
|
||||
.await;
|
||||
resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())?
|
||||
} else if decommissioned != fivs.versions.len() || expired > 0 {
|
||||
warn!(
|
||||
@@ -3266,7 +3113,7 @@ impl ECStore {
|
||||
let mut listing_workers = Vec::with_capacity(pool.disk_set.len());
|
||||
|
||||
let mut lifecycle_config = None;
|
||||
let mut object_lock_config = None;
|
||||
let mut lock_retention = None;
|
||||
let mut replication_config = None;
|
||||
|
||||
if bi.name != RUSTFS_META_BUCKET {
|
||||
@@ -3275,9 +3122,8 @@ impl ECStore {
|
||||
"versioning",
|
||||
BucketVersioningSys::get(&bi.name).await,
|
||||
)?;
|
||||
let expiry_configs = get_expiry_configs(self, &bi.name).await?;
|
||||
lifecycle_config = expiry_configs.lifecycle.map(|config| (*config).clone());
|
||||
object_lock_config = expiry_configs.object_lock.map(|config| (*config).clone());
|
||||
lifecycle_config = runtime_sources::bucket_lifecycle_config(&bi.name).await;
|
||||
lock_retention = BucketObjectLockSys::get(&bi.name).await;
|
||||
replication_config = resolve_decommission_optional_bucket_config_result(
|
||||
&bi.name,
|
||||
"replication",
|
||||
@@ -3309,7 +3155,7 @@ impl ECStore {
|
||||
let workers = workers.clone();
|
||||
let set = set.clone();
|
||||
let lifecycle_config = lifecycle_config.clone();
|
||||
let object_lock_config = object_lock_config.clone();
|
||||
let lock_retention = lock_retention.clone();
|
||||
let replication_config = replication_config.clone();
|
||||
let entry_error = entry_error.clone();
|
||||
let callback_rx = rx.clone();
|
||||
@@ -3319,7 +3165,7 @@ impl ECStore {
|
||||
let workers = workers.clone();
|
||||
let set = set.clone();
|
||||
let lifecycle_config = lifecycle_config.clone();
|
||||
let object_lock_config = object_lock_config.clone();
|
||||
let lock_retention = lock_retention.clone();
|
||||
let replication_config = replication_config.clone();
|
||||
let entry_error = entry_error.clone();
|
||||
let callback_rx = callback_rx.clone();
|
||||
@@ -3381,7 +3227,7 @@ impl ECStore {
|
||||
set,
|
||||
worker_permit,
|
||||
lifecycle_config,
|
||||
object_lock_config,
|
||||
lock_retention,
|
||||
replication_config,
|
||||
)
|
||||
.await
|
||||
@@ -3403,21 +3249,78 @@ impl ECStore {
|
||||
let set_id = set_idx;
|
||||
let worker = tokio::spawn(async move {
|
||||
let _listing_permit = listing_permit;
|
||||
run_decommission_listing_with_retry(
|
||||
rx_clone.clone(),
|
||||
bi.name.clone(),
|
||||
decommission_entry.clone(),
|
||||
idx,
|
||||
set_id,
|
||||
DECOMMISSION_LISTING_MAX_ATTEMPTS,
|
||||
|callback| {
|
||||
let set = set.clone();
|
||||
let rx = rx_clone.clone();
|
||||
let bucket = bi.clone();
|
||||
async move { set.list_objects_to_decommission(rx, bucket, callback).await }
|
||||
},
|
||||
)
|
||||
.await
|
||||
loop {
|
||||
if rx_clone.is_cancelled() {
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_worker_cancelled",
|
||||
"Decommission listing worker cancelled"
|
||||
);
|
||||
break;
|
||||
}
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_started",
|
||||
"Decommission listing started"
|
||||
);
|
||||
|
||||
match set
|
||||
.list_objects_to_decommission(rx_clone.clone(), bi.clone(), decommission_entry.clone())
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_completed",
|
||||
"Decommission listing completed"
|
||||
);
|
||||
break;
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_failed",
|
||||
error = ?err,
|
||||
"Decommission listing failed"
|
||||
);
|
||||
if is_err_bucket_not_found(&err) {
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
set_index = set_id,
|
||||
bucket = %bi.name,
|
||||
state = "listing_bucket_missing",
|
||||
"Decommission listing bucket missing"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
listing_workers.push((set_id, worker));
|
||||
}
|
||||
@@ -4057,11 +3960,10 @@ impl ECStore {
|
||||
for set in &pool.disk_set {
|
||||
for bucket_info in &buckets {
|
||||
let mut lifecycle_config = None;
|
||||
let mut object_lock_config = None;
|
||||
let mut lock_retention = None;
|
||||
if bucket_info.name != RUSTFS_META_BUCKET {
|
||||
let expiry_configs = get_expiry_configs(self, &bucket_info.name).await?;
|
||||
lifecycle_config = expiry_configs.lifecycle.map(|config| (*config).clone());
|
||||
object_lock_config = expiry_configs.object_lock.map(|config| (*config).clone());
|
||||
lifecycle_config = runtime_sources::bucket_lifecycle_config(&bucket_info.name).await;
|
||||
lock_retention = BucketObjectLockSys::get(&bucket_info.name).await;
|
||||
}
|
||||
|
||||
let versions_found = Arc::new(AtomicUsize::new(0));
|
||||
@@ -4071,7 +3973,7 @@ impl ECStore {
|
||||
let entry_error_cb = entry_error.clone();
|
||||
let bucket_name = bucket_info.name.clone();
|
||||
let lifecycle_config_cb = lifecycle_config.clone();
|
||||
let object_lock_config_cb = object_lock_config.clone();
|
||||
let lock_retention_cb = lock_retention.clone();
|
||||
let store = Arc::clone(self);
|
||||
let callback_rx_cb = callback_rx.clone();
|
||||
|
||||
@@ -4080,7 +3982,7 @@ impl ECStore {
|
||||
let entry_error = entry_error_cb.clone();
|
||||
let bucket_name = bucket_name.clone();
|
||||
let lifecycle_config = lifecycle_config_cb.clone();
|
||||
let object_lock_config = object_lock_config_cb.clone();
|
||||
let lock_retention = lock_retention_cb.clone();
|
||||
let store = Arc::clone(&store);
|
||||
let callback_rx = callback_rx_cb.clone();
|
||||
Box::pin(async move {
|
||||
@@ -4122,7 +4024,7 @@ impl ECStore {
|
||||
&bucket_name,
|
||||
version,
|
||||
lifecycle_config.as_ref(),
|
||||
object_lock_config.as_ref(),
|
||||
lock_retention.clone(),
|
||||
false,
|
||||
&LcEventSrc::Decom,
|
||||
)
|
||||
@@ -5044,8 +4946,8 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi
|
||||
mod pools_tests {
|
||||
use super::{
|
||||
DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DecomBucketInfo,
|
||||
DecommissionStartPoolState, DecommissionTerminalState, ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo,
|
||||
PoolStatus, apply_decommission_status_space_info, bind_decommission_cancelers, bind_missing_decommission_cancelers,
|
||||
DecommissionStartPoolState, DecommissionTerminalState, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus,
|
||||
apply_decommission_status_space_info, bind_decommission_cancelers, bind_missing_decommission_cancelers,
|
||||
cancel_decommission_canceler, classify_decommission_terminal_state, count_decommission_item,
|
||||
decommission_cancel_signal_result, decommission_item_size, decommission_meta_bucket_options,
|
||||
decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency,
|
||||
@@ -5067,18 +4969,17 @@ mod pools_tests {
|
||||
resolve_decommission_spawn_failure_result, resolve_decommission_terminal_mark_after_error_result,
|
||||
resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result,
|
||||
resolve_start_decommission_pool_meta_reload_result, rollback_start_decommission_pool_meta,
|
||||
run_decommission_buckets_bounded, run_decommission_listing_with_retry, should_cleanup_decommission_source_entry,
|
||||
should_continue_decommission_queue, should_count_decommission_version_complete,
|
||||
should_preserve_decommission_canceled_state, should_reject_decommission_cancel_as_terminal,
|
||||
should_retry_decommission_cancel_reload, should_retry_decommission_listing, should_skip_canceled_decommission_routine,
|
||||
split_decommission_buckets, take_and_cancel_decommission_canceler, take_decommission_canceler,
|
||||
touch_decommission_progress, track_decommission_current_object, track_decommission_current_object_stage,
|
||||
validate_start_decommission_request, wait_decommission_listing_retry, wait_decommission_worker_drain,
|
||||
run_decommission_buckets_bounded, should_cleanup_decommission_source_entry, should_continue_decommission_queue,
|
||||
should_count_decommission_version_complete, should_preserve_decommission_canceled_state,
|
||||
should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload,
|
||||
should_skip_canceled_decommission_routine, split_decommission_buckets, take_and_cancel_decommission_canceler,
|
||||
take_decommission_canceler, touch_decommission_progress, track_decommission_current_object,
|
||||
track_decommission_current_object_stage, validate_start_decommission_request, wait_decommission_worker_drain,
|
||||
with_decommission_entry_context,
|
||||
};
|
||||
use crate::data_movement;
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::error::{Error, StorageError};
|
||||
use crate::error::Error;
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats};
|
||||
use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntry, ObjectPartInfo};
|
||||
@@ -5092,10 +4993,6 @@ mod pools_tests {
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
fn noop_decommission_list_callback() -> ListCallback {
|
||||
Arc::new(|_| Box::pin(async {}))
|
||||
}
|
||||
|
||||
fn decommission_test_pool_endpoint(idx: usize, is_local: bool) -> PoolEndpoints {
|
||||
let port = 9000usize + idx;
|
||||
let mut endpoint =
|
||||
@@ -6136,15 +6033,7 @@ mod pools_tests {
|
||||
|
||||
#[test]
|
||||
fn test_resolve_decommission_listing_worker_result_passthrough_ok() {
|
||||
assert!(resolve_decommission_listing_worker_result(2, Ok(Ok(()))).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_decommission_listing_worker_result_passthrough_worker_error() {
|
||||
let err = resolve_decommission_listing_worker_result(2, Ok(Err(Error::SlowDown)))
|
||||
.expect_err("listing worker error should be returned");
|
||||
|
||||
assert!(matches!(err, Error::SlowDown));
|
||||
assert!(resolve_decommission_listing_worker_result(2, Ok(())).is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -6162,80 +6051,6 @@ mod pools_tests {
|
||||
assert!(message.contains("panic"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_retry_decommission_listing_respects_attempt_limit_and_bucket_missing() {
|
||||
assert!(should_retry_decommission_listing(&Error::SlowDown, 0, 2));
|
||||
assert!(!should_retry_decommission_listing(&Error::SlowDown, 1, 2));
|
||||
assert!(!should_retry_decommission_listing(
|
||||
&StorageError::BucketNotFound("bucket".to_string()),
|
||||
0,
|
||||
2
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_wait_decommission_listing_retry_reports_canceled_without_sleeping() {
|
||||
let token = CancellationToken::new();
|
||||
token.cancel();
|
||||
|
||||
assert!(wait_decommission_listing_retry(&token, StdDuration::from_secs(30)).await);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_run_decommission_listing_with_retry_stops_after_attempt_limit() {
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let err = run_decommission_listing_with_retry(
|
||||
CancellationToken::new(),
|
||||
"bucket-a".to_string(),
|
||||
noop_decommission_list_callback(),
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
{
|
||||
let attempts = attempts.clone();
|
||||
move |_| {
|
||||
let attempts = attempts.clone();
|
||||
async move {
|
||||
attempts.fetch_add(1, Ordering::SeqCst);
|
||||
Err(Error::SlowDown)
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("permanent listing failure must not retry forever");
|
||||
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 3);
|
||||
assert!(err.to_string().contains("attempt 3/3"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_run_decommission_listing_with_retry_treats_bucket_missing_as_complete() {
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
run_decommission_listing_with_retry(
|
||||
CancellationToken::new(),
|
||||
"bucket-a".to_string(),
|
||||
noop_decommission_list_callback(),
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
{
|
||||
let attempts = attempts.clone();
|
||||
move |_| {
|
||||
let attempts = attempts.clone();
|
||||
async move {
|
||||
attempts.fetch_add(1, Ordering::SeqCst);
|
||||
Err(StorageError::BucketNotFound("bucket-a".to_string()))
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("missing bucket should keep previous decommission listing behavior");
|
||||
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_count_decommission_version_complete_for_cleanup_safe_ignored_result() {
|
||||
assert!(should_count_decommission_version_complete(true, true, false));
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::error_reduce::count_errs;
|
||||
use crate::error::{Error, Result, is_all_volume_not_found, is_err_object_not_found, is_err_strict_volume_not_found};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::layout::set_heal::{formats_to_drives_info, new_heal_format_sets};
|
||||
use crate::multipart_listing::paginate_multipart_listing;
|
||||
use crate::storage_api_contracts::{
|
||||
@@ -71,10 +71,6 @@ type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
const LIST_MULTIPART_SETS_CONCURRENCY: usize = 4;
|
||||
|
||||
fn is_idempotent_delete_prefix_error(err: &Error) -> bool {
|
||||
is_err_object_not_found(err) || is_err_strict_volume_not_found(err)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Sets {
|
||||
pub id: Uuid,
|
||||
@@ -286,23 +282,6 @@ impl Sets {
|
||||
self.get_disks(self.get_hashed_set_index(key))
|
||||
}
|
||||
|
||||
fn get_disks_for_heal_object(&self, key: &str, opts: &HealOpts) -> Result<Arc<SetDisks>> {
|
||||
match opts.set {
|
||||
Some(set_idx) => self.disk_set.get(set_idx).cloned().ok_or_else(|| {
|
||||
StorageError::InvalidArgument(
|
||||
"heal".to_string(),
|
||||
"set".to_string(),
|
||||
format!(
|
||||
"invalid heal set index {set_idx} for pool {} with {} sets",
|
||||
self.pool_idx,
|
||||
self.disk_set.len()
|
||||
),
|
||||
)
|
||||
}),
|
||||
None => Ok(self.get_disks_by_key(key)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn storage_info_snapshot(&self) -> rustfs_madmin::StorageInfo {
|
||||
let mut futures = Vec::with_capacity(self.disk_set.len());
|
||||
|
||||
@@ -360,19 +339,7 @@ impl Sets {
|
||||
futures.push(set.delete_object(bucket, object, opt.clone()));
|
||||
}
|
||||
|
||||
let errs = join_all(futures)
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|result| result.err())
|
||||
.collect::<Vec<_>>();
|
||||
if is_all_volume_not_found(&errs) {
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
for err in errs.into_iter().flatten() {
|
||||
if !is_idempotent_delete_prefix_error(&err) {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
let _results = join_all(futures).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -766,7 +733,7 @@ impl crate::storage_api_contracts::list::ListOperations for Sets {
|
||||
type WalkCancellation = CancellationToken;
|
||||
type WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>;
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(self))]
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn list_objects_v2(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
@@ -847,19 +814,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets {
|
||||
let upload_id_marker = upload_id_marker.clone();
|
||||
let delimiter = delimiter.clone();
|
||||
async move {
|
||||
// ECStore owns the bucket lifecycle fence and calls the
|
||||
// incarnation-aware pool helper. This lower-level trait
|
||||
// surface has no ECStore guard to propagate.
|
||||
set.list_multipart_uploads_for_incarnation(
|
||||
bucket,
|
||||
prefix,
|
||||
key_marker,
|
||||
upload_id_marker,
|
||||
delimiter,
|
||||
per_set_limit,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
set.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, per_set_limit)
|
||||
.await
|
||||
}
|
||||
})
|
||||
.buffer_unordered(LIST_MULTIPART_SETS_CONCURRENCY)
|
||||
@@ -1110,7 +1066,7 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
#[tracing::instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1118,7 +1074,7 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
|
||||
version_id: &str,
|
||||
opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
self.get_disks_for_heal_object(object, opts)?
|
||||
self.get_disks_by_key(object)
|
||||
.heal_object(bucket, object, version_id, opts)
|
||||
.await
|
||||
}
|
||||
@@ -1318,19 +1274,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_prefix_error_classification_only_ignores_absence() {
|
||||
assert!(is_idempotent_delete_prefix_error(&StorageError::FileNotFound));
|
||||
assert!(is_idempotent_delete_prefix_error(&StorageError::ObjectNotFound(
|
||||
"bucket".to_string(),
|
||||
"prefix".to_string()
|
||||
)));
|
||||
assert!(is_idempotent_delete_prefix_error(&StorageError::VolumeNotFound));
|
||||
assert!(is_idempotent_delete_prefix_error(&StorageError::BucketNotFound("bucket".to_string())));
|
||||
assert!(!is_idempotent_delete_prefix_error(&StorageError::DiskNotFound));
|
||||
assert!(!is_idempotent_delete_prefix_error(&StorageError::ErasureWriteQuorum));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sets_get_pool_and_set_returns_matching_coordinates() {
|
||||
let format = FormatV3::new(2, 2);
|
||||
@@ -1448,208 +1391,6 @@ mod tests {
|
||||
(temp_dirs, sets)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_uses_explicit_set_scope() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
let selected = sets
|
||||
.get_disks_for_heal_object(
|
||||
"object",
|
||||
&HealOpts {
|
||||
set: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("requested set should be selected");
|
||||
|
||||
assert!(Arc::ptr_eq(&selected, &sets.disk_set[1]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_without_set_scope_keeps_hash_routing() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
let object = "object";
|
||||
let selected = sets
|
||||
.get_disks_for_heal_object(object, &HealOpts::default())
|
||||
.expect("hash-routed set should be selected");
|
||||
|
||||
assert!(Arc::ptr_eq(&selected, &sets.get_disks_by_key(object)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_rejects_invalid_set_scope() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
let err = sets
|
||||
.get_disks_for_heal_object(
|
||||
"object",
|
||||
&HealOpts {
|
||||
set: Some(2),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect_err("out-of-range set scope must fail closed");
|
||||
|
||||
assert!(
|
||||
matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
|
||||
if field == "set" && reason.contains("invalid heal set index 2 for pool 0 with 2 sets")),
|
||||
"unexpected invalid set error: {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_surfaces_a_hard_error_from_any_set() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
let bucket = format!("delete-prefix-{}", Uuid::new_v4().simple());
|
||||
sets.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created across both sets");
|
||||
|
||||
let healthy_disks = sets.disk_set[0].disks.read().await.clone();
|
||||
for disk in healthy_disks.iter().flatten() {
|
||||
disk.write_all(&bucket, "blocked/prefix/object", bytes::Bytes::from_static(b"data"))
|
||||
.await
|
||||
.expect("healthy set should contain the prefix");
|
||||
}
|
||||
|
||||
let failing_disks = sets.disk_set[1].disks.read().await.clone();
|
||||
for disk in failing_disks.iter().flatten() {
|
||||
disk.write_all(&bucket, "blocked", bytes::Bytes::from_static(b"not-a-directory"))
|
||||
.await
|
||||
.expect("failing set should contain a parent file");
|
||||
}
|
||||
|
||||
let err = sets
|
||||
.delete_object(
|
||||
&bucket,
|
||||
"blocked/prefix",
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("a hard failure from one set must not be reported as success");
|
||||
|
||||
match err {
|
||||
StorageError::PrefixAccessDenied(error_bucket, error_prefix) => {
|
||||
assert_eq!(error_bucket, bucket);
|
||||
assert_eq!(error_prefix, "blocked/prefix");
|
||||
}
|
||||
other => panic!("unexpected recursive delete error: {other:?}"),
|
||||
}
|
||||
for disk in healthy_disks.iter().flatten() {
|
||||
assert!(
|
||||
matches!(disk.read_all(&bucket, "blocked/prefix/object").await, Err(DiskError::FileNotFound)),
|
||||
"the healthy set should still complete its prefix deletion"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_keeps_a_missing_bucket_idempotent_across_sets() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
let bucket = format!("delete-prefix-{}", Uuid::new_v4().simple());
|
||||
sets.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created across both sets");
|
||||
|
||||
let healthy_disks = sets.disk_set[0].disks.read().await.clone();
|
||||
for disk in healthy_disks.iter().flatten() {
|
||||
disk.write_all(&bucket, "existing/prefix/object", bytes::Bytes::from_static(b"data"))
|
||||
.await
|
||||
.expect("healthy set should contain the prefix");
|
||||
}
|
||||
let missing_bucket_disks = sets.disk_set[1].disks.read().await.clone();
|
||||
for disk in missing_bucket_disks.iter().flatten() {
|
||||
disk.delete_volume(&bucket, true)
|
||||
.await
|
||||
.expect("the bucket should be removed from one set");
|
||||
}
|
||||
|
||||
sets.delete_object(
|
||||
&bucket,
|
||||
"existing/prefix",
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a missing bucket on one set should remain an idempotent success");
|
||||
for disk in healthy_disks.iter().flatten() {
|
||||
assert!(
|
||||
matches!(disk.read_all(&bucket, "existing/prefix/object").await, Err(DiskError::FileNotFound)),
|
||||
"the healthy set should still complete its prefix deletion"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_preserves_a_completely_missing_bucket_error() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
let bucket = format!("delete-prefix-missing-{}", Uuid::new_v4().simple());
|
||||
|
||||
let err = sets
|
||||
.delete_object(
|
||||
&bucket,
|
||||
"missing/prefix",
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("a completely missing bucket must not be reported as a successful object deletion");
|
||||
|
||||
assert_eq!(err, StorageError::BucketNotFound(bucket));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_fails_when_one_set_is_entirely_offline() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
let bucket = format!("delete-prefix-{}", Uuid::new_v4().simple());
|
||||
sets.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created across both sets");
|
||||
|
||||
let online_disks = sets.disk_set[0].disks.read().await.clone();
|
||||
let offline_disks = sets.disk_set[1].disks.read().await.clone();
|
||||
for disk in online_disks.iter().chain(offline_disks.iter()).flatten() {
|
||||
disk.write_all(&bucket, "offline/prefix/object", bytes::Bytes::from_static(b"data"))
|
||||
.await
|
||||
.expect("each set should contain the prefix before the outage");
|
||||
}
|
||||
*sets.disk_set[1].disks.write().await = vec![None, None];
|
||||
|
||||
let err = sets
|
||||
.delete_object(
|
||||
&bucket,
|
||||
"offline/prefix",
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("an entirely offline set must make the recursive delete fail");
|
||||
|
||||
assert!(
|
||||
matches!(err, StorageError::InsufficientWriteQuorum(ref error_bucket, ref error_prefix)
|
||||
if error_bucket == &bucket && error_prefix == "offline/prefix"),
|
||||
"unexpected offline-set error: {err:?}"
|
||||
);
|
||||
for disk in online_disks.iter().flatten() {
|
||||
assert!(matches!(
|
||||
disk.read_all(&bucket, "offline/prefix/object").await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
for disk in offline_disks.iter().flatten() {
|
||||
disk.read_all(&bucket, "offline/prefix/object")
|
||||
.await
|
||||
.expect("the offline set's untouched prefix must still be present");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_format_heal_accepts_quorum_from_a_nonzero_set() {
|
||||
let (_temp_dirs, sets) = two_set_test_sets().await;
|
||||
@@ -1813,19 +1554,7 @@ mod tests {
|
||||
upload_id_marker = page.next_upload_id_marker;
|
||||
}
|
||||
|
||||
// Compare only the decoded `<uuid>x<timestamp>` suffixes: the full
|
||||
// upload id embeds the process-global deployment id, which a
|
||||
// concurrently running test can swap between create and list time.
|
||||
let normalize = |uploads: &[(String, String)]| {
|
||||
let mut normalized = uploads
|
||||
.iter()
|
||||
.map(|(key, upload_id)| (key.clone(), runtime_sources::upload_uuid_suffix(upload_id)))
|
||||
.collect::<Vec<_>>();
|
||||
normalized.sort();
|
||||
normalized
|
||||
};
|
||||
let actual = normalize(&actual);
|
||||
assert_eq!(actual, normalize(&expected), "set-level merge must return every upload exactly once");
|
||||
assert_eq!(actual, expected, "set-level merge must return every upload exactly once");
|
||||
let mut deduped = actual.clone();
|
||||
deduped.dedup();
|
||||
assert_eq!(deduped.len(), actual.len(), "set-level pagination must not duplicate uploads");
|
||||
|
||||
@@ -25,7 +25,7 @@ use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
|
||||
use crate::storage_api_contracts::{
|
||||
multipart::{CompletePart, MultipartOperations as _},
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{HTTPPreconditions, ObjectIO as _, ObjectOperations as _},
|
||||
object::{ObjectIO as _, ObjectOperations as _},
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use bytes::Bytes;
|
||||
@@ -228,7 +228,6 @@ fn data_movement_complete_multipart_opts(object_info: &ObjectInfo, src_pool_idx:
|
||||
ObjectOptions {
|
||||
versioned: object_info.version_id.is_some(),
|
||||
version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
|
||||
http_preconditions: data_movement_unversioned_target_precondition(object_info),
|
||||
data_movement: true,
|
||||
mod_time: object_info.mod_time,
|
||||
preserve_etag: object_info.etag.clone(),
|
||||
@@ -243,7 +242,6 @@ fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize)
|
||||
src_pool_idx,
|
||||
data_movement: true,
|
||||
version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
|
||||
http_preconditions: data_movement_unversioned_target_precondition(object_info),
|
||||
mod_time: object_info.mod_time,
|
||||
user_defined: data_movement_user_defined(object_info),
|
||||
preserve_etag: object_info.etag.clone(),
|
||||
@@ -251,17 +249,6 @@ fn data_movement_put_object_opts(object_info: &ObjectInfo, src_pool_idx: usize)
|
||||
}
|
||||
}
|
||||
|
||||
fn is_unversioned_data_movement_object(object_info: &ObjectInfo) -> bool {
|
||||
object_info.version_id.is_none_or(|version_id| version_id.is_nil())
|
||||
}
|
||||
|
||||
fn data_movement_unversioned_target_precondition(object_info: &ObjectInfo) -> Option<HTTPPreconditions> {
|
||||
is_unversioned_data_movement_object(object_info).then(|| HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
fn data_movement_put_object_reader(
|
||||
bucket: &str,
|
||||
object_info: &ObjectInfo,
|
||||
@@ -350,7 +337,7 @@ fn schedule_data_movement_multipart_abort_cleanup(
|
||||
}
|
||||
|
||||
fn should_check_data_movement_overwrite_resume(err: &Error) -> bool {
|
||||
is_err_data_movement_overwrite(err) || matches!(err, Error::PreconditionFailed)
|
||||
is_err_data_movement_overwrite(err)
|
||||
}
|
||||
|
||||
fn effective_actual_size(info: &ObjectInfo) -> Option<i64> {
|
||||
@@ -416,16 +403,6 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo)
|
||||
&& are_equivalent_data_movement_parts(&source.parts, &target.parts)
|
||||
}
|
||||
|
||||
fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
|
||||
is_unversioned_data_movement_object(source)
|
||||
&& is_unversioned_data_movement_object(target)
|
||||
&& !target.delete_marker
|
||||
&& source
|
||||
.mod_time
|
||||
.zip(target.mod_time)
|
||||
.is_some_and(|(source_time, target_time)| target_time > source_time)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
struct SourceCleanupPartIdentity {
|
||||
number: usize,
|
||||
@@ -437,15 +414,6 @@ struct SourceCleanupPartIdentity {
|
||||
checksums: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
struct SourceCleanupErasureIdentity {
|
||||
algorithm: String,
|
||||
data_blocks: usize,
|
||||
parity_blocks: usize,
|
||||
block_size: usize,
|
||||
distribution: Vec<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub(crate) struct SourceCleanupVersionIdentity {
|
||||
name: String,
|
||||
@@ -456,28 +424,10 @@ pub(crate) struct SourceCleanupVersionIdentity {
|
||||
etag: Option<String>,
|
||||
checksum: Option<Vec<u8>>,
|
||||
data_dir: Option<uuid::Uuid>,
|
||||
transition_status: String,
|
||||
transitioned_objname: String,
|
||||
transition_tier: String,
|
||||
transition_version_id: Option<uuid::Uuid>,
|
||||
transition_version: Option<String>,
|
||||
transition_version_state: u8,
|
||||
expire_restored: bool,
|
||||
erasure: SourceCleanupErasureIdentity,
|
||||
metadata: BTreeMap<String, String>,
|
||||
parts: Vec<SourceCleanupPartIdentity>,
|
||||
}
|
||||
|
||||
fn source_cleanup_erasure_identity(erasure: &rustfs_filemeta::ErasureInfo) -> SourceCleanupErasureIdentity {
|
||||
SourceCleanupErasureIdentity {
|
||||
algorithm: erasure.algorithm.clone(),
|
||||
data_blocks: erasure.data_blocks,
|
||||
parity_blocks: erasure.parity_blocks,
|
||||
block_size: erasure.block_size,
|
||||
distribution: erasure.distribution.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn source_cleanup_part_identity(part: &ObjectPartInfo) -> SourceCleanupPartIdentity {
|
||||
SourceCleanupPartIdentity {
|
||||
number: part.number,
|
||||
@@ -507,19 +457,6 @@ pub(crate) fn source_cleanup_version_identity(version: &FileInfo) -> SourceClean
|
||||
etag: version.get_etag(),
|
||||
checksum: version.checksum.as_ref().map(|checksum| checksum.to_vec()),
|
||||
data_dir: version.data_dir,
|
||||
transition_status: version.transition_status.clone(),
|
||||
transitioned_objname: version.transitioned_objname.clone(),
|
||||
transition_tier: version.transition_tier.clone(),
|
||||
transition_version_id: version.transition_version_id,
|
||||
transition_version: version.transition_version.clone(),
|
||||
transition_version_state: match version.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => 0,
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled => 1,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull => 2,
|
||||
rustfs_filemeta::TransitionVersionState::Exact => 3,
|
||||
},
|
||||
expire_restored: version.expire_restored,
|
||||
erasure: source_cleanup_erasure_identity(&version.erasure),
|
||||
metadata: version
|
||||
.metadata
|
||||
.iter()
|
||||
@@ -535,6 +472,10 @@ fn source_cleanup_version_identities(fivs: &FileInfoVersions) -> Vec<SourceClean
|
||||
identities
|
||||
}
|
||||
|
||||
fn source_cleanup_versions_match(expected: &FileInfoVersions, current: &FileInfoVersions) -> bool {
|
||||
source_cleanup_versions_match_with_allowed_missing(expected, current, &[])
|
||||
}
|
||||
|
||||
fn source_cleanup_versions_match_with_allowed_missing(
|
||||
expected: &FileInfoVersions,
|
||||
current: &FileInfoVersions,
|
||||
@@ -566,26 +507,6 @@ fn source_cleanup_versions_match_with_allowed_missing(
|
||||
.all(|(identity, count)| allowed_counts.get(&identity).copied().unwrap_or_default() >= count)
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum SourceCleanupError {
|
||||
#[error("source versions changed after migration started")]
|
||||
SourceChanged,
|
||||
#[error(transparent)]
|
||||
Storage(#[from] Error),
|
||||
}
|
||||
|
||||
fn ensure_source_cleanup_versions_match(
|
||||
expected: &FileInfoVersions,
|
||||
current: &FileInfoVersions,
|
||||
allowed_missing: &[SourceCleanupVersionIdentity],
|
||||
) -> std::result::Result<(), SourceCleanupError> {
|
||||
if source_cleanup_versions_match_with_allowed_missing(expected, current, allowed_missing) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(SourceCleanupError::SourceChanged)
|
||||
}
|
||||
}
|
||||
|
||||
fn source_cleanup_preflight_error(op_label: &str, bucket: &str, object: &str, err: impl std::fmt::Display) -> Error {
|
||||
Error::other(format!("{op_label}: source cleanup preflight failed for {bucket}/{object}: {err}"))
|
||||
}
|
||||
@@ -608,87 +529,21 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged(
|
||||
expected: &FileInfoVersions,
|
||||
allowed_missing: &[SourceCleanupVersionIdentity],
|
||||
op_label: &str,
|
||||
) -> std::result::Result<(), SourceCleanupError> {
|
||||
) -> Result<()> {
|
||||
let Some(current) = load_source_cleanup_versions(set, bucket, object, op_label).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
ensure_source_cleanup_versions_match(expected, ¤t, allowed_missing)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct SourceCleanupDeleteBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct SourceCleanupDeleteBarrier {
|
||||
state: Arc<SourceCleanupDeleteBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<SourceCleanupDeleteBarrierState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl SourceCleanupDeleteBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(SourceCleanupDeleteBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison");
|
||||
assert!(slot.is_none(), "source cleanup delete barrier must be unique");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
Self { state }
|
||||
if source_cleanup_versions_match_with_allowed_missing(expected, ¤t, allowed_missing) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("source cleanup should reach the pre-delete barrier");
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for SourceCleanupDeleteBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) {
|
||||
let barrier = SOURCE_CLEANUP_DELETE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
Err(source_cleanup_preflight_error(
|
||||
op_label,
|
||||
bucket,
|
||||
object,
|
||||
"source versions changed after migration started",
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
@@ -698,32 +553,29 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
expected: &FileInfoVersions,
|
||||
allowed_missing: &[SourceCleanupVersionIdentity],
|
||||
op_label: &str,
|
||||
) -> std::result::Result<ObjectInfo, SourceCleanupError> {
|
||||
) -> Result<ObjectInfo> {
|
||||
let cleanup_key = encode_dir_object(object);
|
||||
let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
|
||||
let _guard = ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(Error::from)?;
|
||||
let _guard = ns_lock.get_write_lock(get_lock_acquire_timeout()).await?;
|
||||
|
||||
ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;
|
||||
|
||||
#[cfg(test)]
|
||||
pause_source_cleanup_before_delete(bucket, object).await;
|
||||
|
||||
let mut opts = ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
data_movement: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_namespace_lock_guard(&_guard);
|
||||
let result = set.delete_object(bucket, cleanup_key.as_str(), opts).await;
|
||||
let result = set
|
||||
.delete_object(
|
||||
bucket,
|
||||
cleanup_key.as_str(),
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
crate::store::list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
result.map_err(SourceCleanupError::from)
|
||||
result
|
||||
}
|
||||
|
||||
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
|
||||
@@ -774,11 +626,7 @@ fn resolve_data_movement_overwrite_resume_result(
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
if is_equivalent_data_movement_object(source, &target) {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target))
|
||||
Ok(is_equivalent_data_movement_object(source, &target))
|
||||
}
|
||||
|
||||
async fn should_treat_data_movement_overwrite_as_complete(
|
||||
@@ -989,6 +837,7 @@ pub(crate) async fn migrate_object(
|
||||
bucket.as_str(),
|
||||
object_info.name.as_str()
|
||||
);
|
||||
mark_multipart_upload_completed(&abort_multipart_flag);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -1007,32 +856,6 @@ pub(crate) async fn migrate_object(
|
||||
}
|
||||
.await;
|
||||
|
||||
if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
|
||||
let abort_result = match store.pools.get(target_pool_idx) {
|
||||
Some(pool) => {
|
||||
pool.abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
}
|
||||
None => Err(Error::other(format!(
|
||||
"{op_label}: target pool {target_pool_idx} is out of range while aborting superseded multipart upload"
|
||||
))),
|
||||
};
|
||||
if let Err(abort_err) = abort_result
|
||||
&& !is_err_invalid_upload_id(&abort_err)
|
||||
{
|
||||
error!("{op_label}: abort superseded multipart upload err {:?}", &abort_err);
|
||||
schedule_data_movement_multipart_abort_cleanup(
|
||||
store.clone(),
|
||||
target_pool_idx,
|
||||
bucket.clone(),
|
||||
object_info.name.clone(),
|
||||
res.upload_id.clone(),
|
||||
op_label,
|
||||
);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Err(primary_err) = multipart_result {
|
||||
if should_abort_multipart_upload(&abort_multipart_flag) {
|
||||
return match store
|
||||
@@ -1232,7 +1055,7 @@ mod tests {
|
||||
let expected = cleanup_test_versions(vec![first.clone(), second.clone()]);
|
||||
let current = cleanup_test_versions(vec![second, first]);
|
||||
|
||||
assert!(source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &[]));
|
||||
assert!(source_cleanup_versions_match(&expected, ¤t));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1240,40 +1063,7 @@ mod tests {
|
||||
let expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]);
|
||||
let current = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "changed")]);
|
||||
|
||||
let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[])
|
||||
.expect_err("changed source metadata must defer cleanup");
|
||||
assert!(matches!(err, SourceCleanupError::SourceChanged));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_source_cleanup_preflight_rejects_changed_transition_or_erasure() {
|
||||
let expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]);
|
||||
let mut current = expected.clone();
|
||||
current.versions[0].transition_tier = "COLD".to_string();
|
||||
let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[])
|
||||
.expect_err("transition metadata changes must defer cleanup");
|
||||
assert!(matches!(err, SourceCleanupError::SourceChanged));
|
||||
|
||||
let mut current = expected.clone();
|
||||
current.versions[0].erasure.algorithm = "changed".to_string();
|
||||
let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[])
|
||||
.expect_err("erasure metadata changes must defer cleanup");
|
||||
assert!(matches!(err, SourceCleanupError::SourceChanged));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_source_cleanup_preflight_ignores_per_disk_erasure_fields() {
|
||||
let mut expected = cleanup_test_versions(vec![cleanup_test_file_info("object.txt", Uuid::from_u128(1), "source")]);
|
||||
expected.versions[0].erasure.checksums = vec![rustfs_filemeta::ChecksumInfo {
|
||||
part_number: 1,
|
||||
hash: Bytes::from_static(b"disk-a-checksum"),
|
||||
..Default::default()
|
||||
}];
|
||||
let mut current = expected.clone();
|
||||
current.versions[0].erasure.index = 7;
|
||||
current.versions[0].erasure.checksums[0].hash = Bytes::from_static(b"disk-b-checksum");
|
||||
|
||||
assert!(source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &[]));
|
||||
assert!(!source_cleanup_versions_match(&expected, ¤t));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1284,9 +1074,7 @@ mod tests {
|
||||
cleanup_test_file_info("object.txt", Uuid::from_u128(2), "new-version"),
|
||||
]);
|
||||
|
||||
let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[])
|
||||
.expect_err("an added source version must defer cleanup");
|
||||
assert!(matches!(err, SourceCleanupError::SourceChanged));
|
||||
assert!(!source_cleanup_versions_match(&expected, ¤t));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1307,9 +1095,7 @@ mod tests {
|
||||
let expected = cleanup_test_versions(vec![migrated.clone(), protected]);
|
||||
let current = cleanup_test_versions(vec![migrated]);
|
||||
|
||||
let err = ensure_source_cleanup_versions_match(&expected, ¤t, &[])
|
||||
.expect_err("an unexpected missing version must defer cleanup");
|
||||
assert!(matches!(err, SourceCleanupError::SourceChanged));
|
||||
assert!(!source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1321,9 +1107,7 @@ mod tests {
|
||||
let current = cleanup_test_versions(vec![migrated, new_version]);
|
||||
let allowed_missing = vec![source_cleanup_version_identity(&expired)];
|
||||
|
||||
let err = ensure_source_cleanup_versions_match(&expected, ¤t, &allowed_missing)
|
||||
.expect_err("a new source version must defer cleanup even when an expired version may be missing");
|
||||
assert!(matches!(err, SourceCleanupError::SourceChanged));
|
||||
assert!(!source_cleanup_versions_match_with_allowed_missing(&expected, ¤t, &allowed_missing));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1386,13 +1170,12 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_check_data_movement_overwrite_resume_accepts_conflict_errors() {
|
||||
fn test_should_check_data_movement_overwrite_resume_only_for_overwrite_error() {
|
||||
assert!(should_check_data_movement_overwrite_resume(&Error::DataMovementOverwriteErr(
|
||||
"bucket-a".to_string(),
|
||||
"object-a".to_string(),
|
||||
"version-a".to_string(),
|
||||
)));
|
||||
assert!(should_check_data_movement_overwrite_resume(&Error::PreconditionFailed));
|
||||
assert!(!should_check_data_movement_overwrite_resume(&Error::SlowDown));
|
||||
}
|
||||
|
||||
@@ -1767,7 +1550,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_data_movement_complete_multipart_opts_preserves_mod_time_version_and_etag() {
|
||||
let mod_time = OffsetDateTime::now_utc();
|
||||
let version_id = Uuid::from_u128(7);
|
||||
let version_id = Uuid::nil();
|
||||
let object_info = ObjectInfo {
|
||||
version_id: Some(version_id),
|
||||
mod_time: Some(mod_time),
|
||||
@@ -1783,12 +1566,11 @@ mod tests {
|
||||
assert_eq!(opts.version_id.as_deref(), Some(version_id.to_string().as_str()));
|
||||
assert_eq!(opts.preserve_etag.as_deref(), Some("etag-value"));
|
||||
assert_eq!(opts.src_pool_idx, 7);
|
||||
assert!(opts.http_preconditions.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_movement_put_object_opts_preserves_version_and_etag() {
|
||||
let version_id = Uuid::from_u128(9);
|
||||
let version_id = Uuid::nil();
|
||||
let object_info = ObjectInfo {
|
||||
version_id: Some(version_id),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
@@ -1806,35 +1588,6 @@ mod tests {
|
||||
assert_eq!(opts.src_pool_idx, 9);
|
||||
assert!(opts.data_movement);
|
||||
assert_eq!(opts.mod_time, object_info.mod_time);
|
||||
assert!(opts.http_preconditions.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_movement_unversioned_put_and_complete_require_absent_target() {
|
||||
for version_id in [None, Some(Uuid::nil())] {
|
||||
let object_info = ObjectInfo {
|
||||
version_id,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let put_opts = data_movement_put_object_opts(&object_info, 9);
|
||||
let complete_opts = data_movement_complete_multipart_opts(&object_info, 9);
|
||||
|
||||
assert_eq!(
|
||||
put_opts
|
||||
.http_preconditions
|
||||
.as_ref()
|
||||
.and_then(HTTPPreconditions::if_none_match_value),
|
||||
Some("*")
|
||||
);
|
||||
assert_eq!(
|
||||
complete_opts
|
||||
.http_preconditions
|
||||
.as_ref()
|
||||
.and_then(HTTPPreconditions::if_none_match_value),
|
||||
Some("*")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2083,154 +1836,6 @@ mod tests {
|
||||
assert!(should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_accepts_newer_unversioned_target() {
|
||||
for version_id in [None, Some(Uuid::nil())] {
|
||||
let source = ObjectInfo {
|
||||
version_id,
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
etag: Some("etag-client-write".to_string()),
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("precondition conflict target should be evaluated");
|
||||
|
||||
assert!(should_resume);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_accepts_equivalent_target() {
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(source.clone())), &source, 0, 1)
|
||||
.expect("equivalent precondition target should be evaluated");
|
||||
|
||||
assert!(should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_rejects_non_newer_unversioned_target() {
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
etag: Some("etag-conflict".to_string()),
|
||||
..source.clone()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("precondition conflict target should be evaluated");
|
||||
|
||||
assert!(!should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_rejects_newer_delete_marker() {
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
delete_marker: true,
|
||||
etag: None,
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("delete marker conflict should be evaluated");
|
||||
|
||||
assert!(!should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_overwrite_error_rejects_newer_unversioned_target() {
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
etag: Some("etag-client-write".to_string()),
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
|
||||
|
||||
let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("pool-selection overwrite must require target equivalence");
|
||||
|
||||
assert!(!should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_rejects_newer_versioned_target() {
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
version_id: Some(Uuid::from_u128(2)),
|
||||
etag: Some("etag-conflict".to_string()),
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("versioned conflict target should be evaluated");
|
||||
|
||||
assert!(!should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_rejects_versioned_source_with_unversioned_target() {
|
||||
let source = ObjectInfo {
|
||||
version_id: Some(Uuid::from_u128(1)),
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
version_id: None,
|
||||
etag: Some("etag-conflict".to_string()),
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("versioned source conflict should be evaluated");
|
||||
|
||||
assert!(!should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rebalance_overwrite_resume_accepts_equivalent_target_version() {
|
||||
let source = ObjectInfo {
|
||||
|
||||
@@ -20,7 +20,7 @@ pub mod local_snapshot;
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketOperations as _, BucketOptions},
|
||||
list::{ListOperations as _, StorageListObjectVersionsInfo},
|
||||
object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _, ObjectOperations as _},
|
||||
object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _},
|
||||
};
|
||||
use crate::{
|
||||
bucket::{metadata_sys::get_replication_config, versioning::VersioningApi as _, versioning_sys::BucketVersioningSys},
|
||||
@@ -33,9 +33,8 @@ use crate::{
|
||||
};
|
||||
pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapshot, snapshot_path};
|
||||
use rustfs_data_usage::{
|
||||
BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
|
||||
DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary,
|
||||
VersionsHistogram, observed_data_usage_is_newer,
|
||||
BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DataUsageCache, DataUsageEntry,
|
||||
DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary, VersionsHistogram,
|
||||
};
|
||||
use rustfs_io_metrics::record_system_path_failure;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
@@ -86,57 +85,12 @@ static USAGE_CACHE_UPDATING: OnceLock<CacheUpdating> = OnceLock::new();
|
||||
static LIVE_BUCKET_USAGE_CACHE: OnceLock<LiveBucketUsageCache> = OnceLock::new();
|
||||
static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
/// Best-available persisted usage for `bucket` when no authoritative source
|
||||
/// exists yet (issue #5716): after an upgrade from a pre-v2 release the only
|
||||
/// persisted usage data is the legacy `.usage.json`, which is demoted to
|
||||
/// non-authoritative, and the authoritative caches stay empty until the
|
||||
/// scanner's first complete cycle lands. Quota admission degrades to the
|
||||
/// pre-discard per-bucket sizes retained on the cached snapshot instead of
|
||||
/// failing every write closed.
|
||||
///
|
||||
/// The baseline is static between snapshot loads — live writes do not advance
|
||||
/// it — so hard-quota enforcement during the degraded window is advisory: the
|
||||
/// overrun is bounded only by the writes issued before the next complete
|
||||
/// scanner cycle replaces the baseline with authoritative usage. That is
|
||||
/// strictly tighter than beta.11 (usage treated as 0) and strictly more
|
||||
/// available than a blanket 503. The fallback applies to any window without
|
||||
/// authoritative usage, not only pre-v2 upgrades; the values always come from
|
||||
/// the last persisted scanner output. Loads go through the TTL-bounded
|
||||
/// snapshot cache, so the quota path adds at most one backend read per
|
||||
/// [`DATA_USAGE_CACHE_TTL_SECS`] window. Returns `None` for buckets absent
|
||||
/// from every persisted snapshot — those still fail closed.
|
||||
pub async fn lookup_degraded_bucket_usage_baseline(store: Arc<ECStore>, bucket: &str) -> Option<u64> {
|
||||
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||
{
|
||||
let cache = data_usage_snapshot_cache().read().await;
|
||||
if let Some(cached) = cache
|
||||
.as_ref()
|
||||
.filter(|cached| tokio::time::Instant::now().duration_since(cached.loaded_at) < ttl)
|
||||
{
|
||||
return cached.degraded_baseline.get(bucket).copied();
|
||||
}
|
||||
}
|
||||
|
||||
// Stale or empty cache: refresh through the TTL-bounded loader. A failed
|
||||
// refresh carries the previous baseline forward, so quota admission keeps
|
||||
// its last grounded values through a backend read outage.
|
||||
let _ = load_data_usage_from_backend_cached(store).await;
|
||||
let cache = data_usage_snapshot_cache().read().await;
|
||||
cache
|
||||
.as_ref()
|
||||
.and_then(|cached| cached.degraded_baseline.get(bucket).copied())
|
||||
}
|
||||
|
||||
/// Cached copy of the last persisted data usage snapshot, served to admin
|
||||
/// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
|
||||
#[derive(Debug, Clone)]
|
||||
struct CachedDataUsageSnapshot {
|
||||
info: Option<DataUsageInfo>,
|
||||
loaded_at: tokio::time::Instant,
|
||||
/// Pre-discard per-bucket sizes from the same load, retained even when the
|
||||
/// snapshot is incomplete and its bucket data is discarded. Consumed only
|
||||
/// by [`lookup_degraded_bucket_usage_baseline`] for quota admission.
|
||||
degraded_baseline: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
impl CachedDataUsageSnapshot {
|
||||
@@ -160,34 +114,24 @@ fn fresh_cached_data_usage_snapshot(
|
||||
|
||||
fn cache_data_usage_snapshot_result(
|
||||
cache: &mut Option<CachedDataUsageSnapshot>,
|
||||
result: Result<(DataUsageInfo, HashMap<String, u64>), Error>,
|
||||
result: Result<DataUsageInfo, Error>,
|
||||
loaded_at: tokio::time::Instant,
|
||||
refresh_generation: u64,
|
||||
current_generation: u64,
|
||||
) -> Option<Result<DataUsageInfo, Error>> {
|
||||
if current_generation != refresh_generation {
|
||||
if data_usage_snapshot_generation() != refresh_generation {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(match result {
|
||||
Ok((info, degraded_baseline)) => {
|
||||
Ok(info) => {
|
||||
*cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(info.clone()),
|
||||
loaded_at,
|
||||
degraded_baseline,
|
||||
});
|
||||
Ok(info)
|
||||
}
|
||||
Err(e) => {
|
||||
// Keep the previous baseline through a failed refresh: quota
|
||||
// admission must not lose its last grounded values because one
|
||||
// backend read errored.
|
||||
let degraded_baseline = cache.take().map(|cached| cached.degraded_baseline).unwrap_or_default();
|
||||
*cache = Some(CachedDataUsageSnapshot {
|
||||
info: None,
|
||||
loaded_at,
|
||||
degraded_baseline,
|
||||
});
|
||||
*cache = Some(CachedDataUsageSnapshot { info: None, loaded_at });
|
||||
Err(e)
|
||||
}
|
||||
})
|
||||
@@ -198,9 +142,6 @@ type DataUsageSnapshotCache = Arc<RwLock<Option<CachedDataUsageSnapshot>>>;
|
||||
static DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
|
||||
static DATA_USAGE_SNAPSHOT_REFRESH: OnceLock<Arc<TokioMutex<()>>> = OnceLock::new();
|
||||
static DATA_USAGE_SNAPSHOT_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
static ADMIN_DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
|
||||
static ADMIN_DATA_USAGE_SNAPSHOT_REFRESH: OnceLock<Arc<TokioMutex<()>>> = OnceLock::new();
|
||||
static ADMIN_DATA_USAGE_SNAPSHOT_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
// Always-on revert detector for rustfs/backlog#1306: one relaxed increment per
|
||||
// full-bucket version listing is negligible and lets tests prove that admin
|
||||
@@ -259,24 +200,11 @@ fn data_usage_snapshot_generation() -> u64 {
|
||||
DATA_USAGE_SNAPSHOT_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn admin_data_usage_snapshot_cache() -> &'static DataUsageSnapshotCache {
|
||||
ADMIN_DATA_USAGE_SNAPSHOT_CACHE.get_or_init(|| Arc::new(RwLock::new(None)))
|
||||
}
|
||||
|
||||
fn admin_data_usage_snapshot_generation() -> u64 {
|
||||
ADMIN_DATA_USAGE_SNAPSHOT_GENERATION.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn clear_data_usage_snapshot_cache(cache: &mut Option<CachedDataUsageSnapshot>) {
|
||||
DATA_USAGE_SNAPSHOT_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||
*cache = None;
|
||||
}
|
||||
|
||||
fn clear_admin_data_usage_snapshot_cache(cache: &mut Option<CachedDataUsageSnapshot>) {
|
||||
ADMIN_DATA_USAGE_SNAPSHOT_GENERATION.fetch_add(1, Ordering::AcqRel);
|
||||
*cache = None;
|
||||
}
|
||||
|
||||
fn live_bucket_usage_cache() -> &'static LiveBucketUsageCache {
|
||||
LIVE_BUCKET_USAGE_CACHE.get_or_init(|| {
|
||||
moka::future::Cache::builder()
|
||||
@@ -301,11 +229,6 @@ lazy_static::lazy_static! {
|
||||
SLASH_SEPARATOR,
|
||||
DATA_USAGE_OBJECT_NAME
|
||||
);
|
||||
pub static ref DATA_USAGE_OBSERVED_OBJ_NAME_PATH: String = format!("{}{}{}",
|
||||
crate::disk::BUCKET_META_PREFIX,
|
||||
SLASH_SEPARATOR,
|
||||
DATA_USAGE_OBSERVED_OBJECT_NAME
|
||||
);
|
||||
static ref DATA_USAGE_OBJ_BACKUP_PATH: String = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
static ref LEGACY_DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}",
|
||||
crate::disk::BUCKET_META_PREFIX,
|
||||
@@ -380,11 +303,6 @@ fn stale_data_usage_persist_reason_for_source(
|
||||
/// Store data usage info to backend storage
|
||||
#[instrument(skip(store))]
|
||||
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
|
||||
if data_usage_info.usage_snapshot_converged == Some(false) {
|
||||
return Err(Error::other(
|
||||
"nonconverged data usage observations cannot replace the quota-authoritative snapshot",
|
||||
));
|
||||
}
|
||||
// Prevent older data from overwriting newer persisted stats
|
||||
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
|
||||
&& source.is_authoritative()
|
||||
@@ -405,12 +323,10 @@ async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<E
|
||||
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
|
||||
|
||||
// Save to backend using the same mechanism as original code
|
||||
crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
|
||||
crate::config::com::save_config(store, &DATA_USAGE_OBJ_NAME_PATH, data)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
|
||||
cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await;
|
||||
|
||||
// Invalidate the cached snapshot so readers observe the new save on their
|
||||
// next request instead of waiting out the remaining TTL. The next cached
|
||||
// read reloads through `load_data_usage_from_backend`, keeping its
|
||||
@@ -420,64 +336,6 @@ async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<E
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
trait ObservedDataUsageSnapshotCleanup {
|
||||
async fn delete_observed_data_usage_snapshot(&self, revision: &str) -> Result<(), Error>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObservedDataUsageSnapshotCleanup for ECStore {
|
||||
async fn delete_observed_data_usage_snapshot(&self, revision: &str) -> Result<(), Error> {
|
||||
self.delete_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(revision.to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
|
||||
where
|
||||
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
|
||||
{
|
||||
let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
|
||||
Ok(Some(snapshot)) => snapshot,
|
||||
Ok(None) => return,
|
||||
Err(err) => {
|
||||
record_usage_snapshot_failure(
|
||||
"read_observed_before_authoritative_cleanup",
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
&err,
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
if observed_data_usage_is_newer(&observed, authoritative) {
|
||||
return;
|
||||
}
|
||||
|
||||
match store.delete_observed_data_usage_snapshot(&revision).await {
|
||||
Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {}
|
||||
Err(err) => {
|
||||
record_usage_snapshot_failure(
|
||||
"delete_observed_after_authoritative_save",
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
&err,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
|
||||
data_usage_info.buckets_count = u64::try_from(data_usage_info.buckets_usage.len()).unwrap_or(u64::MAX);
|
||||
}
|
||||
@@ -531,11 +389,6 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
|
||||
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
|
||||
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache cleanup")?;
|
||||
clear_data_usage_snapshot_cache(&mut snapshot_cache);
|
||||
drop(snapshot_cache);
|
||||
|
||||
let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
|
||||
ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache cleanup")?;
|
||||
clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -551,10 +404,6 @@ where
|
||||
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
|
||||
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
|
||||
clear_data_usage_snapshot_cache(&mut snapshot_cache);
|
||||
drop(snapshot_cache);
|
||||
let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
|
||||
ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?;
|
||||
clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
|
||||
result
|
||||
}
|
||||
|
||||
@@ -648,23 +497,6 @@ where
|
||||
)
|
||||
.await?;
|
||||
|
||||
ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?;
|
||||
if let Err(err) = remove_bucket_usage_from_object_with_retries(
|
||||
store,
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
bucket,
|
||||
DATA_USAGE_REMOVE_CAS_RETRIES,
|
||||
None,
|
||||
guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
// The authoritative timestamp was already advanced above, so admin
|
||||
// selection rejects this observation even if optional cleanup fails.
|
||||
// Never make an admin-only freshness artifact block DeleteBucket.
|
||||
record_usage_snapshot_failure("remove_bucket_from_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
|
||||
}
|
||||
|
||||
for object in [
|
||||
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
|
||||
@@ -929,72 +761,10 @@ async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo,
|
||||
/// Load data usage info from backend storage
|
||||
#[instrument(skip(store))]
|
||||
pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
Ok(load_data_usage_from_backend_with_baseline(store).await?.0)
|
||||
}
|
||||
|
||||
/// Like [`load_data_usage_from_backend`], but also returns the pre-discard
|
||||
/// per-bucket sizes so the cached loader can retain them as the degraded
|
||||
/// quota-admission baseline (issue #5716).
|
||||
async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Result<(DataUsageInfo, HashMap<String, u64>), Error> {
|
||||
let (data_usage_info, source) = load_data_usage_snapshot(store).await?;
|
||||
Ok(normalize_loaded_data_usage(data_usage_info, source.is_authoritative()).await)
|
||||
}
|
||||
|
||||
async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUsageInfo> {
|
||||
let data = match read_config_preserve_empty(store, &DATA_USAGE_OBSERVED_OBJ_NAME_PATH).await {
|
||||
Ok(data) => data,
|
||||
Err(Error::ConfigNotFound) => return None,
|
||||
Err(err) => {
|
||||
record_usage_snapshot_failure("read_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
match parse_usage_snapshot(&data) {
|
||||
Ok(info) if info.usage_snapshot_converged == Some(false) && info.is_complete_bucket_usage_snapshot() => Some(info),
|
||||
Ok(_) => {
|
||||
error!(
|
||||
event = "data_usage_snapshot_load_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "data_usage",
|
||||
state = "invalid_observed_snapshot",
|
||||
object = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
"observed data usage snapshot was not a structurally complete nonconverged view"
|
||||
);
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
record_usage_snapshot_decode_failure("parse_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn select_admin_data_usage_snapshot(
|
||||
mut authoritative: DataUsageInfo,
|
||||
authoritative_format: bool,
|
||||
observed: Option<DataUsageInfo>,
|
||||
) -> (DataUsageInfo, bool) {
|
||||
if authoritative_format
|
||||
&& authoritative.is_complete_bucket_usage_snapshot()
|
||||
&& authoritative.usage_snapshot_converged.is_none()
|
||||
{
|
||||
authoritative.usage_snapshot_converged = Some(true);
|
||||
}
|
||||
match observed {
|
||||
Some(observed) if observed_data_usage_is_newer(&observed, &authoritative) => (observed, true),
|
||||
_ => (authoritative, authoritative_format),
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
let (authoritative, source) = load_data_usage_snapshot(store.clone()).await?;
|
||||
let observed = load_observed_data_usage_snapshot(store).await;
|
||||
let (selected, selected_is_current_format) =
|
||||
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
|
||||
Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
|
||||
}
|
||||
|
||||
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
|
||||
if !data_usage_info.is_complete_bucket_usage_snapshot() {
|
||||
data_usage_info.usage_snapshot_complete = false;
|
||||
@@ -1037,13 +807,7 @@ fn populate_backward_compatible_usage_maps(data_usage_info: &mut DataUsageInfo)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the normalized snapshot plus the pre-discard per-bucket sizes: the
|
||||
/// degraded quota-admission baseline captured before an incomplete snapshot
|
||||
/// drops its bucket data (issue #5716).
|
||||
async fn normalize_loaded_data_usage(
|
||||
mut data_usage_info: DataUsageInfo,
|
||||
authoritative_format: bool,
|
||||
) -> (DataUsageInfo, HashMap<String, u64>) {
|
||||
async fn normalize_loaded_data_usage(mut data_usage_info: DataUsageInfo, authoritative_format: bool) -> DataUsageInfo {
|
||||
info!("Loaded data usage info from backend with {} buckets", data_usage_info.buckets_count);
|
||||
|
||||
if !authoritative_format {
|
||||
@@ -1051,7 +815,6 @@ async fn normalize_loaded_data_usage(
|
||||
}
|
||||
populate_backward_compatible_usage_maps(&mut data_usage_info);
|
||||
validate_complete_usage_snapshot(&mut data_usage_info);
|
||||
let degraded_baseline = data_usage_info.bucket_sizes.clone();
|
||||
discard_incomplete_bucket_usage(&mut data_usage_info);
|
||||
|
||||
// Handle replication info
|
||||
@@ -1077,7 +840,7 @@ async fn normalize_loaded_data_usage(
|
||||
}
|
||||
}
|
||||
|
||||
(data_usage_info, degraded_baseline)
|
||||
data_usage_info
|
||||
}
|
||||
|
||||
/// Load the persisted data usage snapshot through a small in-process cache.
|
||||
@@ -1110,58 +873,10 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
|
||||
}
|
||||
|
||||
let refresh_generation = data_usage_snapshot_generation();
|
||||
let result = load_data_usage_from_backend_with_baseline(store.clone()).await;
|
||||
let result = load_data_usage_from_backend(store.clone()).await;
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let mut cache = data_usage_snapshot_cache().write().await;
|
||||
if let Some(result) =
|
||||
cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation, data_usage_snapshot_generation())
|
||||
{
|
||||
return result;
|
||||
}
|
||||
drop(cache);
|
||||
drop(refresh_guard);
|
||||
}
|
||||
}
|
||||
|
||||
/// Load the freshest structurally complete snapshot for authenticated admin
|
||||
/// observability. A scan raced by namespace activity may be selected here, but
|
||||
/// never by [`load_data_usage_from_backend_cached`], which remains the
|
||||
/// converged source for quota admission.
|
||||
pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||
|
||||
loop {
|
||||
{
|
||||
let cache = admin_data_usage_snapshot_cache().read().await;
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
let refresh_guard = ADMIN_DATA_USAGE_SNAPSHOT_REFRESH
|
||||
.get_or_init(|| Arc::new(TokioMutex::new(())))
|
||||
.lock()
|
||||
.await;
|
||||
{
|
||||
let cache = admin_data_usage_snapshot_cache().read().await;
|
||||
if let Some(result) = fresh_cached_data_usage_snapshot(&cache, tokio::time::Instant::now(), ttl) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
let refresh_generation = admin_data_usage_snapshot_generation();
|
||||
let result = load_admin_data_usage_from_backend(store.clone())
|
||||
.await
|
||||
.map(|info| (info, HashMap::new()));
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let mut cache = admin_data_usage_snapshot_cache().write().await;
|
||||
if let Some(result) = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
result,
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
admin_data_usage_snapshot_generation(),
|
||||
) {
|
||||
if let Some(result) = cache_data_usage_snapshot_result(&mut cache, result, loaded_at, refresh_generation) {
|
||||
return result;
|
||||
}
|
||||
drop(cache);
|
||||
@@ -1174,16 +889,6 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
|
||||
pub async fn invalidate_data_usage_snapshot_cache() {
|
||||
let mut cache = data_usage_snapshot_cache().write().await;
|
||||
clear_data_usage_snapshot_cache(&mut cache);
|
||||
|
||||
let mut admin_cache = admin_data_usage_snapshot_cache().write().await;
|
||||
clear_admin_data_usage_snapshot_cache(&mut admin_cache);
|
||||
}
|
||||
|
||||
/// Invalidate only the admin/console view after an observational save. Quota
|
||||
/// admission continues to use the independently cached converged snapshot.
|
||||
pub async fn invalidate_admin_data_usage_snapshot_cache() {
|
||||
let mut cache = admin_data_usage_snapshot_cache().write().await;
|
||||
clear_admin_data_usage_snapshot_cache(&mut cache);
|
||||
}
|
||||
|
||||
/// Aggregate usage information from local disk snapshots.
|
||||
@@ -2307,7 +2012,6 @@ mod tests {
|
||||
struct UsageCasState {
|
||||
object: Option<(Vec<u8>, u64)>,
|
||||
backup_object: Option<(Vec<u8>, u64)>,
|
||||
observed_object: Option<(Vec<u8>, u64)>,
|
||||
legacy_object: Option<(Vec<u8>, u64)>,
|
||||
legacy_backup_object: Option<(Vec<u8>, u64)>,
|
||||
interleaving_snapshot: Option<Vec<u8>>,
|
||||
@@ -2327,24 +2031,10 @@ mod tests {
|
||||
state: Mutex<UsageCasState>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObservedDataUsageSnapshotCleanup for UsageCasStore {
|
||||
async fn delete_observed_data_usage_snapshot(&self, revision: &str) -> Result<(), Error> {
|
||||
let mut state = self.state.lock().await;
|
||||
let current = state.observed_object.as_ref().ok_or(Error::FileNotFound)?.1;
|
||||
if revision != format!("usage-{current}") {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
state.observed_object = None;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum UsageObjectSlot {
|
||||
Primary,
|
||||
Backup,
|
||||
Observed,
|
||||
LegacyPrimary,
|
||||
LegacyBackup,
|
||||
}
|
||||
@@ -2373,7 +2063,6 @@ mod tests {
|
||||
let slot = match object {
|
||||
object if object == DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Primary,
|
||||
object if object == DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::Backup,
|
||||
object if object == DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Observed,
|
||||
object if object == LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::LegacyPrimary,
|
||||
object if object == LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::LegacyBackup,
|
||||
_ => return Err(Error::FileNotFound),
|
||||
@@ -2382,7 +2071,6 @@ mod tests {
|
||||
let stored = match slot {
|
||||
UsageObjectSlot::Primary => &state.object,
|
||||
UsageObjectSlot::Backup => &state.backup_object,
|
||||
UsageObjectSlot::Observed => &state.observed_object,
|
||||
UsageObjectSlot::LegacyPrimary => &state.legacy_object,
|
||||
UsageObjectSlot::LegacyBackup => &state.legacy_backup_object,
|
||||
};
|
||||
@@ -2422,7 +2110,6 @@ mod tests {
|
||||
let slot = match object {
|
||||
object if object == DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Primary,
|
||||
object if object == DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::Backup,
|
||||
object if object == DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() => UsageObjectSlot::Observed,
|
||||
object if object == LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() => UsageObjectSlot::LegacyPrimary,
|
||||
object if object == LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str() => UsageObjectSlot::LegacyBackup,
|
||||
_ => return Err(Error::FileNotFound),
|
||||
@@ -2458,9 +2145,6 @@ mod tests {
|
||||
let revision = state.backup_object.as_ref().map_or(1, |(_, revision)| revision + 1);
|
||||
state.backup_object = Some((interleaving, revision));
|
||||
}
|
||||
if slot == UsageObjectSlot::Observed {
|
||||
return Err(Error::other("observed test fixture writes are injected directly"));
|
||||
}
|
||||
if slot == UsageObjectSlot::LegacyPrimary
|
||||
&& let Some(interleaving) = state.interleaving_legacy_snapshot.take()
|
||||
{
|
||||
@@ -2476,7 +2160,6 @@ mod tests {
|
||||
let current_revision = match slot {
|
||||
UsageObjectSlot::Primary => state.object.as_ref(),
|
||||
UsageObjectSlot::Backup => state.backup_object.as_ref(),
|
||||
UsageObjectSlot::Observed => state.observed_object.as_ref(),
|
||||
UsageObjectSlot::LegacyPrimary => state.legacy_object.as_ref(),
|
||||
UsageObjectSlot::LegacyBackup => state.legacy_backup_object.as_ref(),
|
||||
}
|
||||
@@ -2507,7 +2190,6 @@ mod tests {
|
||||
match slot {
|
||||
UsageObjectSlot::Primary => state.object = Some((buf, revision)),
|
||||
UsageObjectSlot::Backup => state.backup_object = Some((buf, revision)),
|
||||
UsageObjectSlot::Observed => state.observed_object = Some((buf, revision)),
|
||||
UsageObjectSlot::LegacyPrimary => state.legacy_object = Some((buf, revision)),
|
||||
UsageObjectSlot::LegacyBackup => state.legacy_backup_object = Some((buf, revision)),
|
||||
}
|
||||
@@ -2679,7 +2361,7 @@ mod tests {
|
||||
legacy.bucket_sizes.insert("large".to_string(), 0);
|
||||
legacy.buckets_count = 2;
|
||||
|
||||
let (normalized, degraded_baseline) = normalize_loaded_data_usage(legacy, false).await;
|
||||
let normalized = normalize_loaded_data_usage(legacy, false).await;
|
||||
|
||||
assert_eq!(normalized.buckets_count, 0);
|
||||
assert!(normalized.buckets_usage.is_empty());
|
||||
@@ -2687,10 +2369,6 @@ mod tests {
|
||||
assert_eq!(normalized.objects_total_count, 0);
|
||||
assert_eq!(normalized.objects_total_size, 0);
|
||||
assert!(!normalized.usage_snapshot_complete);
|
||||
// Issue #5716: the discarded sizes must survive as the degraded
|
||||
// quota-admission baseline.
|
||||
assert_eq!(degraded_baseline.get("control").copied(), Some(10_285));
|
||||
assert_eq!(degraded_baseline.get("large").copied(), Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2726,7 +2404,7 @@ mod tests {
|
||||
info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
|
||||
info.buckets_count = 2;
|
||||
|
||||
let (normalized, _) = normalize_loaded_data_usage(info, true).await;
|
||||
let normalized = normalize_loaded_data_usage(info, true).await;
|
||||
|
||||
assert_eq!(normalized.buckets_count, 2);
|
||||
assert!(normalized.usage_snapshot_complete);
|
||||
@@ -2761,7 +2439,7 @@ mod tests {
|
||||
info.bucket_sizes.insert("partial".to_string(), 196_870_144);
|
||||
info.buckets_count = 1;
|
||||
|
||||
let (normalized, _) = normalize_loaded_data_usage(info, true).await;
|
||||
let normalized = normalize_loaded_data_usage(info, true).await;
|
||||
|
||||
assert_eq!(normalized.buckets_count, 0);
|
||||
assert!(!normalized.buckets_usage.contains_key("control"));
|
||||
@@ -2776,7 +2454,7 @@ mod tests {
|
||||
info.buckets_count = 2;
|
||||
|
||||
assert!(!data_usage_contains_bucket(&info, "missing"));
|
||||
let (normalized, _) = normalize_loaded_data_usage(info, true).await;
|
||||
let normalized = normalize_loaded_data_usage(info, true).await;
|
||||
|
||||
assert!(!normalized.usage_snapshot_complete);
|
||||
assert!(normalized.buckets_usage.is_empty());
|
||||
@@ -2785,7 +2463,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_empty_snapshot_remains_authoritative() {
|
||||
let (normalized, _) = normalize_loaded_data_usage(
|
||||
let normalized = normalize_loaded_data_usage(
|
||||
DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
usage_snapshot_complete: true,
|
||||
@@ -2800,71 +2478,6 @@ mod tests {
|
||||
assert!(normalized.buckets_usage.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_snapshot_selection_requires_the_current_authoritative_baseline() {
|
||||
let authoritative = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH),
|
||||
scanner_epoch: Some(4),
|
||||
scanner_cycle: Some(10),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let observed = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
|
||||
scanner_epoch: Some(4),
|
||||
scanner_cycle: Some(11),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (selected, _) = select_admin_data_usage_snapshot(authoritative.clone(), true, Some(observed.clone()));
|
||||
assert_eq!(selected.usage_snapshot_converged, Some(false));
|
||||
|
||||
let mut namespace_changed = authoritative;
|
||||
namespace_changed.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(2));
|
||||
let (selected, _) = select_admin_data_usage_snapshot(namespace_changed, true, Some(observed));
|
||||
assert_eq!(selected.usage_snapshot_converged, Some(true));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
|
||||
let store = UsageCasStore::default();
|
||||
let authoritative = data_usage_info_for_test("bucket", 1, 10, SystemTime::UNIX_EPOCH + Duration::from_secs(2));
|
||||
let stale_observed = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
|
||||
scanner_epoch: Some(4),
|
||||
scanner_cycle: Some(10),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
..Default::default()
|
||||
};
|
||||
store.state.lock().await.observed_object =
|
||||
Some((serde_json::to_vec(&stale_observed).expect("observed snapshot should encode"), 1));
|
||||
|
||||
cleanup_observed_data_usage_after_authoritative_save(&store, &authoritative).await;
|
||||
assert!(store.state.lock().await.observed_object.is_none());
|
||||
|
||||
cleanup_observed_data_usage_after_authoritative_save(&store, &authoritative).await;
|
||||
assert!(store.state.lock().await.observed_object.is_none());
|
||||
|
||||
let newer_observed = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(3)),
|
||||
scanner_epoch: Some(4),
|
||||
scanner_cycle: Some(11),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
|
||||
..Default::default()
|
||||
};
|
||||
store.state.lock().await.observed_object =
|
||||
Some((serde_json::to_vec(&newer_observed).expect("observed snapshot should encode"), 2));
|
||||
|
||||
cleanup_observed_data_usage_after_authoritative_save(&store, &authoritative).await;
|
||||
assert!(store.state.lock().await.observed_object.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn cached_snapshot_failure_is_reused_until_ttl_expires() {
|
||||
@@ -2872,14 +2485,8 @@ mod tests {
|
||||
let mut cache = None;
|
||||
let refresh_generation = data_usage_snapshot_generation();
|
||||
|
||||
let first = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Err(Error::ErasureReadQuorum),
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
)
|
||||
.expect("an uninterrupted refresh should populate the cache");
|
||||
let first = cache_data_usage_snapshot_result(&mut cache, Err(Error::ErasureReadQuorum), loaded_at, refresh_generation)
|
||||
.expect("an uninterrupted refresh should populate the cache");
|
||||
assert!(matches!(first, Err(Error::ErasureReadQuorum)));
|
||||
|
||||
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
|
||||
@@ -2897,15 +2504,9 @@ mod tests {
|
||||
let mut cache = None;
|
||||
let refresh_generation = data_usage_snapshot_generation();
|
||||
|
||||
let first = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok((expected, HashMap::new())),
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
)
|
||||
.expect("an uninterrupted refresh should populate the cache")
|
||||
.expect("successful load must be returned");
|
||||
let first = cache_data_usage_snapshot_result(&mut cache, Ok(expected), loaded_at, refresh_generation)
|
||||
.expect("an uninterrupted refresh should populate the cache")
|
||||
.expect("successful load must be returned");
|
||||
assert_snapshot_bucket(&first, "bucket");
|
||||
|
||||
let cached = fresh_cached_data_usage_snapshot(&cache, loaded_at + Duration::from_secs(1), Duration::from_secs(30))
|
||||
@@ -2922,16 +2523,14 @@ mod tests {
|
||||
let mut cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
clear_data_usage_snapshot_cache(&mut cache);
|
||||
|
||||
let stale_result = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok((data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH), HashMap::new())),
|
||||
Ok(data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
);
|
||||
|
||||
assert!(stale_result.is_none());
|
||||
@@ -3934,7 +3533,6 @@ mod tests {
|
||||
*snapshot_cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(successor),
|
||||
loaded_at: tokio::time::Instant::now(),
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
memory_cache()
|
||||
.write()
|
||||
@@ -3997,7 +3595,6 @@ mod tests {
|
||||
*snapshot_cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(successor),
|
||||
loaded_at: tokio::time::Instant::now(),
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
|
||||
let store_for_cleanup = store.clone();
|
||||
@@ -4049,7 +3646,6 @@ mod tests {
|
||||
*data_usage_snapshot_cache().write().await = Some(CachedDataUsageSnapshot {
|
||||
info: Some(stale),
|
||||
loaded_at: tokio::time::Instant::now(),
|
||||
degraded_baseline: HashMap::new(),
|
||||
});
|
||||
|
||||
remove_bucket_usage_from_backend_with_guard(&store, BUCKET, None)
|
||||
|
||||
@@ -24,7 +24,6 @@ pub(crate) const GET_OBJECT_PATH_EMPTY: &str = "empty";
|
||||
pub(crate) const GET_OBJECT_PATH_DIRECT_MEMORY: &str = "direct_memory";
|
||||
pub(crate) const GET_OBJECT_PATH_BODY_CACHE: &str = "body_cache";
|
||||
pub(crate) const GET_OBJECT_PATH_INLINE_DIRECT: &str = "inline_direct";
|
||||
pub(crate) const GET_OBJECT_PATH_INTERNAL_META: &str = "internal_meta";
|
||||
pub(crate) const GET_OBJECT_PATH_LEGACY_DUPLEX: &str = "legacy_duplex";
|
||||
pub(crate) const GET_OBJECT_PATH_REMOTE_TRANSITION: &str = "remote_transition";
|
||||
pub(crate) const GET_OBJECT_PATH_SET_DISK: &str = "set_disk";
|
||||
@@ -164,7 +163,6 @@ pub(crate) enum GetObjectFailureReason {
|
||||
DecodeError,
|
||||
DownstreamClosed,
|
||||
Io,
|
||||
MetadataMissing,
|
||||
RangeOrLengthInvalid,
|
||||
ReadQuorum,
|
||||
ShortRead,
|
||||
@@ -179,7 +177,6 @@ impl GetObjectFailureReason {
|
||||
Self::DecodeError => "decode_error",
|
||||
Self::DownstreamClosed => "downstream_closed",
|
||||
Self::Io => "io",
|
||||
Self::MetadataMissing => "metadata_missing",
|
||||
Self::RangeOrLengthInvalid => "range_or_length_invalid",
|
||||
Self::ReadQuorum => "read_quorum",
|
||||
Self::ShortRead => "short_read",
|
||||
@@ -193,13 +190,6 @@ pub(crate) fn classify_storage_error(err: &StorageError) -> GetObjectFailureReas
|
||||
match err {
|
||||
StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _) => GetObjectFailureReason::ReadQuorum,
|
||||
StorageError::FileCorrupt => GetObjectFailureReason::BitrotMismatch,
|
||||
StorageError::FileNotFound
|
||||
| StorageError::FileVersionNotFound
|
||||
| StorageError::VolumeNotFound
|
||||
| StorageError::BucketNotFound(_)
|
||||
| StorageError::ObjectNotFound(_, _)
|
||||
| StorageError::VersionNotFound(_, _, _)
|
||||
| StorageError::ConfigNotFound => GetObjectFailureReason::MetadataMissing,
|
||||
StorageError::InvalidRangeSpec(_) => GetObjectFailureReason::RangeOrLengthInvalid,
|
||||
StorageError::Io(io_err) => classify_io_error(io_err),
|
||||
_ => GetObjectFailureReason::Unknown,
|
||||
@@ -303,34 +293,6 @@ mod tests {
|
||||
classify_storage_error(&StorageError::InvalidRangeSpec("bad range".to_string())),
|
||||
GetObjectFailureReason::RangeOrLengthInvalid
|
||||
);
|
||||
assert_eq!(
|
||||
classify_storage_error(&StorageError::FileNotFound),
|
||||
GetObjectFailureReason::MetadataMissing
|
||||
);
|
||||
assert_eq!(
|
||||
classify_storage_error(&StorageError::VolumeNotFound),
|
||||
GetObjectFailureReason::MetadataMissing
|
||||
);
|
||||
assert_eq!(
|
||||
classify_storage_error(&StorageError::ObjectNotFound("bucket".to_string(), "object".to_string())),
|
||||
GetObjectFailureReason::MetadataMissing
|
||||
);
|
||||
assert_eq!(
|
||||
classify_storage_error(&StorageError::BucketNotFound("bucket".to_string())),
|
||||
GetObjectFailureReason::MetadataMissing
|
||||
);
|
||||
assert_eq!(
|
||||
classify_storage_error(&StorageError::VersionNotFound(
|
||||
"bucket".to_string(),
|
||||
"object".to_string(),
|
||||
"version".to_string()
|
||||
)),
|
||||
GetObjectFailureReason::MetadataMissing
|
||||
);
|
||||
assert_eq!(
|
||||
classify_storage_error(&StorageError::ConfigNotFound),
|
||||
GetObjectFailureReason::MetadataMissing
|
||||
);
|
||||
|
||||
let internal_broken_pipe = StorageError::Io(io::Error::from(io::ErrorKind::BrokenPipe));
|
||||
assert_eq!(classify_storage_error(&internal_broken_pipe), GetObjectFailureReason::Io);
|
||||
@@ -392,12 +354,10 @@ mod tests {
|
||||
assert_eq!(GetObjectFailureReason::DownstreamClosed.as_str(), "downstream_closed");
|
||||
assert_eq!(GetObjectFailureReason::BitrotMismatch.as_str(), "bitrot_mismatch");
|
||||
assert_eq!(GetObjectFailureReason::DecodeError.as_str(), "decode_error");
|
||||
assert_eq!(GetObjectFailureReason::MetadataMissing.as_str(), "metadata_missing");
|
||||
assert_eq!(GET_READER_BUFFER_OUTPUT, "output");
|
||||
assert_eq!(GET_READER_BUFFER_PREFETCH, "prefetch");
|
||||
assert_eq!(GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, "codec_streaming_legacy_engine");
|
||||
assert_eq!(GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, "codec_streaming_rustfs_engine");
|
||||
assert_eq!(GET_OBJECT_PATH_INTERNAL_META, "internal_meta");
|
||||
assert_eq!(GET_DIRECT_MEMORY_DECISION_USE, "use");
|
||||
assert_eq!(GET_DIRECT_MEMORY_DECISION_FALLBACK, "fallback");
|
||||
assert_eq!(GET_DIRECT_MEMORY_REASON_NONE, "none");
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user