diff --git a/.config/make/lint-fmt.mak b/.config/make/lint-fmt.mak index 46f659440..55ed1efc6 100644 --- a/.config/make/lint-fmt.mak +++ b/.config/make/lint-fmt.mak @@ -70,6 +70,11 @@ fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabi @echo "๐Ÿ“ฃ Checking cryptographic capability wording guard..." ./scripts/check_fips_wording.sh +.PHONY: embedded-secrets-check +embedded-secrets-check: ## Check no private key material or credential literal is committed + @echo "๐Ÿ”‘ Checking embedded secret material guard..." + ./scripts/check_embedded_secrets.sh + .PHONY: log-analyzer-rules-check log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source @echo "๐Ÿฉบ Checking log-analyzer rule anchors..." diff --git a/.config/make/pre-commit.mak b/.config/make/pre-commit.mak index 12eb9693e..b4ba093b0 100644 --- a/.config/make/pre-commit.mak +++ b/.config/make/pre-commit.mak @@ -19,13 +19,13 @@ planning-docs-check: ## Check that no planning-type documents are committed ./scripts/check_no_planning_docs.sh .PHONY: pre-commit -pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests +pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests @echo "โœ… All pre-commit checks passed!" .PHONY: pre-pr -pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests +pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests @echo "โœ… All pre-PR checks passed!" .PHONY: dev-check -dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks +dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks @echo "โœ… Fast development checks passed!" diff --git a/.config/make/tests.mak b/.config/make/tests.mak index cdb8cd25e..937a22aca 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -34,6 +34,7 @@ script-tests: ## Run shell script tests ./scripts/test_exact_1mib_handoff_abba.sh ./scripts/test_pinned_paired_abba_bench.sh ./scripts/test_manual_transition_runbooks.sh + ./scripts/check_embedded_secrets.sh --self-test bash -n ./scripts/validate_object_data_cache_cold_stampede.sh python3 ./scripts/check_object_data_cache_follower_samples.py --self-test ./scripts/validate_object_data_cache_cold_stampede.sh --self-test diff --git a/.github/workflows/architecture-migration-rules.yml b/.github/workflows/architecture-migration-rules.yml index 365168dc7..c9eabcea7 100644 --- a/.github/workflows/architecture-migration-rules.yml +++ b/.github/workflows/architecture-migration-rules.yml @@ -53,9 +53,9 @@ jobs: persist-credentials: false - name: Install ripgrep - run: | - sudo apt-get update - sudo apt-get install -y ripgrep + uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 + with: + tool: ripgrep@15.2.0 - name: Check architecture migration rules run: ./scripts/check_architecture_migration_rules.sh diff --git a/.github/workflows/ci-docs-only.yml b/.github/workflows/ci-docs-only.yml index a156a61b9..89ae4d18f 100644 --- a/.github/workflows/ci-docs-only.yml +++ b/.github/workflows/ci-docs-only.yml @@ -83,7 +83,9 @@ jobs: persist-credentials: false - name: Install ripgrep - run: sudo apt-get update && sudo apt-get install -y ripgrep + uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 + with: + tool: ripgrep@15.2.0 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -120,6 +122,9 @@ jobs: - name: Check cryptographic capability wording run: ./scripts/check_fips_wording.sh + - name: Check no embedded secret material + run: ./scripts/check_embedded_secrets.sh + - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 412145661..4812b118f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,9 @@ jobs: persist-credentials: false - name: Install ripgrep - run: sudo apt-get update && sudo apt-get install -y ripgrep + uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 + with: + tool: ripgrep@15.2.0 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -155,6 +157,9 @@ jobs: - name: Check cryptographic capability wording run: ./scripts/check_fips_wording.sh + - name: Check no embedded secret material + run: ./scripts/check_embedded_secrets.sh + - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh diff --git a/Cargo.lock b/Cargo.lock index 3a6bbb935..698da5972 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9934,6 +9934,7 @@ dependencies = [ "libc", "log", "metrics", + "metrics-util", "num_cpus", "nvml-wrapper", "opentelemetry", diff --git a/Cargo.toml b/Cargo.toml index b6a37b028..438472db2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -264,6 +264,7 @@ lazy_static = "1.5.0" libc = "0.2.189" libsystemd = "0.7.2" local-ip-address = "0.6.13" +log = "0.4" memmap2 = "0.9.11" lz4 = "1.28.1" matchit = "0.9.2" @@ -326,6 +327,7 @@ zstd = "0.13.3" # Observability and Metrics metrics = "0.24.6" +metrics-util = "0.20" dial9-tokio-telemetry = "0.3" opentelemetry = { version = "0.32.0" } opentelemetry-appender-tracing = { version = "0.32.0" } diff --git a/crates/audit/tests/pipeline_layer_test.rs b/crates/audit/tests/pipeline_layer_test.rs index 3b28b5863..5d5608db2 100644 --- a/crates/audit/tests/pipeline_layer_test.rs +++ b/crates/audit/tests/pipeline_layer_test.rs @@ -236,12 +236,19 @@ async fn audit_pipeline_reports_empty_runtime_snapshots() { } #[tokio::test] -async fn audit_runtime_facade_stops_empty_replay_workers() { +async fn stopping_audit_replay_workers_is_a_no_op_when_there_are_none() { let registry = Arc::new(Mutex::new(AuditRegistry::new())); let replay_workers = Arc::new(RwLock::new(rustfs_targets::ReplayWorkerManager::new())); - let facade = AuditRuntimeFacade::new(registry, replay_workers); + let facade = AuditRuntimeFacade::new(registry, Arc::clone(&replay_workers)); facade.stop_replay_workers().await; + + // The stop path takes the manager's workers and hands them to the adapter, + // so an empty facade must leave it empty rather than wedge it, and a second + // call โ€” which shutdown paths make โ€” must stay harmless (rustfs/backlog#1836). + assert!(replay_workers.read().await.is_empty()); + facade.stop_replay_workers().await; + assert!(replay_workers.read().await.is_empty()); } #[tokio::test] diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index 4c409279f..00521b55b 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -1401,25 +1401,11 @@ impl CurrentCycle { } /// OTEL metric name constants for scanner metrics -const OTEL_SCANNER_OBJECTS_SCANNED: &str = "rustfs_scanner_objects_scanned_total"; -const OTEL_SCANNER_DIRECTORIES_SCANNED: &str = "rustfs_scanner_directories_scanned_total"; const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total"; const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total"; const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds"; const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds"; -fn emit_otel_counter(metric: usize, count: u64) { - match Metric::from_index(metric) { - Some(Metric::ScanObject) => { - metrics::counter!(OTEL_SCANNER_OBJECTS_SCANNED).increment(count); - } - Some(Metric::ScanFolder) => { - metrics::counter!(OTEL_SCANNER_DIRECTORIES_SCANNED).increment(count); - } - _ => {} - } -} - fn scan_cycle_result_label(result: u8) -> &'static str { match result { SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL, @@ -1960,7 +1946,6 @@ impl Metrics { let duration = SystemTime::now().duration_since(start).unwrap_or_default(); global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed); global_metrics().record_source_work_for_metric(metric, 1); - emit_otel_counter(metric_idx, 1); if metric_idx < Metric::LastRealtime as usize { global_metrics().latency[metric_idx].add(duration); } @@ -1976,7 +1961,6 @@ impl Metrics { let duration = SystemTime::now().duration_since(start).unwrap_or_default(); global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed); global_metrics().record_source_work_for_metric(metric, 1); - emit_otel_counter(metric_idx, 1); if metric_idx < Metric::LastRealtime as usize { global_metrics().latency[metric_idx].add_size(duration, size); } @@ -1992,7 +1976,6 @@ impl Metrics { let duration = SystemTime::now().duration_since(start).unwrap_or_default(); global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed); global_metrics().record_source_work_for_metric(metric, 1); - emit_otel_counter(metric_idx, 1); if metric_idx < Metric::LastRealtime as usize { global_metrics().latency[metric_idx].add(duration); } @@ -2010,7 +1993,6 @@ impl Metrics { let count = usize_to_u64_saturated(count); global_metrics().operations[metric_idx].fetch_add(count, Ordering::Relaxed); global_metrics().record_source_work_for_metric(metric, count); - emit_otel_counter(metric_idx, count); if metric_idx < Metric::LastRealtime as usize { global_metrics().latency[metric_idx].add(duration); } @@ -2031,7 +2013,6 @@ impl Metrics { let duration = SystemTime::now().duration_since(start).unwrap_or_default(); let metric_idx = Metric::Ilm as usize; global_metrics().operations[metric_idx].fetch_add(versions, Ordering::Relaxed); - emit_otel_counter(metric_idx, versions); global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed); global_metrics().actions_latency[a_idx].add(duration); }) @@ -2044,7 +2025,6 @@ impl Metrics { let metric_idx = metric as usize; global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed); global_metrics().record_source_work_for_metric(metric, 1); - emit_otel_counter(metric_idx, 1); if metric_idx < Metric::LastRealtime as usize { global_metrics().latency[metric_idx].add(duration); } diff --git a/crates/config/src/constants/scanner.rs b/crates/config/src/constants/scanner.rs index bcb730c2d..8086c3229 100644 --- a/crates/config/src/constants/scanner.rs +++ b/crates/config/src/constants/scanner.rs @@ -228,15 +228,6 @@ pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 4; /// Default object interval for cooperative scanner yields. pub const DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS: u64 = 128; -/// Compatibility flag kept for Patch 3 rollback windows. -/// -/// Inline scanner heal execution has been removed in favor of heal-candidate enqueue. -/// When this flag is enabled, RustFS logs a warning and continues to use enqueue-based heal. -pub const ENV_SCANNER_INLINE_HEAL_ENABLE: &str = "RUSTFS_SCANNER_INLINE_HEAL_ENABLE"; - -/// Default inline scanner heal compatibility mode. -pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false; - /// Scanner speed preset controlling throttling behavior. /// /// Each preset defines three parameters: diff --git a/crates/crypto/src/license_token.rs b/crates/crypto/src/license_token.rs index 8128605ec..cfec3a7ef 100644 --- a/crates/crypto/src/license_token.rs +++ b/crates/crypto/src/license_token.rs @@ -203,14 +203,6 @@ mod tests { assert!(result.is_err()); } - #[test] - fn test_source_does_not_embed_private_key() { - let source = include_str!("license_token.rs"); - let forbidden = ["BEGIN", "PRIVATE KEY"].join(" "); - - assert!(!source.contains(&forbidden)); - } - #[test] fn test_parse_signed_license_token_rejects_invalid_token() { let mut rng = rand::rng(); diff --git a/crates/e2e_test/src/admin_auth_test.rs b/crates/e2e_test/src/admin_auth_test.rs index 3cdd92c75..96e7878c4 100644 --- a/crates/e2e_test/src/admin_auth_test.rs +++ b/crates/e2e_test/src/admin_auth_test.rs @@ -38,7 +38,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; use std::io::Read; use std::process::{Command, Stdio}; @@ -162,7 +161,6 @@ mod tests { /// A fully authenticated but non-admin credential must be rejected with /// `403 AccessDenied` on an admin API, while the root credential succeeds. #[tokio::test(flavor = "multi_thread")] - #[serial] async fn non_admin_credential_denied_on_admin_api() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -198,7 +196,6 @@ mod tests { } #[tokio::test(flavor = "multi_thread")] - #[serial] async fn non_admin_credential_denied_on_manual_transition_run() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -326,7 +323,6 @@ mod tests { /// credential is accepted and the old one is rejected, on both the S3 data /// plane and the admin plane. #[tokio::test(flavor = "multi_thread")] - #[serial] async fn root_credential_rotation_takes_effect() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -389,7 +385,6 @@ mod tests { /// runtime. We capture the child's stdout/stderr directly (the shared /// harness inherits stdio) and poll for the warning until it appears. #[tokio::test(flavor = "multi_thread")] - #[serial] async fn default_credentials_emit_startup_warning() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/admin_iam_crud_test.rs b/crates/e2e_test/src/admin_iam_crud_test.rs index 508edf810..84b50709c 100644 --- a/crates/e2e_test/src/admin_iam_crud_test.rs +++ b/crates/e2e_test/src/admin_iam_crud_test.rs @@ -33,7 +33,6 @@ use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::{Client, Config}; use reqwest::StatusCode; -use serial_test::serial; use std::error::Error; use tokio::time::{Duration, sleep}; @@ -178,7 +177,6 @@ async fn assert_admin_status( } #[tokio::test] -#[serial] async fn test_update_service_account_enforces_owner_and_parent_scope() -> TestResult { init_logging(); @@ -348,7 +346,6 @@ async fn test_update_service_account_enforces_owner_and_parent_scope() -> TestRe /// Full user -> policy -> service-account lifecycle, proving each management /// call takes effect on the data plane, not just that the endpoint answers 200. #[tokio::test] -#[serial] async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult { init_logging(); @@ -573,7 +570,6 @@ async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult { /// non-admin credential with 403 AccessDenied (sec-4 assertion pattern; the /// gate implementation itself is owned by sec-4 / admin_auth_test). #[tokio::test] -#[serial] async fn test_admin_iam_endpoints_deny_non_admin_credential() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/admin_timeout_regression_test.rs b/crates/e2e_test/src/admin_timeout_regression_test.rs index c561e5d76..94a91f03e 100644 --- a/crates/e2e_test/src/admin_timeout_regression_test.rs +++ b/crates/e2e_test/src/admin_timeout_regression_test.rs @@ -21,7 +21,6 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; use serde::Deserialize; -use serial_test::serial; use std::error::Error; use std::process::Command; use tokio::time::{Duration, sleep, timeout}; @@ -100,7 +99,6 @@ fn offline_server_count(info: &InfoMessage) -> usize { } #[tokio::test(flavor = "multi_thread")] -#[serial] async fn test_single_admin_timeout_does_not_immediately_mark_peer_offline() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/anonymous_access_test.rs b/crates/e2e_test/src/anonymous_access_test.rs index 0891ac0a9..68a4f4095 100644 --- a/crates/e2e_test/src/anonymous_access_test.rs +++ b/crates/e2e_test/src/anonymous_access_test.rs @@ -18,7 +18,6 @@ use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; use aws_sdk_s3::types::PublicAccessBlockConfiguration; -use serial_test::serial; use tracing::info; async fn setup_public_bucket( @@ -73,7 +72,6 @@ async fn anonymous_get_object( /// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it /// and no PublicAccessBlock configuration exists (ConfigNotFound). #[tokio::test] -#[serial] async fn test_anonymous_access_allowed_when_public_access_block_missing() -> Result<(), Box> { init_logging(); @@ -100,7 +98,6 @@ async fn test_anonymous_access_allowed_when_public_access_block_missing() -> Res /// Anonymous GetObject should be denied when RestrictPublicBuckets is true. #[tokio::test] -#[serial] async fn test_anonymous_access_denied_when_restrict_public_buckets_enabled() -> Result<(), Box> { init_logging(); @@ -137,7 +134,6 @@ async fn test_anonymous_access_denied_when_restrict_public_buckets_enabled() /// Anonymous GetObject should succeed when PublicAccessBlock exists but /// RestrictPublicBuckets is explicitly false. #[tokio::test] -#[serial] async fn test_anonymous_access_allowed_when_restrict_public_buckets_disabled() -> Result<(), Box> { init_logging(); @@ -176,7 +172,6 @@ async fn test_anonymous_access_allowed_when_restrict_public_buckets_disabled() /// reaches authorization through a fallback branch, and that branch has to apply the /// same public-access gate as a direct grant. #[tokio::test] -#[serial] async fn ghsa_x298_anonymous_list_object_versions_denied_when_restrict_public_buckets_enabled() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/api_rate_limit_test.rs b/crates/e2e_test/src/api_rate_limit_test.rs index 28f80517d..e7a3e682d 100644 --- a/crates/e2e_test/src/api_rate_limit_test.rs +++ b/crates/e2e_test/src/api_rate_limit_test.rs @@ -18,13 +18,11 @@ //! completely inert with default configuration. use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; -use serial_test::serial; use tracing::info; type TestResult = Result<(), Box>; #[tokio::test] -#[serial] async fn api_rate_limit_enforces_429_with_retry_after_when_enabled() -> TestResult { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -87,7 +85,6 @@ async fn api_rate_limit_enforces_429_with_retry_after_when_enabled() -> TestResu } #[tokio::test] -#[serial] async fn api_rate_limit_bucket_dimension_throttles_per_bucket() -> TestResult { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -133,7 +130,6 @@ async fn api_rate_limit_bucket_dimension_throttles_per_bucket() -> TestResult { } #[tokio::test] -#[serial] async fn api_rate_limit_stays_inert_by_default() -> TestResult { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/archive_download_integrity_test.rs b/crates/e2e_test/src/archive_download_integrity_test.rs index ac23f5d70..e043c6105 100644 --- a/crates/e2e_test/src/archive_download_integrity_test.rs +++ b/crates/e2e_test/src/archive_download_integrity_test.rs @@ -24,7 +24,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::{pre_sign_v4, sign_v4}; use s3s::Body; - use serial_test::serial; use sha2::{Digest, Sha256}; use std::error::Error; use std::io::{Cursor, Write}; @@ -339,7 +338,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_put_allows_content_encoding_by_default() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -367,7 +365,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_put_rejects_content_encoding_when_strict_mode_enabled() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -391,7 +388,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_put_with_aws_chunked_does_not_persist_content_encoding_by_default() -> Result<(), Box> { init_logging(); @@ -427,7 +423,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_put_with_aws_chunked_and_effective_encoding_roundtrips_by_default() -> Result<(), Box> { init_logging(); @@ -463,7 +458,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_put_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -498,7 +492,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_put_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled() -> Result<(), Box> { init_logging(); @@ -529,7 +522,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_download_roundtrip_with_http_compression_enabled() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -591,7 +583,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_multipart_roundtrip_preserves_bytes() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -687,7 +678,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multipart_get_ignores_empty_conditional_etag_headers() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -723,7 +713,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_roundtrips_by_default() -> Result<(), Box> { init_logging(); @@ -753,7 +742,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_multipart_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box> { init_logging(); @@ -783,7 +771,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled() -> Result<(), Box> { init_logging(); @@ -816,7 +803,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_presigned_get_and_reverse_proxy_preserve_multipart_bytes() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/bucket_logging_test.rs b/crates/e2e_test/src/bucket_logging_test.rs index 79d2e6843..e60483097 100644 --- a/crates/e2e_test/src/bucket_logging_test.rs +++ b/crates/e2e_test/src/bucket_logging_test.rs @@ -24,11 +24,9 @@ mod tests { }; use http::Method; use http::header::CONTENT_TYPE; - use serial_test::serial; use tracing::info; #[tokio::test] - #[serial] async fn test_dummy_bucket_compatibility_endpoints() { init_logging(); info!("Starting test: dummy-compat bucket APIs should match S3-compatible behavior"); @@ -236,7 +234,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_dummy_bucket_compatibility_endpoints_no_such_bucket() { init_logging(); info!("Starting test: dummy-compat bucket APIs should return NoSuchBucket for missing bucket"); @@ -392,7 +389,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_dummy_bucket_endpoints_http_contracts() { init_logging(); info!("Starting test: dummy-compat bucket API HTTP contracts"); diff --git a/crates/e2e_test/src/bucket_policy_check_test.rs b/crates/e2e_test/src/bucket_policy_check_test.rs index 9b958f343..e71e5fe97 100644 --- a/crates/e2e_test/src/bucket_policy_check_test.rs +++ b/crates/e2e_test/src/bucket_policy_check_test.rs @@ -18,7 +18,6 @@ use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::{Client, Config}; -use serial_test::serial; use tracing::info; async fn create_user( @@ -51,7 +50,6 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: } #[tokio::test] -#[serial] async fn test_bucket_policy_authenticated_user() -> Result<(), Box> { init_logging(); if !crate::common::awscurl_available() { diff --git a/crates/e2e_test/src/bucket_stats_regression_test.rs b/crates/e2e_test/src/bucket_stats_regression_test.rs index 3df98f1ef..7754c0639 100644 --- a/crates/e2e_test/src/bucket_stats_regression_test.rs +++ b/crates/e2e_test/src/bucket_stats_regression_test.rs @@ -35,7 +35,6 @@ mod tests { use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; use rustfs_data_usage::DataUsageInfo; - use serial_test::serial; use std::error::Error; use tokio::time::{Duration, sleep}; use tracing::info; @@ -59,7 +58,6 @@ mod tests { /// 3. Query admin data usage API /// 4. Verify object count > 0 #[tokio::test] - #[serial] async fn test_bucket_object_count_updates_after_put() -> TestResult { init_logging(); info!("RT-09: bucket object count updates after PUT"); @@ -126,7 +124,6 @@ mod tests { /// Regression pattern: stats remain unchanged after objects are deleted /// (rustfs#5615). #[tokio::test] - #[serial] async fn test_bucket_object_count_updates_after_delete() -> TestResult { init_logging(); info!("RT-09b: bucket object count updates after DELETE"); @@ -220,7 +217,6 @@ mod tests { /// Regression pattern: DataUsageInfo undercounts versioned bucket versions /// and delete markers (rustfs#3898). #[tokio::test] - #[serial] async fn test_versioned_bucket_stats_count_all_versions() -> TestResult { init_logging(); info!("RT-09c: versioned bucket stats count all versions"); diff --git a/crates/e2e_test/src/checksum_upload_test.rs b/crates/e2e_test/src/checksum_upload_test.rs index efa4a81b8..814d18d5a 100644 --- a/crates/e2e_test/src/checksum_upload_test.rs +++ b/crates/e2e_test/src/checksum_upload_test.rs @@ -26,7 +26,6 @@ mod tests { use base64::Engine; use md5::{Digest as Md5Digest, Md5}; use rustfs_rio::{Checksum, ChecksumType as RioChecksumType}; - use serial_test::serial; use sha2::Sha256; use tracing::info; @@ -90,7 +89,6 @@ mod tests { /// PutObject with Content-MD5: upload succeeds and GetObject returns same content. #[tokio::test] - #[serial] async fn test_put_object_with_content_md5() { init_logging(); info!("TEST: PutObject with Content-MD5"); @@ -126,7 +124,6 @@ mod tests { /// PutObject with x-amz-checksum-sha256: upload succeeds and GetObject returns same content. #[tokio::test] - #[serial] async fn test_put_object_with_checksum_sha256() { init_logging(); info!("TEST: PutObject with x-amz-checksum-sha256"); @@ -164,7 +161,6 @@ mod tests { /// PutObject with a SHA256 checksum that does NOT match the body must be /// rejected (BadDigest / checksum mismatch), NOT accepted with HTTP 200. #[tokio::test] - #[serial] async fn test_put_object_rejects_mismatched_sha256() { init_logging(); info!("TEST: PutObject rejects mismatched x-amz-checksum-sha256 (issue #4341)"); @@ -212,7 +208,6 @@ mod tests { /// After PutObject with a correct SHA256 checksum, HeadObject with /// ChecksumMode=ENABLED must return that stored base64 SHA256 digest. #[tokio::test] - #[serial] async fn test_head_object_returns_stored_sha256() { init_logging(); info!("TEST: HeadObject returns stored SHA256 with ChecksumMode=ENABLED (issue #4341)"); @@ -258,7 +253,6 @@ mod tests { /// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content. /// Uses part size >= 5MB (server minimum) for two parts. #[tokio::test] - #[serial] async fn test_multipart_upload_with_checksum() { init_logging(); info!("TEST: MultipartUpload with checksum (checksum_sha256 on parts)"); @@ -356,7 +350,6 @@ mod tests { /// Regression test for issue #2282: /// CRC64NVME full-object checksum should match between direct PutObject and multipart upload. #[tokio::test] - #[serial] async fn test_crc64nvme_matches_between_put_object_and_multipart_upload() { init_logging(); info!("TEST: CRC64NVME matches between direct PutObject and multipart upload"); @@ -492,7 +485,6 @@ mod tests { /// value is rejected with BadDigest and nothing is stored. Full HEAD/GET header /// echo round-trip is additionally exercised by the boto3+awscrt e2e. #[tokio::test] - #[serial] async fn test_additional_checksums_verify_on_write() { init_logging(); info!("TEST: additional checksums (XXHash3/64/128, SHA-512, MD5) verify-on-write"); diff --git a/crates/e2e_test/src/cluster_concurrency_test.rs b/crates/e2e_test/src/cluster_concurrency_test.rs index 7702dfac3..cc2f2cddc 100644 --- a/crates/e2e_test/src/cluster_concurrency_test.rs +++ b/crates/e2e_test/src/cluster_concurrency_test.rs @@ -16,7 +16,6 @@ use crate::common::RustFSTestClusterEnvironment; use aws_sdk_s3::Client; use aws_sdk_s3::error::SdkError; use bytes::Bytes; -use serial_test::serial; use std::sync::Arc; use tokio::sync::Barrier; use tracing::{info, warn}; @@ -135,7 +134,6 @@ async fn run_race_iteration( } #[tokio::test] -#[serial] async fn test_conditional_put_race_cluster() -> Result<(), Box> { crate::common::init_logging(); info!("Starting conditional PUT race test with auto cluster"); @@ -192,7 +190,6 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box Result<(), Box> { crate::common::init_logging(); info!("Starting basic conditional PUT test with auto cluster"); diff --git a/crates/e2e_test/src/cluster_multidrive_pool_test.rs b/crates/e2e_test/src/cluster_multidrive_pool_test.rs index 3f9193984..3f88e5af4 100644 --- a/crates/e2e_test/src/cluster_multidrive_pool_test.rs +++ b/crates/e2e_test/src/cluster_multidrive_pool_test.rs @@ -31,7 +31,6 @@ //! (toxiproxy / socket proxy) and 5GiB large-object budgets. use crate::common::{ClusterTopology, RustFSTestClusterEnvironment}; -use serial_test::serial; type TestResult = Result<(), Box>; @@ -58,7 +57,6 @@ async fn put_get_roundtrip(cluster: &RustFSTestClusterEnvironment, key: &str, pa /// 4 nodes x 2 drives, single pool: the multi-drive layout boots and round-trips. #[tokio::test] -#[serial] async fn cluster_multidrive_single_pool_smoke() -> TestResult { crate::common::init_logging(); @@ -81,7 +79,6 @@ async fn cluster_multidrive_single_pool_smoke() -> TestResult { /// Two single-node pools, 2 drives each: the multi-pool layout boots and /// round-trips. Every pool is a distinct erasure pool (`pool_idx` 0 and 1). #[tokio::test] -#[serial] async fn cluster_two_pool_smoke() -> TestResult { crate::common::init_logging(); diff --git a/crates/e2e_test/src/compression_test.rs b/crates/e2e_test/src/compression_test.rs index 775decfa6..7014f5f18 100644 --- a/crates/e2e_test/src/compression_test.rs +++ b/crates/e2e_test/src/compression_test.rs @@ -3,7 +3,6 @@ use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; -use serial_test::serial; use std::fs; use std::path::PathBuf; use std::process::Command; @@ -102,7 +101,6 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul } #[tokio::test] -#[serial] async fn test_compression_roundtrip() -> Result<(), Box> { init_logging(); info!("Starting compression roundtrip test"); @@ -230,7 +228,6 @@ async fn fetch_range( /// (rustfs/rustfs#5957: multipart uploads previously bypassed disk compression /// entirely). #[tokio::test] -#[serial] async fn test_compression_multipart_roundtrip() -> Result<(), Box> { init_logging(); info!("Starting multipart compression roundtrip test"); @@ -349,7 +346,6 @@ const MPU_HIGH_RATIO_BUCKET: &str = "compression-mpu-high-ratio-bucket"; /// reproduced the mid-payload Pending truncation (rustfs/rustfs#5957). Every GET shape must return /// the exact original bytes, and the stored size must show the data really was compressed. #[tokio::test] -#[serial] async fn test_compression_multipart_high_ratio_binary_roundtrip() -> Result<(), Box> { init_logging(); info!("Starting multipart high-ratio binary compression roundtrip test"); @@ -446,7 +442,6 @@ const MPU_COPY_RANGE_LEN: usize = 5 * 1024 * 1024; /// range must be decompressed on read and re-compressed into the destination part, so the final /// object has to match "source prefix + uploaded tail" byte for byte. #[tokio::test] -#[serial] async fn test_compression_multipart_upload_part_copy_roundtrip() -> Result<(), Box> { init_logging(); info!("Starting multipart upload-part-copy compression roundtrip test"); @@ -570,7 +565,6 @@ const MPU_THREE_PARTS_TAIL_SIZE: usize = 512 * 1024; /// Three-part upload with uneven part sizes: each partNumber GET must map back to exactly one /// compressed part stream, and a suffix range must resolve inside the trailing part. #[tokio::test] -#[serial] async fn test_compression_multipart_three_parts_part_number_gets() -> Result<(), Box> { init_logging(); info!("Starting three-part multipart compression partNumber test"); @@ -689,7 +683,6 @@ async fn start_rustfs_with_compression_and_sse( /// shape must still return the original plaintext bytes. Physical size must shrink because the /// compression runs before encryption. #[tokio::test] -#[serial] async fn test_compression_multipart_sse_s3_roundtrip() -> Result<(), Box> { use aws_sdk_s3::types::ServerSideEncryption; diff --git a/crates/e2e_test/src/connection_cap_test.rs b/crates/e2e_test/src/connection_cap_test.rs index bb835a712..b59584164 100644 --- a/crates/e2e_test/src/connection_cap_test.rs +++ b/crates/e2e_test/src/connection_cap_test.rs @@ -18,7 +18,6 @@ //! concurrency โ€” a queued connection is served only after a held one closes. use crate::common::{RustFSTestEnvironment, init_logging}; -use serial_test::serial; use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; @@ -57,7 +56,6 @@ async fn read_response_head(stream: &mut TcpStream, dur: Duration) -> Option TestResult { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -89,7 +87,6 @@ async fn open_and_stall(addr: &str) -> std::io::Result { } #[tokio::test] -#[serial] async fn connection_cap_blocks_excess_connections_until_permits_free() -> TestResult { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/console_smoke_test.rs b/crates/e2e_test/src/console_smoke_test.rs index cca86b7e3..431eee313 100644 --- a/crates/e2e_test/src/console_smoke_test.rs +++ b/crates/e2e_test/src/console_smoke_test.rs @@ -33,7 +33,6 @@ //! serve the unauthenticated console endpoints at all. use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; -use serial_test::serial; use std::error::Error; use tokio::time::{Duration, sleep}; @@ -58,7 +57,6 @@ async fn wait_for_console_ready(console_base: &str) -> Result TestResult { init_logging(); diff --git a/crates/e2e_test/src/content_encoding_test.rs b/crates/e2e_test/src/content_encoding_test.rs index f99117424..abd0bb1ba 100644 --- a/crates/e2e_test/src/content_encoding_test.rs +++ b/crates/e2e_test/src/content_encoding_test.rs @@ -22,12 +22,10 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use tracing::info; /// Verify Content-Encoding header roundtrips through PUT, GET, and HEAD operations #[tokio::test] - #[serial] async fn test_content_encoding_roundtrip() { init_logging(); info!("Starting Content-Encoding roundtrip test"); @@ -105,7 +103,6 @@ mod tests { /// Issue #1857: Content-Encoding "aws-chunked" is used by SigV4 streaming clients and must /// not be stored or returned. Upload with aws-chunked and verify GET/HEAD do not return it. #[tokio::test] - #[serial] async fn test_content_encoding_aws_chunked_not_returned_issue_1857() { init_logging(); info!("Issue #1857: aws-chunked must not be persisted or returned"); @@ -161,7 +158,6 @@ mod tests { /// Issue #2475 / Route A: when aws-chunked is combined with an effective object encoding, /// only the effective encoding should roundtrip through GET/HEAD. #[tokio::test] - #[serial] async fn test_content_encoding_aws_chunked_with_effective_encoding_roundtrip() { init_logging(); info!("aws-chunked,gzip should persist only gzip"); diff --git a/crates/e2e_test/src/copy_object_checksum_test.rs b/crates/e2e_test/src/copy_object_checksum_test.rs index bdd6251a9..b4ac11bbb 100644 --- a/crates/e2e_test/src/copy_object_checksum_test.rs +++ b/crates/e2e_test/src/copy_object_checksum_test.rs @@ -30,7 +30,6 @@ mod tests { use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use rustfs_rio::{Checksum, ChecksumType as RioChecksumType}; - use serial_test::serial; use sha2::{Digest, Sha256}; use tracing::info; @@ -114,7 +113,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_copy_supports_all_checksum_algorithms() { init_logging(); @@ -196,7 +194,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_copy_without_algorithm_preserves_every_supported_source_checksum() { init_logging(); @@ -262,7 +259,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_copy_without_algorithm_preserves_composite_checksum_type() { init_logging(); @@ -352,7 +348,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_copy_rejects_unknown_algorithm_without_destination_mutation() { init_logging(); @@ -453,7 +448,6 @@ mod tests { /// bytes, return it in `CopyObjectResult.ChecksumSHA256`, and persist it so a checksum-mode /// HEAD on the destination returns the identical value. #[tokio::test] - #[serial] async fn test_copy_with_checksum_algorithm_returns_and_persists_sha256() { init_logging(); info!("Issue #4996: CopyObject with ChecksumAlgorithm=SHA256 must return and persist the checksum"); @@ -523,7 +517,6 @@ mod tests { /// No algorithm requested: when the source object already carries a checksum, the copy must /// preserve it on the destination (AWS default), visible via a checksum-mode HEAD. #[tokio::test] - #[serial] async fn test_copy_without_algorithm_preserves_source_checksum() { init_logging(); info!("Issue #4996: CopyObject without ChecksumAlgorithm must preserve the source object's checksum"); @@ -603,7 +596,6 @@ mod tests { /// checksum-not-inherited path, and exercises the CRC32 code path (a different branch of /// ChecksumType::from_string than SHA256). #[tokio::test] - #[serial] async fn test_copy_requested_algorithm_overrides_source_checksum() { init_logging(); info!("Issue #4996: a requested CopyObject checksum algorithm must override the source object's algorithm"); diff --git a/crates/e2e_test/src/copy_object_metadata_test.rs b/crates/e2e_test/src/copy_object_metadata_test.rs index f56fe7f41..289c2abc2 100644 --- a/crates/e2e_test/src/copy_object_metadata_test.rs +++ b/crates/e2e_test/src/copy_object_metadata_test.rs @@ -22,11 +22,9 @@ mod tests { use aws_sdk_s3::types::{ BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, MetadataDirective, StorageClass, VersioningConfiguration, }; - use serial_test::serial; use tracing::info; #[tokio::test] - #[serial] async fn copy_object_standard_metadata_copy_replace_and_clear() { init_logging(); info!("Issue #2789: self-copy metadata replacement must preserve object data"); @@ -300,7 +298,6 @@ mod tests { } #[tokio::test] - #[serial] async fn copy_object_replace_accepts_each_standard_field_independently() { init_logging(); let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); @@ -416,7 +413,6 @@ mod tests { } #[tokio::test] - #[serial] async fn copy_object_replace_handles_versioned_multipart_source() { init_logging(); let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); @@ -530,7 +526,6 @@ mod tests { } #[tokio::test] - #[serial] async fn invalid_replacement_metadata_does_not_mutate_destination() { init_logging(); let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); diff --git a/crates/e2e_test/src/copy_object_tagging_test.rs b/crates/e2e_test/src/copy_object_tagging_test.rs index 23861a705..ad1103658 100644 --- a/crates/e2e_test/src/copy_object_tagging_test.rs +++ b/crates/e2e_test/src/copy_object_tagging_test.rs @@ -21,7 +21,6 @@ mod tests { use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, MetadataDirective, TaggingDirective, VersioningConfiguration}; - use serial_test::serial; use std::collections::BTreeMap; async fn object_tags(client: &Client, bucket: &str, key: &str) -> BTreeMap { @@ -39,7 +38,6 @@ mod tests { } #[tokio::test] - #[serial] async fn copy_object_applies_copy_replace_and_empty_tagging_directives() { init_logging(); let mut env = RustFSTestEnvironment::new() @@ -305,7 +303,6 @@ mod tests { } #[tokio::test] - #[serial] async fn copy_object_tag_replacement_honors_request_tag_policy_denial() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/copy_object_version_restore_test.rs b/crates/e2e_test/src/copy_object_version_restore_test.rs index 25958bc35..e99da5843 100644 --- a/crates/e2e_test/src/copy_object_version_restore_test.rs +++ b/crates/e2e_test/src/copy_object_version_restore_test.rs @@ -21,11 +21,9 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; - use serial_test::serial; use tracing::info; #[tokio::test] - #[serial] async fn test_self_copy_of_historical_version_restores_data_and_metadata() { init_logging(); info!("Issue #4238: self-copy of a historical version must be allowed and preserve metadata"); @@ -165,7 +163,6 @@ mod tests { /// version copied via `x-amz-copy-source-version-id` (SDK `CopySourceVersionId`), kept distinct /// from the newly created destination `x-amz-version-id`. #[tokio::test] - #[serial] async fn test_copy_of_non_latest_source_version_returns_copy_source_version_id() { init_logging(); info!("Issue #4976: versioned CopyObject must return x-amz-copy-source-version-id for the exact source version"); diff --git a/crates/e2e_test/src/copy_source_invalid_date_test.rs b/crates/e2e_test/src/copy_source_invalid_date_test.rs index 1513dddce..09f15a31b 100644 --- a/crates/e2e_test/src/copy_source_invalid_date_test.rs +++ b/crates/e2e_test/src/copy_source_invalid_date_test.rs @@ -47,7 +47,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; /// Signed raw `PUT` copy request with an explicit copy-source conditional @@ -84,7 +83,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_copy_source_if_unmodified_since_valid_and_invalid() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/create_bucket_region_test.rs b/crates/e2e_test/src/create_bucket_region_test.rs index 44f3ea13b..7f19b2297 100644 --- a/crates/e2e_test/src/create_bucket_region_test.rs +++ b/crates/e2e_test/src/create_bucket_region_test.rs @@ -25,13 +25,11 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::types::{BucketLocationConstraint, CreateBucketConfiguration}; - use serial_test::serial; use std::error::Error; /// `CreateBucket` with a `LocationConstraint` body must pass SigV4 validation /// and create the bucket, mirroring `minio-go` `MakeBucket(bucket, "us-east-1")`. #[tokio::test] - #[serial] async fn test_create_bucket_with_us_east_1_location_constraint() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -62,7 +60,6 @@ mod tests { /// A plain `CreateBucket` (no body) must also succeed; guards against a /// regression where an empty body would be hashed incorrectly during SigV4. #[tokio::test] - #[serial] async fn test_create_bucket_without_location_constraint() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/data_usage_test.rs b/crates/e2e_test/src/data_usage_test.rs index d99d0750a..2dfd56eb5 100644 --- a/crates/e2e_test/src/data_usage_test.rs +++ b/crates/e2e_test/src/data_usage_test.rs @@ -15,7 +15,6 @@ use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; use rustfs_data_usage::DataUsageInfo; -use serial_test::serial; use tokio::time::{Duration, sleep}; use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging}; @@ -60,7 +59,6 @@ where /// Regression test for data usage accuracy (issue #1012). /// Launches rustfs, writes 1000 objects, then asserts admin data usage reports the full count. #[tokio::test(flavor = "multi_thread")] -#[serial] #[ignore = "Starts a rustfs server and requires awscurl; enable when running full E2E"] async fn data_usage_reports_all_objects() -> Result<(), Box> { init_logging(); @@ -118,7 +116,6 @@ async fn data_usage_reports_all_objects() -> Result<(), Box Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/delete_marker_migration_semantics_test.rs b/crates/e2e_test/src/delete_marker_migration_semantics_test.rs index 5a51c053a..8ae3b75c3 100644 --- a/crates/e2e_test/src/delete_marker_migration_semantics_test.rs +++ b/crates/e2e_test/src/delete_marker_migration_semantics_test.rs @@ -18,7 +18,6 @@ mod tests { use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; - use serial_test::serial; async fn create_versioned_bucket(client: &Client, bucket: &str) { client @@ -72,7 +71,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_versioning_only_delete_marker_has_minio_compatible_visibility_for_migration_proof() { init_logging(); let mut env = RustFSTestEnvironment::new().await.expect("create test environment"); @@ -113,7 +111,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_versioning_delete_marker_plus_history_remains_visible_for_migration_proof() { init_logging(); let mut env = RustFSTestEnvironment::new().await.expect("create test environment"); diff --git a/crates/e2e_test/src/delete_object_no_content_length_test.rs b/crates/e2e_test/src/delete_object_no_content_length_test.rs index 76bee5e3d..6f74ebf3a 100644 --- a/crates/e2e_test/src/delete_object_no_content_length_test.rs +++ b/crates/e2e_test/src/delete_object_no_content_length_test.rs @@ -24,7 +24,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; @@ -92,7 +91,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_delete_object_version_without_content_length_succeeds() -> Result<(), Box> { init_logging(); info!("๐Ÿงช TEST: signed DELETE Object?versionId succeeds without Content-Length"); diff --git a/crates/e2e_test/src/delete_objects_versioning_test.rs b/crates/e2e_test/src/delete_objects_versioning_test.rs index 41f872370..acab0d320 100644 --- a/crates/e2e_test/src/delete_objects_versioning_test.rs +++ b/crates/e2e_test/src/delete_objects_versioning_test.rs @@ -29,7 +29,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration}; - use serial_test::serial; use tracing::info; fn create_s3_client(env: &RustFSTestEnvironment) -> Client { @@ -42,7 +41,6 @@ mod tests { /// a versioned bucket, calling `list_object_versions` **immediately** (with /// no sleep) returns the newly-created DeleteMarker with `is_latest = true`. #[tokio::test] - #[serial] async fn test_delete_objects_delete_marker_immediately_visible() { init_logging(); info!("๐Ÿงช TEST: DeleteMarker from delete_objects is immediately visible via list_object_versions"); @@ -190,7 +188,6 @@ mod tests { /// a single `delete_objects` call all have their delete markers visible /// immediately afterwards. #[tokio::test] - #[serial] async fn test_delete_objects_multiple_keys_delete_markers_immediately_visible() { init_logging(); info!("๐Ÿงช TEST: Multiple delete markers from delete_objects are immediately visible"); diff --git a/crates/e2e_test/src/delete_regression_test.rs b/crates/e2e_test/src/delete_regression_test.rs index 1f61a55c2..3bcc6fa5c 100644 --- a/crates/e2e_test/src/delete_regression_test.rs +++ b/crates/e2e_test/src/delete_regression_test.rs @@ -33,7 +33,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration}; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -51,7 +50,6 @@ mod tests { /// 4. Verify the object is NOT in LIST /// 5. Verify HEAD returns 404 #[tokio::test] - #[serial] async fn test_delete_removes_object_from_list() -> TestResult { init_logging(); info!("RT-05: delete removes object from list"); @@ -132,7 +130,6 @@ mod tests { /// Regression pattern: batch delete returns success but some objects /// remain in LIST. #[tokio::test] - #[serial] async fn test_batch_delete_removes_all_objects() -> TestResult { init_logging(); info!("RT-05c: batch delete removes all objects"); @@ -212,7 +209,6 @@ mod tests { /// Covers the pattern where permanent deletion of a specific version /// fails with FileAccessDenied (rustfs#4978). #[tokio::test] - #[serial] async fn test_versioned_permanent_delete() -> TestResult { init_logging(); info!("RT-05d: versioned permanent delete"); @@ -283,7 +279,6 @@ mod tests { /// Covers the pattern where creating a delete marker and then listing /// versions shows incorrect state (rustfs#760). #[tokio::test] - #[serial] async fn test_versioned_delete_marker_and_list_consistency() -> TestResult { init_logging(); info!("RT-05e: versioned delete marker and list consistency"); @@ -379,7 +374,6 @@ mod tests { /// Regression pattern: after delete, the object data files remain on disk /// (rustfs#5029: Node Does Not Remove Files After Reconnect). #[tokio::test] - #[serial] async fn test_delete_removes_object_head_returns_404() -> TestResult { init_logging(); info!("RT-05f: delete โ†’ HEAD 404 consistency"); diff --git a/crates/e2e_test/src/distributed_startup_regression_test.rs b/crates/e2e_test/src/distributed_startup_regression_test.rs index b7387b50b..ff2da755b 100644 --- a/crates/e2e_test/src/distributed_startup_regression_test.rs +++ b/crates/e2e_test/src/distributed_startup_regression_test.rs @@ -32,7 +32,6 @@ mod tests { use crate::common::{RustFSTestClusterEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use std::error::Error; use tokio::time::{Duration, sleep}; use tracing::info; @@ -50,7 +49,6 @@ mod tests { /// 3. Verify all nodes report healthy /// 4. Verify S3 operations work through any node #[tokio::test] - #[serial] async fn test_four_node_cluster_startup_and_health() -> TestResult { init_logging(); info!("RT-10: 4-node cluster startup and health"); @@ -103,7 +101,6 @@ mod tests { /// Regression pattern: after a node restart, it cannot rejoin the cluster /// or enters a faulty state (rustfs#2601). #[tokio::test] - #[serial] async fn test_cluster_survives_node_restart() -> TestResult { init_logging(); info!("RT-10b: cluster survives node restart"); @@ -168,7 +165,6 @@ mod tests { /// Regression pattern: bucket metadata is not replicated to all nodes, /// causing NoSuchBucket errors on some nodes (rustfs#3191). #[tokio::test] - #[serial] async fn test_bucket_visible_from_all_nodes() -> TestResult { init_logging(); info!("RT-10c: bucket visible from all nodes"); diff --git a/crates/e2e_test/src/existing_object_tag_policy_test.rs b/crates/e2e_test/src/existing_object_tag_policy_test.rs index 9a26e95db..17c34f166 100644 --- a/crates/e2e_test/src/existing_object_tag_policy_test.rs +++ b/crates/e2e_test/src/existing_object_tag_policy_test.rs @@ -23,7 +23,6 @@ use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging}; use aws_sdk_s3::{Client, Config}; -use serial_test::serial; use tracing::info; use uuid::Uuid; @@ -174,7 +173,6 @@ async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) { /// IAM identity policy: GetObject allowed only when `s3:ExistingObjectTag/security` == `public`. #[tokio::test] -#[serial] async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box> { init_logging(); if !awscurl_available() { @@ -233,7 +231,6 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box< /// Bucket policy: same `ExistingObjectTag` condition; user has no canned IAM policy attached. #[tokio::test] -#[serial] async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box> { init_logging(); if !awscurl_available() { @@ -295,7 +292,6 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B /// STS `AssumeRole` with inline `Policy` (session policy): GetObject only when `ExistingObjectTag/security` is `public`. #[tokio::test] -#[serial] async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box> { init_logging(); if !awscurl_available() { @@ -372,7 +368,6 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result /// STS inline session policy: DeleteObjects must evaluate `s3:DeleteObject` per requested object key. #[tokio::test] -#[serial] async fn test_e2e_sts_session_policy_delete_objects_object_prefix_only() -> Result<(), Box> { init_logging(); if !awscurl_available() { diff --git a/crates/e2e_test/src/get_codec_streaming_compat_test.rs b/crates/e2e_test/src/get_codec_streaming_compat_test.rs index eb758aa7f..4fdc8326f 100644 --- a/crates/e2e_test/src/get_codec_streaming_compat_test.rs +++ b/crates/e2e_test/src/get_codec_streaming_compat_test.rs @@ -66,7 +66,6 @@ mod tests { use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; - use serial_test::serial; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::error::Error; @@ -277,7 +276,6 @@ mod tests { } #[tokio::test] - #[serial] async fn codec_streaming_matches_legacy_duplex_body_and_headers() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/group_delete_test.rs b/crates/e2e_test/src/group_delete_test.rs index 147feb89d..38d053f99 100644 --- a/crates/e2e_test/src/group_delete_test.rs +++ b/crates/e2e_test/src/group_delete_test.rs @@ -17,7 +17,6 @@ use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging}; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::{Client, Config}; -use serial_test::serial; use tracing::info; fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { @@ -84,7 +83,6 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo /// Test that deleting a group with members fails, and deleting an empty group succeeds. #[tokio::test(flavor = "multi_thread")] -#[serial] #[ignore = "requires awscurl and spawns a real RustFS server"] async fn test_delete_group_requires_empty_membership() -> Result<(), Box> { init_logging(); @@ -144,7 +142,6 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box Result<(), Box> { init_logging(); @@ -212,7 +209,6 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/head_object_consistency_test.rs b/crates/e2e_test/src/head_object_consistency_test.rs index 93e7b8a4e..a7499224b 100644 --- a/crates/e2e_test/src/head_object_consistency_test.rs +++ b/crates/e2e_test/src/head_object_consistency_test.rs @@ -16,7 +16,6 @@ use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; use aws_sdk_s3::presigning::PresigningConfig; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; -use serial_test::serial; use std::time::Duration; use tracing::info; @@ -29,7 +28,6 @@ fn list_contains_key(output: &aws_sdk_s3::operation::list_objects_v2::ListObject } #[tokio::test] -#[serial] async fn head_object_consistency_after_write_and_multipart_and_presigned_head() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/head_object_range_test.rs b/crates/e2e_test/src/head_object_range_test.rs index 5cc4b7a12..577440020 100644 --- a/crates/e2e_test/src/head_object_range_test.rs +++ b/crates/e2e_test/src/head_object_range_test.rs @@ -1,6 +1,5 @@ use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; -use serial_test::serial; use tracing::info; const RANGE_HEAD_BUCKET: &str = "range-head-test-bucket"; @@ -8,7 +7,6 @@ const RANGE_HEAD_KEY: &str = "range-head-object.bin"; const ACCEPT_RANGES_BYTES: &str = "bytes"; #[tokio::test] -#[serial] async fn head_object_advertises_accept_ranges() -> Result<(), Box> { init_logging(); info!("Starting HeadObject Accept-Ranges regression test"); diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index 749f21d7f..dac1eef5e 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -19,7 +19,6 @@ mod tests { use crate::chaos::signed_admin_post; use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use std::collections::HashSet; use std::error::Error; use std::path::{Path, PathBuf}; @@ -63,7 +62,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_auto_heal_rebuilds_runtime_wiped_disk_without_restart() { init_logging(); info!("Issue #1533: auto heal should rebuild a runtime-wiped disk in a 4-disk single-node erasure set without restart"); @@ -182,7 +180,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_admin_deep_heal_rebuilds_cleared_disk_in_single_node_erasure_set() { init_logging(); info!("Discussion #2964: admin deep heal should rebuild a wiped disk in a 4-disk single-node erasure set"); @@ -332,7 +329,6 @@ mod tests { } #[tokio::test(flavor = "multi_thread")] - #[serial] async fn test_cluster_root_heal_rebuilds_replaced_remote_disk() -> Result<(), Box> { init_logging(); info!("Root recursive heal should rebuild data on a remote node after its disk is replaced and the node rejoins"); @@ -444,7 +440,6 @@ mod tests { /// topology early-return or the merge hard-fail) turns the down-window /// response into a 500 and fails this test. #[tokio::test] - #[serial] async fn test_background_heal_status_degrades_while_peer_down_and_recovers_after_rejoin() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs b/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs index c7632d013..bb751fe55 100644 --- a/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs +++ b/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs @@ -101,7 +101,6 @@ use rustfs_config::{ }; use rustfs_protos::canonical_make_volume_request_body; use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse, PingRequest, PingResponse}; -use serial_test::serial; use sha2::{Digest, Sha256}; use std::error::Error; use tonic::{Code, Request, Response, Status}; @@ -397,7 +396,6 @@ fn assert_rejected(result: Result, expected: Code, e /// Grouped into one server start because each case is independent and spawning /// a `rustfs` process per assertion would dominate the runtime. #[tokio::test] -#[serial] async fn internode_rpc_signature_default_posture_e2e() -> TestResult { init_logging(); align_rpc_secret_with_server(); @@ -424,7 +422,6 @@ async fn internode_rpc_signature_default_posture_e2e() -> TestResult { /// epoch is learned from a real response, then the same server is restarted in place to prove its /// replacement epoch rejects the captured request even though the nonce cache is necessarily new. #[tokio::test] -#[serial] async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> TestResult { init_logging(); align_rpc_secret_with_server(); @@ -497,7 +494,6 @@ async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> Te /// A mutating v2 request cannot use that lane; once the epoch proof is returned, the first v3 /// mutation succeeds. This protects a server restart without reopening a general downgrade path. #[tokio::test] -#[serial] async fn replay_scope_strict_requires_v3_after_ping_bootstrap_e2e() -> TestResult { init_logging(); align_rpc_secret_with_server(); @@ -704,7 +700,6 @@ async fn legacy_only_signature_is_accepted_in_default_posture(url: &str) { /// /// The paired v2 positive control rules out "strict simply breaks everything". #[tokio::test] -#[serial] async fn signature_strict_rejects_legacy_only_downgrade() -> TestResult { init_logging(); align_rpc_secret_with_server(); @@ -741,7 +736,6 @@ async fn signature_strict_rejects_legacy_only_downgrade() -> TestResult { /// takes the still-open legacy lane), which is what pins the rejection to the /// handler's digest gate; the cited message confirms which check spoke. #[tokio::test] -#[serial] async fn body_digest_strict_rejects_digestless_mutation() -> TestResult { init_logging(); align_rpc_secret_with_server(); diff --git a/crates/e2e_test/src/kms/bucket_default_encryption_test.rs b/crates/e2e_test/src/kms/bucket_default_encryption_test.rs index 965b33d02..fecba2b89 100644 --- a/crates/e2e_test/src/kms/bucket_default_encryption_test.rs +++ b/crates/e2e_test/src/kms/bucket_default_encryption_test.rs @@ -27,12 +27,10 @@ use aws_sdk_s3::types::{ ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, }; use rustfs_rio::{Checksum, ChecksumType}; -use serial_test::serial; use tracing::{debug, info, warn}; /// Test 1: When bucket is configured with default SSE-S3 encryption, put_object should automatically apply encryption #[tokio::test] -#[serial] async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box> { init_logging(); info!("Testing bucket default SSE-S3 encryption impact on put_object"); @@ -155,7 +153,6 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box Result<(), Box> { init_logging(); info!("Testing bucket default SSE-KMS encryption impact on put_object"); @@ -275,7 +272,6 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box Result<(), Box> { init_logging(); info!("Testing bucket default encryption impact on create_multipart_upload"); @@ -473,7 +469,6 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box Result<(), Box> { init_logging(); info!("Testing explicitly specified encryption parameters override bucket default configuration"); @@ -569,7 +564,6 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box Result<(), Box> { init_logging(); info!("Testing SSE-KMS without explicit key ID populates default key"); diff --git a/crates/e2e_test/src/kms/configured_roundtrip_test.rs b/crates/e2e_test/src/kms/configured_roundtrip_test.rs index a666ef8d7..512105d4f 100644 --- a/crates/e2e_test/src/kms/configured_roundtrip_test.rs +++ b/crates/e2e_test/src/kms/configured_roundtrip_test.rs @@ -20,7 +20,6 @@ use super::common::{ }; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, ServerSideEncryption, VersioningConfiguration}; -use serial_test::serial; use std::error::Error; use uuid::Uuid; @@ -386,7 +385,6 @@ async fn assert_versioned_sse_kms_roundtrip_and_cleanup( } #[tokio::test] -#[serial] async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult { let mut env = LocalKMSTestEnvironment::new().await?; env.base_env.start_rustfs_server(Vec::new()).await?; @@ -434,7 +432,6 @@ async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult { } #[tokio::test] -#[serial] #[ignore = "requires a Vault binary"] async fn test_configured_vault_kms_admin_and_versioned_cleanup() -> TestResult { let mut env = VaultTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs b/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs index 85abf316e..1cf19a565 100644 --- a/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs +++ b/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs @@ -32,11 +32,9 @@ use aws_sdk_s3::types::{ MetadataDirective, ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, }; -use serial_test::serial; use tracing::info; #[tokio::test] -#[serial] async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() { init_logging(); info!("same-key CopyObject with REPLACE metadata must not re-key an SSE-S3 object"); @@ -136,7 +134,6 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() { } #[tokio::test] -#[serial] async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() { init_logging(); info!("same-key CopyObject that drops SSE must rewrite the data, not orphan the ciphertext"); @@ -233,7 +230,6 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() { } #[tokio::test] -#[serial] async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decryptable() { init_logging(); info!("bucket default encryption must also keep a same-key copy off the metadata-only path"); diff --git a/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs b/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs index 34dd682d7..3241a217d 100644 --- a/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs +++ b/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs @@ -25,11 +25,9 @@ use super::common::{LocalKMSTestEnvironment, create_key_with_specific_id}; use crate::common::init_logging; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, ServerSideEncryption, VersioningConfiguration}; -use serial_test::serial; use tracing::info; #[tokio::test] -#[serial] async fn test_self_copy_of_historical_sse_s3_version_is_readable() { init_logging(); info!("Issue #4238 (SSE): restoring an encrypted historical version must stay decryptable"); diff --git a/crates/e2e_test/src/kms/encryption_metadata_test.rs b/crates/e2e_test/src/kms/encryption_metadata_test.rs index e1e082df8..a316668f6 100644 --- a/crates/e2e_test/src/kms/encryption_metadata_test.rs +++ b/crates/e2e_test/src/kms/encryption_metadata_test.rs @@ -22,7 +22,6 @@ use aws_sdk_s3::types::{ CompletedMultipartUpload, CompletedPart, ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, }; -use serial_test::serial; use std::collections::{HashMap, VecDeque}; use tracing::info; @@ -82,7 +81,6 @@ pub(super) fn assert_storage_encrypted(storage_root: &std::path::Path, bucket: & } #[tokio::test] -#[serial] async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box> { init_logging(); info!("Validating SSE-S3 managed encryption metadata exposure"); @@ -143,7 +141,6 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box Result<(), Box> { init_logging(); info!("Validating SSE-KMS managed encryption metadata (including copy)"); @@ -247,7 +244,6 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(), } #[tokio::test] -#[serial] async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box> { init_logging(); info!("Validating ciphertext persistence for multipart SSE-KMS uploads"); diff --git a/crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs b/crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs index bc6ef5b24..a6fd6cb43 100644 --- a/crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs +++ b/crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs @@ -35,7 +35,6 @@ use aws_sdk_s3::config::{Config, Credentials, Region}; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::ServerSideEncryption; -use serial_test::serial; use std::time::Duration; use tracing::info; @@ -209,7 +208,6 @@ fn disable_body(key_id: &str) -> String { /// Data-path matrix: SSE-KMS writes and reads are authorized against the resolved key. #[tokio::test] -#[serial] async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult { init_logging(); @@ -355,7 +353,6 @@ async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult { /// Runs without the SSE enforcement switch: admin scoping is unconditional, and /// leaving the switch off proves the two planes are independent. #[tokio::test] -#[serial] async fn kms_admin_per_key_authorization_negative_matrix() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/kms/kms_comprehensive_test.rs b/crates/e2e_test/src/kms/kms_comprehensive_test.rs index b4a61a7f6..584ae64eb 100644 --- a/crates/e2e_test/src/kms/kms_comprehensive_test.rs +++ b/crates/e2e_test/src/kms/kms_comprehensive_test.rs @@ -24,13 +24,11 @@ use super::common::{ test_sse_kms_encryption, test_sse_s3_encryption, }; use crate::common::{TEST_BUCKET, init_logging}; -use serial_test::serial; use tokio::time::{Duration, sleep}; use tracing::info; /// Comprehensive test: Full KMS workflow with all encryption types #[tokio::test] -#[serial] async fn test_comprehensive_kms_full_workflow() -> Result<(), Box> { init_logging(); info!("๐Ÿ Start the KMS full-featured synthesis test"); @@ -99,7 +97,6 @@ async fn test_mixed_encryption_workload( /// Comprehensive stress test: Large dataset with multiple encryption types #[tokio::test] -#[serial] async fn test_comprehensive_stress_test() -> Result<(), Box> { init_logging(); info!("๐Ÿ’ช Start the KMS stress test"); @@ -134,7 +131,6 @@ async fn test_comprehensive_stress_test() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿ” Begin the comprehensive test of encryption key isolation"); @@ -206,7 +202,6 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box Result<(), Box> { init_logging(); info!("โšก Started comprehensive testing of concurrent encryption operations"); @@ -252,7 +247,6 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿ“Š Start KMS performance benchmarking"); diff --git a/crates/e2e_test/src/kms/kms_edge_cases_test.rs b/crates/e2e_test/src/kms/kms_edge_cases_test.rs index fe4dcb66d..9d9d1a765 100644 --- a/crates/e2e_test/src/kms/kms_edge_cases_test.rs +++ b/crates/e2e_test/src/kms/kms_edge_cases_test.rs @@ -26,7 +26,6 @@ use crate::common::{TEST_BUCKET, init_logging}; use aws_sdk_s3::types::ServerSideEncryption; use base64::Engine; use md5::{Digest as Md5Digest, Md5}; -use serial_test::serial; use std::sync::Arc; use tokio::sync::Semaphore; use tracing::{info, warn}; @@ -39,7 +38,6 @@ fn md5_hex(input: impl AsRef<[u8]>) -> String { /// Test encryption of zero-byte files (empty files) #[tokio::test] -#[serial] async fn test_kms_zero_byte_file_encryption() -> Result<(), Box> { init_logging(); info!("๐Ÿงช Testing KMS encryption with zero-byte files"); @@ -113,7 +111,6 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿงช Testing KMS encryption with single-byte files"); @@ -206,7 +203,6 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿงช Testing KMS multipart upload boundary conditions"); @@ -282,7 +278,6 @@ async fn test_kms_multipart_boundary_conditions() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿงช Testing KMS invalid key scenarios and error handling"); @@ -370,7 +365,6 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿงช Testing KMS concurrent encryption operations"); @@ -478,7 +472,6 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿงช Testing KMS key validation and security properties"); diff --git a/crates/e2e_test/src/kms/kms_fault_recovery_test.rs b/crates/e2e_test/src/kms/kms_fault_recovery_test.rs index 2325281ee..fa6c08170 100644 --- a/crates/e2e_test/src/kms/kms_fault_recovery_test.rs +++ b/crates/e2e_test/src/kms/kms_fault_recovery_test.rs @@ -24,7 +24,6 @@ use super::common::LocalKMSTestEnvironment; use crate::common::{TEST_BUCKET, init_logging}; use aws_sdk_s3::types::ServerSideEncryption; -use serial_test::serial; use std::fs; use std::time::Duration; use tokio::time::sleep; @@ -32,7 +31,6 @@ use tracing::{info, warn}; /// Test KMS behavior when key directory is temporarily unavailable #[tokio::test] -#[serial] async fn test_kms_key_directory_unavailable() -> Result<(), Box> { init_logging(); info!("๐Ÿงช Testing KMS behavior with unavailable key directory"); @@ -123,7 +121,6 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿงช Testing KMS behavior with corrupted key files"); @@ -215,7 +212,6 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿงช Testing KMS multipart upload interruption and recovery"); @@ -399,7 +395,6 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿงช Testing KMS behavior under resource constraints"); diff --git a/crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs b/crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs index 173e87767..3058f4be9 100644 --- a/crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs +++ b/crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs @@ -51,7 +51,6 @@ use aws_sdk_s3::types::{ TransitionStorageClass, }; use serde::Deserialize; -use serial_test::serial; use std::time::{Duration as StdDuration, Instant}; use tracing::info; @@ -424,7 +423,6 @@ async fn wait_for_restore_complete(client: &Client, bucket: &str, key: &str, dea /// filter as the cause of the deletion and proves the encrypted bucket stays /// readable end to end after the scanner has run. #[tokio::test] -#[serial] async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult { init_logging(); @@ -485,7 +483,6 @@ async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult { /// (the mechanism `reliant/tiering.rs` established), so the test does not /// depend on scanner scheduling; the 1s scanner cycle stays on as a backstop. #[tokio::test] -#[serial] async fn ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/kms/kms_local_test.rs b/crates/e2e_test/src/kms/kms_local_test.rs index 82b44a154..ccda86989 100644 --- a/crates/e2e_test/src/kms/kms_local_test.rs +++ b/crates/e2e_test/src/kms/kms_local_test.rs @@ -24,11 +24,9 @@ use super::common::{ test_kms_key_management, test_sse_c_encryption, }; use crate::common::{TEST_BUCKET, init_logging}; -use serial_test::serial; use tracing::{error, info}; #[tokio::test] -#[serial] async fn test_local_kms_end_to_end() -> Result<(), Box> { init_logging(); if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") { @@ -114,7 +112,6 @@ async fn test_local_kms_end_to_end() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿงช Step 1: Test the basic single-file encryption function"); @@ -85,7 +83,6 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿงช Step 2: Test unencrypted shard uploads"); @@ -184,7 +181,6 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo /// Step 3: Test Shard Upload + SSE-S3 Encryption (Focus Test) #[tokio::test] -#[serial] async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box> { init_logging(); info!("๐Ÿงช Step 3: Test Shard Upload + SSE-S3 Encryption"); @@ -308,7 +304,6 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿงช Step 4: test large-file multipart encryption"); @@ -434,7 +429,6 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box Result<(), Box> { init_logging(); info!("๐Ÿงช Step 5: test multipart uploads for every encryption mode"); diff --git a/crates/e2e_test/src/kms/test_runner.rs b/crates/e2e_test/src/kms/test_runner.rs index d5e831b79..558c14631 100644 --- a/crates/e2e_test/src/kms/test_runner.rs +++ b/crates/e2e_test/src/kms/test_runner.rs @@ -19,7 +19,6 @@ //! filtering, and comprehensive reporting capabilities. use crate::common::init_logging; -use serial_test::serial; use std::time::Instant; use tokio::time::{Duration, sleep}; use tracing::{debug, error, info, warn}; @@ -458,7 +457,6 @@ impl KMSTestSuite { /// Quick test suite for critical tests only #[tokio::test] -#[serial] async fn test_kms_critical_suite() -> Result<(), Box> { let config = TestSuiteConfig { categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption], @@ -481,7 +479,6 @@ async fn test_kms_critical_suite() -> Result<(), Box Result<(), Box> { let suite = KMSTestSuite::new(); let results = suite.run_test_suite().await; diff --git a/crates/e2e_test/src/leading_slash_key_test.rs b/crates/e2e_test/src/leading_slash_key_test.rs index 4de4812df..841c4f060 100644 --- a/crates/e2e_test/src/leading_slash_key_test.rs +++ b/crates/e2e_test/src/leading_slash_key_test.rs @@ -24,7 +24,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -36,7 +35,6 @@ mod tests { /// PUT with a leading-slash key must succeed and the object must be /// readable under the normalized key (leading slash stripped). #[tokio::test] - #[serial] async fn test_put_object_with_leading_slash_key() -> Result<(), Box> { init_logging(); info!("Starting test: PUT object with leading slash in key (Issue #2427)"); @@ -94,7 +92,6 @@ mod tests { /// Duplicate and repeated slashes after a leading slash collapse MinIO-style. #[tokio::test] - #[serial] async fn test_put_object_with_duplicate_slashes_normalized() -> Result<(), Box> { init_logging(); info!("Starting test: duplicate slash normalization (Issue #2427)"); diff --git a/crates/e2e_test/src/lifecycle_regression_test.rs b/crates/e2e_test/src/lifecycle_regression_test.rs index e2e1c039e..3a3615dd4 100644 --- a/crates/e2e_test/src/lifecycle_regression_test.rs +++ b/crates/e2e_test/src/lifecycle_regression_test.rs @@ -36,7 +36,6 @@ mod tests { BucketLifecycleConfiguration, BucketVersioningStatus, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, NoncurrentVersionExpiration, VersioningConfiguration, }; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -80,7 +79,6 @@ mod tests { /// /// This tests the rule persistence path (rustfs#4963: 3 days โ†’ 0 days). #[tokio::test] - #[serial] async fn test_lifecycle_expiration_rule_persists_correctly() -> TestResult { init_logging(); info!("RT-03: lifecycle expiration rule persists correctly"); @@ -148,7 +146,6 @@ mod tests { /// Covers the pattern where noncurrent version expiration rules are /// accepted but old versions are never cleaned up. #[tokio::test] - #[serial] async fn test_lifecycle_noncurrent_version_expiration_rule_persists() -> TestResult { init_logging(); info!("RT-03b: noncurrent version expiration rule persists"); @@ -233,7 +230,6 @@ mod tests { /// after restart. Transition rules require a configured remote tier /// (tested in reliant/tiering.rs), so this test uses expiration only. #[tokio::test] - #[serial] async fn test_lifecycle_prefix_rule_persists() -> TestResult { init_logging(); info!("RT-04: lifecycle prefix rule persists"); @@ -294,7 +290,6 @@ mod tests { /// Regression pattern: DELETE on a versioned object fails or does not /// create a delete marker, or the delete marker is not visible in LIST. #[tokio::test] - #[serial] async fn test_delete_marker_creation_and_visibility() -> TestResult { init_logging(); info!("RT-05b: delete marker creation and visibility"); diff --git a/crates/e2e_test/src/list_buckets_double_slash_test.rs b/crates/e2e_test/src/list_buckets_double_slash_test.rs index ddbb68307..592b252b4 100644 --- a/crates/e2e_test/src/list_buckets_double_slash_test.rs +++ b/crates/e2e_test/src/list_buckets_double_slash_test.rs @@ -27,7 +27,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; /// Sends a SigV4-signed `GET` where the signature is computed over `sign_path` @@ -67,7 +66,6 @@ mod tests { /// `GET /` (path-style service call) returns `ListBuckets`. #[tokio::test] - #[serial] async fn test_list_buckets_single_slash() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -86,7 +84,6 @@ mod tests { /// compat layer rewrites `//` to `/` before `s3s` parses/verifies the request, /// so both routing and signature verification operate on `/`. #[tokio::test] - #[serial] async fn test_list_buckets_double_slash_browser_compat() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -105,7 +102,6 @@ mod tests { /// (`GET //bucket`) must be left untouched by the compat layer โ€” it is not a /// `ListBuckets` request and s3s continues to reject the empty bucket name. #[tokio::test] - #[serial] async fn test_double_slash_rewrite_is_narrowly_scoped() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/list_buckets_iam_filter_test.rs b/crates/e2e_test/src/list_buckets_iam_filter_test.rs index ca748be1a..ac15e0713 100644 --- a/crates/e2e_test/src/list_buckets_iam_filter_test.rs +++ b/crates/e2e_test/src/list_buckets_iam_filter_test.rs @@ -15,7 +15,6 @@ use crate::common::{RustFSTestEnvironment, admin_ok, build_test_s3_config, build_test_sts_client, init_logging}; use aws_sdk_s3::Client; use aws_sdk_s3::error::ProvideErrorMetadata; -use serial_test::serial; use tokio::time::{Duration, Instant}; fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client { @@ -76,7 +75,6 @@ async fn create_service_account( } #[tokio::test] -#[serial] async fn list_buckets_filters_with_iam_bucket_resources() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/list_object_versions_metadata_extension_test.rs b/crates/e2e_test/src/list_object_versions_metadata_extension_test.rs index 150c45090..1e5fd0977 100644 --- a/crates/e2e_test/src/list_object_versions_metadata_extension_test.rs +++ b/crates/e2e_test/src/list_object_versions_metadata_extension_test.rs @@ -24,7 +24,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -54,7 +53,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_list_object_versions_metadata_extension_returns_metadata_tags_and_internal() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/list_object_versions_regression_test.rs b/crates/e2e_test/src/list_object_versions_regression_test.rs index b7365a8de..0c29dc110 100644 --- a/crates/e2e_test/src/list_object_versions_regression_test.rs +++ b/crates/e2e_test/src/list_object_versions_regression_test.rs @@ -21,7 +21,6 @@ mod tests { use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; - use serial_test::serial; use tracing::info; fn create_s3_client(env: &RustFSTestEnvironment) -> Client { @@ -29,7 +28,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_list_object_versions_immediately_returns_latest_put_after_delete_marker() { init_logging(); info!("๐Ÿงช TEST: ListObjectVersions returns the newest version immediately after put -> delete -> put"); @@ -182,7 +180,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_list_object_versions_prefix_with_marker_object_returns_children() { init_logging(); info!("๐Ÿงช TEST: ListObjectVersions returns prefix children when a marker object also exists"); diff --git a/crates/e2e_test/src/list_objects_duplicates_test.rs b/crates/e2e_test/src/list_objects_duplicates_test.rs index 2d7a74351..9b8349dcc 100644 --- a/crates/e2e_test/src/list_objects_duplicates_test.rs +++ b/crates/e2e_test/src/list_objects_duplicates_test.rs @@ -17,7 +17,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use tracing::info; /// Helper function to create an S3 client for testing @@ -60,7 +59,6 @@ mod tests { /// The bug was that "folder/" (the object) and "folder/" (derived prefix) were both added to CommonPrefixes /// when delimiter was "/" because the deduplication check was explicitly skipped for "/" delimiter. #[tokio::test] - #[serial] async fn test_list_objects_v2_unique_common_prefixes() { init_logging(); info!("Starting test: ListObjectsV2 should return unique CommonPrefixes"); @@ -140,7 +138,6 @@ mod tests { /// When both "marker/subdir/" and "marker/subdir/file.txt" exist, listing with /// Prefix="marker/" must not duplicate "marker/subdir/file.txt" in Contents. #[tokio::test] - #[serial] async fn test_list_objects_v2_unique_contents_with_explicit_directory_markers() { init_logging(); info!("Starting test: ListObjectsV2 should return unique keys with explicit directory markers"); @@ -208,7 +205,6 @@ mod tests { /// and never produce the prefix entry `a/`. Delimiter="/" listings then /// returned Contents `a` but silently dropped CommonPrefix `a/`. #[tokio::test] - #[serial] async fn test_list_objects_v2_object_and_same_named_prefix_coexist() { init_logging(); info!("Starting test: ListObjectsV2 should return both object `a` and CommonPrefix `a/`"); diff --git a/crates/e2e_test/src/list_objects_v2_metadata_extension_test.rs b/crates/e2e_test/src/list_objects_v2_metadata_extension_test.rs index 72be61b6c..2cad0cc68 100644 --- a/crates/e2e_test/src/list_objects_v2_metadata_extension_test.rs +++ b/crates/e2e_test/src/list_objects_v2_metadata_extension_test.rs @@ -23,7 +23,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -53,7 +52,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_list_objects_v2_metadata_extension_returns_metadata_tags_and_internal() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/list_objects_v2_pagination_test.rs b/crates/e2e_test/src/list_objects_v2_pagination_test.rs index c970713d8..70b00467b 100644 --- a/crates/e2e_test/src/list_objects_v2_pagination_test.rs +++ b/crates/e2e_test/src/list_objects_v2_pagination_test.rs @@ -30,7 +30,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use std::collections::HashSet; use tracing::info; @@ -61,7 +60,6 @@ mod tests { /// Test for Issue #2775: continuation forwarding must not /// skip a child directory when the prefix component repeats in the key. #[tokio::test] - #[serial] async fn test_list_objects_v2_repeated_prefix_continuation() { init_logging(); info!("Starting test: ListObjectsV2 repeated-prefix continuation"); @@ -187,7 +185,6 @@ mod tests { /// This is the core bug from issue #1596: the server was returning /// IsTruncated=true even when all objects fit within the requested max_keys. #[tokio::test] - #[serial] async fn test_list_objects_v2_not_truncated_when_all_objects_returned() { init_logging(); info!("Starting test: ListObjectsV2 should not be truncated when all objects fit within max_keys"); @@ -252,7 +249,6 @@ mod tests { /// 2. NextContinuationToken is returned (not NextMarker) /// 3. Using ContinuationToken fetches the remaining objects #[tokio::test] - #[serial] async fn test_list_objects_v2_pagination_with_continuation_token() { init_logging(); info!("Starting test: ListObjectsV2 pagination with continuation token"); @@ -394,7 +390,6 @@ mod tests { /// Edge case: when max_keys exactly equals the number of objects, /// IsTruncated should be false. #[tokio::test] - #[serial] async fn test_list_objects_v2_max_keys_equals_object_count() { init_logging(); info!("Starting test: ListObjectsV2 with max_keys equal to object count"); @@ -455,7 +450,6 @@ mod tests { /// /// Edge case: IsTruncated should be false for empty bucket. #[tokio::test] - #[serial] async fn test_list_objects_v2_empty_bucket() { init_logging(); info!("Starting test: ListObjectsV2 with empty bucket"); @@ -495,7 +489,6 @@ mod tests { /// Test ListObjectsV2 caps max_keys above the service limit and still paginates. #[tokio::test] - #[serial] async fn test_list_objects_v2_max_keys_above_limit_returns_token() { init_logging(); info!("Starting test: ListObjectsV2 with max_keys above limit"); @@ -563,7 +556,6 @@ mod tests { /// S3 semantics: when max_keys is 0, the response should include no objects /// and IsTruncated should be false. #[tokio::test] - #[serial] async fn test_list_objects_v2_max_keys_zero() { init_logging(); info!("Starting test: ListObjectsV2 with max_keys=0"); @@ -620,7 +612,6 @@ mod tests { /// With max_keys=1000, all 5 visible results (3 prefixes + 2 objects) fit in one /// page, so IsTruncated must be false even though raw entry count is much larger. #[tokio::test] - #[serial] async fn test_list_objects_v2_delimiter_collapsed_prefix_no_false_truncation() { init_logging(); info!("Starting test: ListObjectsV2 delimiter collapsed-prefix no false truncation"); @@ -744,7 +735,6 @@ mod tests { /// Each page returns up to 50 CommonPrefixes. The server must correctly set /// IsTruncated and provide a valid continuation token across all pages. #[tokio::test] - #[serial] async fn test_list_objects_v2_delimiter_small_page_traverses_all() { init_logging(); info!("Starting test: ListObjectsV2 delimiter small page traverses all keys"); @@ -867,7 +857,6 @@ mod tests { /// but after delimiter collapse only 10 CommonPrefixes are visible (10 < 1000). /// IsTruncated must be false since there are no additional visible results. #[tokio::test] - #[serial] async fn test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below() { init_logging(); info!("Starting test: ListObjectsV2 raw > MaxKeys but visible < MaxKeys after collapse"); @@ -970,7 +959,6 @@ mod tests { /// This complements test_list_objects_v2_max_keys_above_limit_returns_token which /// tests the non-delimiter case. #[tokio::test] - #[serial] async fn test_list_objects_v2_maxkeys_above_limit_with_delimiter() { init_logging(); info!("Starting test: ListObjectsV2 MaxKeys above limit with delimiter"); @@ -1041,7 +1029,6 @@ mod tests { /// the next page: with keys `a`, `a.txt`, `zz` and max_keys=1, page 2 /// returned `zz` and `a.txt` was never listed. #[tokio::test] - #[serial] async fn test_list_objects_v2_continuation_keeps_keys_after_marker_stem() { init_logging(); info!("Starting test: continuation must not skip keys sorting below the cursor tag"); diff --git a/crates/e2e_test/src/listing_regression_test.rs b/crates/e2e_test/src/listing_regression_test.rs index 914b3aab3..ee2569f29 100644 --- a/crates/e2e_test/src/listing_regression_test.rs +++ b/crates/e2e_test/src/listing_regression_test.rs @@ -31,7 +31,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use std::collections::HashSet; use std::error::Error; use tracing::info; @@ -49,7 +48,6 @@ mod tests { /// 3. Verify all 100 keys are returned exactly once /// 4. Verify no duplicates or skipped keys #[tokio::test] - #[serial] async fn test_list_objects_v2_completeness_100_objects() -> TestResult { init_logging(); info!("RT-06: listing completeness with 100 objects"); @@ -133,7 +131,6 @@ mod tests { /// Regression pattern: prefix filter returns empty or includes wrong keys /// (rustfs#5051: empty results for shallow prefixes). #[tokio::test] - #[serial] async fn test_list_objects_v2_prefix_filter_correctness() -> TestResult { init_logging(); info!("RT-06b: prefix filter correctness"); @@ -233,7 +230,6 @@ mod tests { /// Regression pattern: delimiter handling produces incorrect CommonPrefixes /// or misses objects at the delimiter boundary. #[tokio::test] - #[serial] async fn test_list_objects_v2_delimiter_common_prefixes() -> TestResult { init_logging(); info!("RT-06c: delimiter and CommonPrefixes"); @@ -290,7 +286,6 @@ mod tests { /// Regression pattern: IsTruncated=false when there are more objects /// (rustfs#4810: walk_dir timeout truncation with false IsTruncated). #[tokio::test] - #[serial] async fn test_list_objects_v2_is_truncated_correctness() -> TestResult { init_logging(); info!("RT-06d: IsTruncated correctness"); diff --git a/crates/e2e_test/src/mc_mirror_small_bucket_test.rs b/crates/e2e_test/src/mc_mirror_small_bucket_test.rs index 6654a8663..2a9c1a507 100644 --- a/crates/e2e_test/src/mc_mirror_small_bucket_test.rs +++ b/crates/e2e_test/src/mc_mirror_small_bucket_test.rs @@ -13,7 +13,6 @@ // limitations under the License. use crate::common::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, RustFSTestEnvironment}; -use serial_test::serial; use std::path::Path; use std::process::Command; use std::time::Duration; @@ -73,7 +72,6 @@ fn count_files(root: &Path) -> usize { } #[tokio::test] -#[serial] async fn test_mc_mirror_small_bucket_completes_without_list_timeout() -> TestResult { crate::common::init_logging(); info!("Starting issue #3107 mc mirror regression test"); diff --git a/crates/e2e_test/src/namespace_lock_quorum_test.rs b/crates/e2e_test/src/namespace_lock_quorum_test.rs index 9b9e1d53f..0af609ecc 100644 --- a/crates/e2e_test/src/namespace_lock_quorum_test.rs +++ b/crates/e2e_test/src/namespace_lock_quorum_test.rs @@ -16,7 +16,6 @@ use crate::common::RustFSTestClusterEnvironment; use aws_sdk_s3::Client; use aws_sdk_s3::error::SdkError; use bytes::Bytes; -use serial_test::serial; use std::sync::Arc; use tokio::sync::Barrier; use tracing::{info, warn}; @@ -51,7 +50,6 @@ fn format_s3_error(err: SdkError TestResult { crate::common::init_logging(); info!("Starting namespace lock quorum regression test with auto cluster"); @@ -128,7 +126,6 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum() /// `StorageError::other(...)` โ†’ `StorageError::Io(...)`, which fell through to /// `S3ErrorCode::InternalError` (500) in the error mapping. #[tokio::test] -#[serial] async fn test_concurrent_put_same_key_never_returns_500() -> TestResult { crate::common::init_logging(); info!("Starting concurrent PUT 500 regression test"); diff --git a/crates/e2e_test/src/negative_sigv4_test.rs b/crates/e2e_test/src/negative_sigv4_test.rs index f0539e0a0..a6a440383 100644 --- a/crates/e2e_test/src/negative_sigv4_test.rs +++ b/crates/e2e_test/src/negative_sigv4_test.rs @@ -37,7 +37,6 @@ use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; use aws_sdk_s3::primitives::ByteStream; use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::request_signature_v4::{SIGN_V4_ALGORITHM, get_scope, get_signature, get_signing_key}; -use serial_test::serial; use std::fmt::Write as _; use time::macros::format_description; use time::{Duration, OffsetDateTime}; @@ -183,7 +182,6 @@ async fn setup(env: &mut RustFSTestEnvironment) -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -214,7 +212,6 @@ async fn valid_header_sigv4_request_succeeds() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -254,7 +251,6 @@ async fn tampered_signature_returns_signature_does_not_match() -> Result<(), Box /// (b) A valid AccessKeyId paired with the wrong secret key must be rejected /// with SignatureDoesNotMatch / 403. #[tokio::test] -#[serial] async fn wrong_secret_key_returns_signature_does_not_match() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -279,7 +275,6 @@ async fn wrong_secret_key_returns_signature_does_not_match() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -320,7 +315,6 @@ async fn tampered_payload_is_rejected() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -344,7 +338,6 @@ async fn skewed_date_returns_request_time_too_skewed() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/notification_startup_regression_test.rs b/crates/e2e_test/src/notification_startup_regression_test.rs index f554d3cfd..0d0feed98 100644 --- a/crates/e2e_test/src/notification_startup_regression_test.rs +++ b/crates/e2e_test/src/notification_startup_regression_test.rs @@ -32,7 +32,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -47,7 +46,6 @@ mod tests { /// starts successfully with notification enabled and can serve S3 requests. /// A full webhook delivery test is in notification_webhook_test.rs. #[tokio::test] - #[serial] async fn test_notification_enabled_server_starts_cleanly() -> TestResult { init_logging(); info!("RT-01: notification enabled server starts cleanly"); @@ -92,7 +90,6 @@ mod tests { /// 3. Restart server /// 4. Verify notification config still exists #[tokio::test] - #[serial] async fn test_notification_config_survives_restart() -> TestResult { init_logging(); info!("RT-02: notification config survives restart"); diff --git a/crates/e2e_test/src/notification_webhook_test.rs b/crates/e2e_test/src/notification_webhook_test.rs index e63cea554..1eec17343 100644 --- a/crates/e2e_test/src/notification_webhook_test.rs +++ b/crates/e2e_test/src/notification_webhook_test.rs @@ -47,7 +47,6 @@ use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS; use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER}; use s3s::Body; use serde_json::Value; -use serial_test::serial; use std::error::Error; use std::io::Cursor; use std::path::Path; @@ -625,7 +624,6 @@ fn assert_generated_request_id_correlation(record: &Value, request_id: &str) { /// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must become /// online and receive a real S3 event POST. #[tokio::test] -#[serial] async fn test_https_webhook_target_delivers_event_with_notify_env_enabled() -> TestResult { init_logging(); @@ -680,7 +678,6 @@ async fn test_https_webhook_target_delivers_event_with_notify_env_enabled() -> T /// PUT / multipart-complete / DELETE each deliver one event with correct fields, /// and the prefix/suffix filter drops non-matching keys. #[tokio::test] -#[serial] async fn test_webhook_event_delivery_and_filtering() -> TestResult { init_logging(); @@ -900,7 +897,6 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult { /// An event queued while the target endpoint rejects delivery survives on the /// durable store and is redelivered once the endpoint comes back. #[tokio::test] -#[serial] async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/object_lambda_test.rs b/crates/e2e_test/src/object_lambda_test.rs index 69d2e46ef..66f259e6d 100644 --- a/crates/e2e_test/src/object_lambda_test.rs +++ b/crates/e2e_test/src/object_lambda_test.rs @@ -20,7 +20,6 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::{pre_sign_v4, sign_v4}; use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS; use s3s::Body; -use serial_test::serial; use std::collections::HashMap; use std::error::Error; use time::OffsetDateTime; @@ -548,7 +547,6 @@ async fn read_listen_notification_event( } #[tokio::test] -#[serial] async fn test_notification_target_persists_across_restart_and_delete() -> Result<(), Box> { init_logging(); @@ -608,7 +606,6 @@ async fn test_notification_target_persists_across_restart_and_delete() -> Result } #[tokio::test] -#[serial] async fn test_notification_target_with_path_is_online_via_transport_probe() -> Result<(), Box> { init_logging(); @@ -641,7 +638,6 @@ async fn test_notification_target_with_path_is_online_via_transport_probe() -> R } #[tokio::test] -#[serial] async fn test_get_object_lambda_accepts_presigned_requests() -> Result<(), Box> { init_logging(); @@ -682,7 +678,6 @@ async fn test_get_object_lambda_accepts_presigned_requests() -> Result<(), Box Result<(), Box> { init_logging(); @@ -722,7 +717,6 @@ async fn test_get_object_lambda_accepts_named_webhook_target_arn() -> Result<(), } #[tokio::test] -#[serial] async fn test_get_object_lambda_invokes_runtime_webhook_target() -> Result<(), Box> { init_logging(); @@ -790,7 +784,6 @@ async fn test_get_object_lambda_invokes_runtime_webhook_target() -> Result<(), B } #[tokio::test] -#[serial] async fn test_get_object_lambda_passthroughs_non_success_webhook_response() -> Result<(), Box> { init_logging(); @@ -850,7 +843,6 @@ async fn test_get_object_lambda_passthroughs_non_success_webhook_response() -> R } #[tokio::test] -#[serial] async fn test_get_object_lambda_rejects_success_response_without_auth_headers() -> Result<(), Box> { init_logging(); @@ -896,7 +888,6 @@ async fn test_get_object_lambda_rejects_success_response_without_auth_headers() } #[tokio::test] -#[serial] async fn test_get_object_lambda_rejects_success_response_with_mismatched_auth_headers() -> Result<(), Box> { init_logging(); @@ -943,7 +934,6 @@ async fn test_get_object_lambda_rejects_success_response_with_mismatched_auth_he } #[tokio::test] -#[serial] async fn test_get_object_lambda_rejects_unsupported_target_type() -> Result<(), Box> { init_logging(); @@ -980,7 +970,6 @@ async fn test_get_object_lambda_rejects_unsupported_target_type() -> Result<(), } #[tokio::test] -#[serial] async fn test_get_object_lambda_rejects_unconfigured_target() -> Result<(), Box> { init_logging(); @@ -1017,7 +1006,6 @@ async fn test_get_object_lambda_rejects_unconfigured_target() -> Result<(), Box< } #[tokio::test] -#[serial] async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box> { init_logging(); @@ -1063,7 +1051,6 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box Result<(), Box> { init_logging(); @@ -1106,7 +1093,6 @@ async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Resul } #[tokio::test] -#[serial] async fn test_configure_object_lambda_notify_webhook_rejects_response_header_timeout_key() -> Result<(), Box> { init_logging(); @@ -1140,7 +1126,6 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim } #[tokio::test] -#[serial] async fn test_listen_notification_emits_after_put_object() -> Result<(), Box> { init_logging(); @@ -1184,7 +1169,6 @@ async fn test_listen_notification_emits_after_put_object() -> Result<(), Box Result<(), Box> { init_logging(); @@ -1219,7 +1203,6 @@ async fn test_listen_notification_emits_on_empty_bucket_when_notify_disabled() - } #[tokio::test] -#[serial] async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/overwrite_cleanup_regression_test.rs b/crates/e2e_test/src/overwrite_cleanup_regression_test.rs index f41113b12..7be7c26ea 100644 --- a/crates/e2e_test/src/overwrite_cleanup_regression_test.rs +++ b/crates/e2e_test/src/overwrite_cleanup_regression_test.rs @@ -13,7 +13,6 @@ // limitations under the License. use aws_sdk_s3::primitives::ByteStream; -use serial_test::serial; use std::path::{Path, PathBuf}; use uuid::Uuid; @@ -24,7 +23,6 @@ const TEST_OBJECT: &str = "large-object.bin"; const PAYLOAD_SIZE: usize = 512 * 1024; #[tokio::test(flavor = "multi_thread")] -#[serial] async fn unversioned_overwrite_removes_previous_physical_data_dir() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/presigned_negative_test.rs b/crates/e2e_test/src/presigned_negative_test.rs index 767cc7ca5..31df39fd0 100644 --- a/crates/e2e_test/src/presigned_negative_test.rs +++ b/crates/e2e_test/src/presigned_negative_test.rs @@ -43,7 +43,6 @@ use aws_sdk_s3::presigning::{PresignedRequest, PresigningConfig}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::{Client, Config}; use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; -use serial_test::serial; use std::time::{Duration, SystemTime}; use tracing::info; @@ -157,7 +156,6 @@ async fn setup(env: &mut RustFSTestEnvironment) -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -182,7 +180,6 @@ async fn valid_presigned_get_succeeds() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -211,7 +208,6 @@ async fn valid_presigned_put_succeeds() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -236,7 +232,6 @@ async fn expired_presigned_get_is_rejected() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -262,7 +257,6 @@ async fn tampered_signature_returns_signature_does_not_match() -> Result<(), Box /// (c) A presigned URL generated with the WRONG secret (but the real access key /// id) must be rejected with 403 / SignatureDoesNotMatch. #[tokio::test] -#[serial] async fn wrong_secret_key_returns_signature_does_not_match() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -290,7 +284,6 @@ async fn wrong_secret_key_returns_signature_does_not_match() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -325,7 +318,6 @@ async fn tampered_target_key_returns_signature_does_not_match() -> Result<(), Bo /// (e / acceptance 4 negative half) Tampering the signature of a presigned PUT /// must be rejected with 403 / SignatureDoesNotMatch โ€” the write must not land. #[tokio::test] -#[serial] async fn tampered_presigned_put_returns_signature_does_not_match() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/protocols/test_runner.rs b/crates/e2e_test/src/protocols/test_runner.rs index 0a578804d..7150ab2cd 100644 --- a/crates/e2e_test/src/protocols/test_runner.rs +++ b/crates/e2e_test/src/protocols/test_runner.rs @@ -22,7 +22,6 @@ use crate::protocols::sftp_compliance::{ }; use crate::protocols::sftp_core::{test_sftp_core_operations, test_sftp_idle_timeout_disconnects}; use crate::protocols::webdav_core::test_webdav_core_operations; -use serial_test::serial; use std::time::Instant; use tokio::time::{Duration, sleep}; use tracing::{error, info}; @@ -229,7 +228,6 @@ fn all_protocol_tests() -> Vec { /// Test suite #[tokio::test] -#[serial] async fn test_protocol_core_suite() -> Result<(), Box> { let suite = ProtocolTestSuite::new(); let results = suite.run_test_suite().await; diff --git a/crates/e2e_test/src/quota_test.rs b/crates/e2e_test/src/quota_test.rs index 87eee5f37..e7c095674 100644 --- a/crates/e2e_test/src/quota_test.rs +++ b/crates/e2e_test/src/quota_test.rs @@ -15,7 +15,6 @@ use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_post, awscurl_put, init_logging}; use aws_sdk_s3::Client; use http::{Method, StatusCode}; -use serial_test::serial; use tokio::time::{Duration, sleep, timeout}; use tracing::{debug, info}; @@ -255,7 +254,6 @@ mod integration_tests { use aws_sdk_s3::error::ProvideErrorMetadata; #[tokio::test] - #[serial] async fn test_quota_basic_operations() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -300,7 +298,6 @@ mod integration_tests { /// with 400 UnexpectedContent, and an over-quota aws-chunked PUT must still get the quota /// rejection. #[tokio::test] - #[serial] async fn test_quota_admission_aws_chunked_declared_encoding() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -352,7 +349,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_update_and_clear() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -388,7 +384,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_delete_operations() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -425,7 +420,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_usage_tracking() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -464,7 +458,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_statistics() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -498,7 +491,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_check_api() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -539,7 +531,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_multiple_buckets() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -580,7 +571,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_error_handling() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -616,7 +606,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_http_endpoints() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -682,7 +671,6 @@ mod integration_tests { /// Test that a normal user with `readwrite` policy can read quota but cannot set/clear quota. #[tokio::test] - #[serial] async fn test_quota_normal_user_permissions() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -738,7 +726,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_copy_operations() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -784,7 +771,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_batch_delete() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -843,7 +829,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_multipart_upload() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { diff --git a/crates/e2e_test/src/reliant/conditional_writes.rs b/crates/e2e_test/src/reliant/conditional_writes.rs index 843ecc4e0..166ce66f5 100644 --- a/crates/e2e_test/src/reliant/conditional_writes.rs +++ b/crates/e2e_test/src/reliant/conditional_writes.rs @@ -6,7 +6,6 @@ use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; use bytes::Bytes; -use serial_test::serial; use std::error::Error; const ENDPOINT: &str = "http://localhost:9000"; @@ -89,7 +88,6 @@ fn generate_test_key(prefix: &str) -> String { } #[tokio::test] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_conditional_put_okay() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -132,7 +130,6 @@ async fn test_conditional_put_okay() -> Result<(), Box> { } #[tokio::test] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_conditional_put_failed() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -195,7 +192,6 @@ async fn test_conditional_put_failed() -> Result<(), Box> } #[tokio::test] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_conditional_put_when_object_does_not_exist() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -240,7 +236,6 @@ async fn test_conditional_put_when_object_does_not_exist() -> Result<(), Box Result<(), Box> { let client = create_aws_s3_client().await?; diff --git a/crates/e2e_test/src/reliant/get_deleted_object_test.rs b/crates/e2e_test/src/reliant/get_deleted_object_test.rs index b34159ec1..2b87a11c6 100644 --- a/crates/e2e_test/src/reliant/get_deleted_object_test.rs +++ b/crates/e2e_test/src/reliant/get_deleted_object_test.rs @@ -24,7 +24,6 @@ use aws_sdk_s3::Client; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::error::SdkError; use bytes::Bytes; -use serial_test::serial; use std::error::Error; use tracing::info; @@ -70,7 +69,6 @@ async fn setup_test_bucket(client: &Client) -> Result<(), Box> { } #[tokio::test] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_get_deleted_object_returns_nosuchkey() -> Result<(), Box> { // Initialize logging @@ -144,7 +142,6 @@ async fn test_get_deleted_object_returns_nosuchkey() -> Result<(), Box Result<(), Box> { let _ = tracing_subscriber::fmt() @@ -196,7 +193,6 @@ async fn test_head_deleted_object_returns_nosuchkey() -> Result<(), Box Result<(), Box> { let _ = tracing_subscriber::fmt() @@ -233,7 +229,6 @@ async fn test_get_nonexistent_object_returns_nosuchkey() -> Result<(), Box Result<(), Box> { let _ = tracing_subscriber::fmt() diff --git a/crates/e2e_test/src/reliant/head_deleted_object_versioning_test.rs b/crates/e2e_test/src/reliant/head_deleted_object_versioning_test.rs index a4d471754..b89579bde 100644 --- a/crates/e2e_test/src/reliant/head_deleted_object_versioning_test.rs +++ b/crates/e2e_test/src/reliant/head_deleted_object_versioning_test.rs @@ -25,7 +25,6 @@ use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; use bytes::Bytes; -use serial_test::serial; use std::error::Error; use tracing::info; @@ -85,7 +84,6 @@ async fn setup_test_bucket(client: &Client) -> Result<(), Box> { /// Test that HeadObject on a deleted object returns NoSuchKey when versioning is enabled #[tokio::test] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_head_deleted_object_versioning_returns_nosuchkey() -> Result<(), Box> { let _ = tracing_subscriber::fmt() diff --git a/crates/e2e_test/src/reliant/head_tls_bodyless_test.rs b/crates/e2e_test/src/reliant/head_tls_bodyless_test.rs index 6ef214a17..a535e431e 100644 --- a/crates/e2e_test/src/reliant/head_tls_bodyless_test.rs +++ b/crates/e2e_test/src/reliant/head_tls_bodyless_test.rs @@ -30,7 +30,6 @@ use reqwest::{Certificate, Client, Response, StatusCode}; use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; -use serial_test::serial; use std::error::Error; use std::path::Path; use std::process::Command; @@ -157,7 +156,6 @@ async fn start_tls_rustfs_server(env: &mut RustFSTestEnvironment, tls_dir: &Path } #[tokio::test] -#[serial] async fn test_head_missing_object_over_tls_http2_is_bodyless() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/reliant/lifecycle.rs b/crates/e2e_test/src/reliant/lifecycle.rs index 3c475bf88..9de87dff4 100644 --- a/crates/e2e_test/src/reliant/lifecycle.rs +++ b/crates/e2e_test/src/reliant/lifecycle.rs @@ -168,6 +168,24 @@ async fn wait_for_version_expired( } } +async fn wait_for_key_versions_empty(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult { + let start = std::time::Instant::now(); + loop { + let listing = client.list_object_versions().bucket(bucket).prefix(key).send().await?; + if listing.versions().is_empty() && listing.delete_markers().is_empty() { + return Ok(()); + } + if start.elapsed() >= deadline { + return Err(format!( + "object {bucket}/{key} still had versions or delete markers after {}s: {listing:?}", + deadline.as_secs() + ) + .into()); + } + tokio::time::sleep(StdDuration::from_millis(500)).await; + } +} + /// Build a prefix-scoped `Days`-based expiration rule. fn expiration_rule(id: &str, prefix: &str, days: i32) -> Result> { let rule = LifecycleRule::builder() @@ -193,6 +211,21 @@ fn noncurrent_expiration_rule( Ok(rule) } +fn noncurrent_expiration_with_delete_marker_cleanup_rule( + id: &str, + prefix: &str, + days: i32, +) -> Result> { + let rule = LifecycleRule::builder() + .id(id) + .filter(LifecycleRuleFilter::builder().prefix(prefix).build()) + .expiration(LifecycleExpiration::builder().expired_object_delete_marker(true).build()) + .noncurrent_version_expiration(NoncurrentVersionExpiration::builder().noncurrent_days(days).build()) + .status(ExpirationStatus::Enabled) + .build()?; + Ok(rule) +} + async fn put_expiration_config(client: &Client, bucket: &str, rule: LifecycleRule) -> TestResult { let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?; client @@ -412,6 +445,143 @@ async fn test_lifecycle_noncurrent_version_expiry_removes_only_old_version() -> Ok(()) } +/// A combined `NoncurrentDays=1` and `ExpiredObjectDeleteMarker=true` rule +/// must remove a noncurrent data version and then its sole latest delete +/// marker, without expiring current-only objects. A second prefix with only +/// noncurrent expiry proves that marker cleanup comes from EODM. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_lifecycle_noncurrent_expiry_then_cleans_expired_delete_marker() -> TestResult { + let mut env = RustFSTestEnvironment::new().await?; + let mut extra_env = fast_lifecycle_env(); + extra_env.push(("RUSTFS_ILM_DEBUG_DAY_SECS", "2")); + env.start_rustfs_server_with_env(vec![], &extra_env).await?; + + let client = env.create_s3_client(); + let bucket = "ilm-expired-delete-marker"; + client.create_bucket().bucket(bucket).send().await?; + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration( + VersioningConfiguration::builder() + .status(BucketVersioningStatus::Enabled) + .build(), + ) + .send() + .await?; + + let cascade_key = "cascade/deleted.txt"; + let cascade_put = client + .put_object() + .bucket(bucket) + .key(cascade_key) + .body(ByteStream::from_static(b"cascade payload")) + .send() + .await?; + let cascade_data_version = cascade_put + .version_id() + .map(str::to_string) + .expect("cascade PUT returns a version id"); + let cascade_delete = client.delete_object().bucket(bucket).key(cascade_key).send().await?; + let cascade_marker_version = cascade_delete + .version_id() + .map(str::to_string) + .expect("cascade DELETE returns a marker version id"); + assert_eq!(cascade_delete.delete_marker(), Some(true)); + + let survivor_key = "cascade/current-only.txt"; + client + .put_object() + .bucket(bucket) + .key(survivor_key) + .body(ByteStream::from_static(b"current payload")) + .send() + .await?; + let survivor_before = client.get_object().bucket(bucket).key(survivor_key).send().await?; + assert_eq!(survivor_before.body.collect().await?.into_bytes().as_ref(), b"current payload"); + + let control_key = "nve-only/deleted.txt"; + let control_put = client + .put_object() + .bucket(bucket) + .key(control_key) + .body(ByteStream::from_static(b"control payload")) + .send() + .await?; + let control_data_version = control_put + .version_id() + .map(str::to_string) + .expect("control PUT returns a version id"); + let control_delete = client.delete_object().bucket(bucket).key(control_key).send().await?; + let control_marker_version = control_delete + .version_id() + .map(str::to_string) + .expect("control DELETE returns a marker version id"); + assert_eq!(control_delete.delete_marker(), Some(true)); + + let cascade_before = client + .list_object_versions() + .bucket(bucket) + .prefix(cascade_key) + .send() + .await?; + assert!( + cascade_before + .versions() + .iter() + .any(|version| version.version_id() == Some(cascade_data_version.as_str())), + "cascade data version must exist before lifecycle is installed: {cascade_before:?}" + ); + assert!( + cascade_before + .delete_markers() + .iter() + .any(|marker| { marker.version_id() == Some(cascade_marker_version.as_str()) && marker.is_latest() == Some(true) }), + "cascade latest delete marker must exist before lifecycle is installed: {cascade_before:?}" + ); + + let lifecycle = BucketLifecycleConfiguration::builder() + .rules(noncurrent_expiration_with_delete_marker_cleanup_rule( + "expire-and-clean-marker", + "cascade/", + 1, + )?) + .rules(noncurrent_expiration_rule("expire-only", "nve-only/", 1)?) + .build()?; + client + .put_bucket_lifecycle_configuration() + .bucket(bucket) + .lifecycle_configuration(lifecycle) + .send() + .await?; + + wait_for_key_versions_empty(&client, bucket, cascade_key, StdDuration::from_secs(90)).await?; + wait_for_version_expired(&client, bucket, control_key, &control_data_version, StdDuration::from_secs(90)).await?; + + let survivor = client.get_object().bucket(bucket).key(survivor_key).send().await?; + assert_eq!(survivor.body.collect().await?.into_bytes().as_ref(), b"current payload"); + + let control_after = client + .list_object_versions() + .bucket(bucket) + .prefix(control_key) + .send() + .await?; + assert!( + control_after.versions().is_empty(), + "NVE-only control must remove its data version: {control_after:?}" + ); + assert!( + control_after + .delete_markers() + .iter() + .any(|marker| { marker.version_id() == Some(control_marker_version.as_str()) && marker.is_latest() == Some(true) }), + "NVE-only control must preserve its latest delete marker: {control_after:?}" + ); + + Ok(()) +} + /// `Days=0` expiration is invalid per S3 semantics (`Days` must be a positive /// integer >= 1). A `PutBucketLifecycleConfiguration` carrying a zero-day rule /// must be rejected with `InvalidArgument` (HTTP 400) - see crates/lifecycle diff --git a/crates/e2e_test/src/reliant/sql.rs b/crates/e2e_test/src/reliant/sql.rs index 35ef8bcae..6f63ddab0 100644 --- a/crates/e2e_test/src/reliant/sql.rs +++ b/crates/e2e_test/src/reliant/sql.rs @@ -20,7 +20,6 @@ use aws_sdk_s3::types::{ CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType, OutputSerialization, }; use bytes::Bytes; -use serial_test::serial; use std::error::Error; const ENDPOINT: &str = "http://localhost:9000"; @@ -118,7 +117,6 @@ async fn process_select_response( } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_select_object_content_csv_basic() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -160,7 +158,6 @@ async fn test_select_object_content_csv_basic() -> Result<(), Box> { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_select_object_content_csv_aggregation() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -206,7 +203,6 @@ async fn test_select_object_content_csv_aggregation() -> Result<(), Box Result<(), Box> { let client = create_aws_s3_client().await?; @@ -248,7 +244,6 @@ async fn test_select_object_content_json_basic() -> Result<(), Box> { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_select_object_content_csv_limit() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -291,7 +286,6 @@ async fn test_select_object_content_csv_limit() -> Result<(), Box> { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_select_object_content_csv_order_by() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -337,7 +331,6 @@ async fn test_select_object_content_csv_order_by() -> Result<(), Box> } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_select_object_content_error_handling() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -373,7 +366,6 @@ async fn test_select_object_content_error_handling() -> Result<(), Box Result<(), Box> { let client = create_aws_s3_client().await?; diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index b9645deba..d0282c83a 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -60,7 +60,6 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; use s3s::header::X_AMZ_REPLICATION_STATUS; -use serial_test::serial; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::convert::Infallible; @@ -4630,7 +4629,6 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult { /// fails without any PUT reaching the target (capability cache, proven from /// the target journal); plaintext objects still replicate COMPLETED. #[tokio::test] -#[serial] async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_headers() -> TestResult { init_logging(); @@ -4740,7 +4738,6 @@ async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_header /// a capability limit, and a plaintext-only deployment against a MinIO-like /// target must not turn red. #[tokio::test] -#[serial] async fn test_replication_check_flags_ssec_passthrough_dropping_target() -> TestResult { init_logging(); @@ -4831,7 +4828,6 @@ async fn test_replication_check_flags_ssec_passthrough_dropping_target() -> Test /// HEAD-back audit against the recovered RustFS target โ€” and the replica must /// be readable with the customer key. #[tokio::test] -#[serial] async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult { init_logging(); @@ -4924,7 +4920,6 @@ async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult /// the existing-object resync (`replicate_all` transport, N2-audited), land as /// a REPLICA, and read back with the customer key. #[tokio::test] -#[serial] async fn test_bucket_replication_sse_c_existing_object_resync() -> TestResult { init_logging(); @@ -8819,7 +8814,6 @@ async fn start_read_proxy_lab( /// headers are forwarded verbatim, and an inbound request that was itself /// proxied is answered locally (404) without touching the target. #[tokio::test] -#[serial] async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() -> TestResult { init_logging(); @@ -9002,7 +8996,6 @@ async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() -> /// proxied to the replication target with the anti-loop marker, mirroring /// MinIO `proxyGetTaggingToRepTarget`. #[tokio::test] -#[serial] async fn test_get_object_tagging_proxies_unreplicated_object_to_replication_target() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/security_boundary_test.rs b/crates/e2e_test/src/security_boundary_test.rs index 1f5fdda83..0a62ef0f5 100644 --- a/crates/e2e_test/src/security_boundary_test.rs +++ b/crates/e2e_test/src/security_boundary_test.rs @@ -25,7 +25,6 @@ use crate::common::{RustFSTestEnvironment, awscurl_available, awscurl_put, init_ use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Tag, Tagging}; -use serial_test::serial; use std::error::Error; use tracing::info; @@ -36,7 +35,6 @@ use tracing::info; /// far beyond that limit and assert the server rejects it with the specific /// error, rather than accepting an arbitrarily large control-plane body. #[tokio::test] -#[serial] async fn test_large_xml_body_rejection() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -92,7 +90,6 @@ async fn test_large_xml_body_rejection() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -149,7 +146,6 @@ async fn test_excessive_multipart_parts() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -232,7 +228,6 @@ async fn test_concurrent_object_operations() -> Result<(), Box Result<(), Box> { init_logging(); if !awscurl_available() { diff --git a/crates/e2e_test/src/server_startup_failfast_test.rs b/crates/e2e_test/src/server_startup_failfast_test.rs index 784a08e56..410779405 100644 --- a/crates/e2e_test/src/server_startup_failfast_test.rs +++ b/crates/e2e_test/src/server_startup_failfast_test.rs @@ -23,7 +23,6 @@ #[cfg(test)] mod tests { use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path}; - use serial_test::serial; use std::net::TcpListener; use std::time::{Duration, Instant}; @@ -31,7 +30,6 @@ mod tests { /// while :9001 is occupied: the server exits at startup, and the harness /// must surface that promptly rather than waiting out the 60s timeout. #[tokio::test] - #[serial] async fn test_start_fails_fast_when_server_exits_during_startup() { init_logging(); diff --git a/crates/e2e_test/src/snowball_auto_extract_test.rs b/crates/e2e_test/src/snowball_auto_extract_test.rs index aad32e7cc..2077af321 100644 --- a/crates/e2e_test/src/snowball_auto_extract_test.rs +++ b/crates/e2e_test/src/snowball_auto_extract_test.rs @@ -17,7 +17,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use std::error::Error; use std::io::Cursor; @@ -101,7 +100,6 @@ mod tests { } #[tokio::test] - #[serial] async fn snowball_auto_extract_supports_minio_prefix_and_directory_markers() -> Result<(), Box> { init_logging(); @@ -138,7 +136,6 @@ mod tests { } #[tokio::test] - #[serial] async fn snowball_auto_extract_supports_standard_headers_with_combined_extract_options() -> Result<(), Box> { init_logging(); @@ -229,7 +226,6 @@ mod tests { } #[tokio::test] - #[serial] async fn snowball_auto_extract_ignores_directories_when_requested() -> Result<(), Box> { init_logging(); @@ -268,7 +264,6 @@ mod tests { } #[tokio::test] - #[serial] async fn snowball_auto_extract_ignores_invalid_entries_when_requested() -> Result<(), Box> { init_logging(); @@ -304,7 +299,6 @@ mod tests { } #[tokio::test] - #[serial] async fn snowball_auto_extract_rejects_parent_dir_entry_without_cross_bucket_write() -> Result<(), Box> { init_logging(); @@ -347,7 +341,6 @@ mod tests { } #[tokio::test] - #[serial] async fn snowball_auto_extract_prefers_exact_minio_prefix_over_suffix_fallback() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/special_chars_test.rs b/crates/e2e_test/src/special_chars_test.rs index 8989b02f3..a4df48e78 100644 --- a/crates/e2e_test/src/special_chars_test.rs +++ b/crates/e2e_test/src/special_chars_test.rs @@ -34,7 +34,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; use tracing::{debug, info}; @@ -93,7 +92,6 @@ mod tests { /// mc cp README.md "local/dummy/a%20f+/b/c/3/README.md" /// ``` #[tokio::test] - #[serial] async fn test_object_with_space_in_path() { init_logging(); info!("Starting test: object with space in path"); @@ -175,7 +173,6 @@ mod tests { /// /test/data/org_main-org/dashboards/ES+net/LHC+Data+Challenge/firefly-details.json /// ``` #[tokio::test] - #[serial] async fn test_object_with_plus_in_path() { init_logging(); info!("Starting test: object with plus sign in path"); @@ -245,7 +242,6 @@ mod tests { /// Test with mixed special characters #[tokio::test] - #[serial] async fn test_object_with_mixed_special_chars() { init_logging(); info!("Starting test: object with mixed special characters"); @@ -305,7 +301,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_signed_get_missing_object_with_trailing_equals_returns_no_such_key() -> Result<(), Box> { init_logging(); @@ -334,7 +329,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_signed_get_existing_object_with_trailing_equals_returns_content() -> Result<(), Box> { init_logging(); @@ -373,7 +367,6 @@ mod tests { /// Test DELETE operation with special characters #[tokio::test] - #[serial] async fn test_delete_object_with_special_chars() { init_logging(); info!("Starting test: DELETE object with special characters"); @@ -421,7 +414,6 @@ mod tests { /// Test exact scenario from the issue #[tokio::test] - #[serial] async fn test_issue_scenario_exact() { init_logging(); info!("Starting test: Exact scenario from GitHub issue"); @@ -494,7 +486,6 @@ mod tests { /// Test HEAD object with special characters #[tokio::test] - #[serial] async fn test_head_object_with_special_chars() { init_logging(); info!("Starting test: HEAD object with special characters"); @@ -538,7 +529,6 @@ mod tests { /// Test COPY object with special characters in both source and destination #[tokio::test] - #[serial] async fn test_copy_object_with_special_chars() { init_logging(); info!("Starting test: COPY object with special characters"); @@ -597,7 +587,6 @@ mod tests { /// Test Unicode characters in object keys #[tokio::test] - #[serial] async fn test_unicode_characters_in_path() { init_logging(); info!("Starting test: Unicode characters in object paths"); @@ -661,7 +650,6 @@ mod tests { /// Test special characters in different parts of the path #[tokio::test] - #[serial] async fn test_special_chars_in_different_path_positions() { init_logging(); info!("Starting test: Special characters in different path positions"); @@ -719,7 +707,6 @@ mod tests { /// Test that control characters are properly rejected #[tokio::test] - #[serial] async fn test_control_characters_rejected() { init_logging(); info!("Starting test: Control characters should be rejected"); @@ -769,7 +756,6 @@ mod tests { /// Test LIST with various special character prefixes #[tokio::test] - #[serial] async fn test_list_with_special_char_prefixes() { init_logging(); info!("Starting test: LIST with special character prefixes"); @@ -838,7 +824,6 @@ mod tests { /// Test delimiter-based listing with special characters #[tokio::test] - #[serial] async fn test_list_with_delimiter_and_special_chars() { init_logging(); info!("Starting test: LIST with delimiter and special characters"); diff --git a/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs b/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs index 618b79b61..c5fc45e17 100644 --- a/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs +++ b/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs @@ -16,7 +16,6 @@ use crate::common::{RustFSTestClusterEnvironment, init_logging}; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::CompletedMultipartUpload; -use serial_test::serial; use tokio::time::{Duration, sleep}; use tracing::info; use uuid::Uuid; @@ -101,7 +100,6 @@ async fn wait_for_cleanup_on_all_nodes( } #[tokio::test] -#[serial] async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/sts_query_compat_test.rs b/crates/e2e_test/src/sts_query_compat_test.rs index 517cc1c82..4f96c2e5c 100644 --- a/crates/e2e_test/src/sts_query_compat_test.rs +++ b/crates/e2e_test/src/sts_query_compat_test.rs @@ -25,7 +25,6 @@ use hyper::server::conn::http1; use hyper::service::service_fn; use hyper_util::rt::TokioIo; use serde_json::Value; -use serial_test::serial; use std::collections::BTreeSet; use std::convert::Infallible; use std::error::Error; @@ -350,7 +349,6 @@ impl Drop for OpaMock { } #[tokio::test] -#[serial] async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult { init_logging(); @@ -487,7 +485,6 @@ async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult { } #[tokio::test] -#[serial] async fn test_sts_assume_role_opa_contract() -> TestResult { init_logging(); @@ -559,7 +556,6 @@ async fn test_sts_assume_role_opa_contract() -> TestResult { } #[tokio::test] -#[serial] async fn test_list_buckets_opa_contract() -> TestResult { init_logging(); @@ -645,7 +641,6 @@ async fn test_list_buckets_opa_contract() -> TestResult { } #[tokio::test] -#[serial] async fn test_sts_and_list_buckets_fail_closed_while_opa_is_initializing() -> TestResult { init_logging(); @@ -662,7 +657,6 @@ async fn test_sts_and_list_buckets_fail_closed_while_opa_is_initializing() -> Te } #[tokio::test] -#[serial] async fn test_sts_and_list_buckets_fail_closed_after_opa_validation_failure() -> TestResult { init_logging(); @@ -679,7 +673,6 @@ async fn test_sts_and_list_buckets_fail_closed_after_opa_validation_failure() -> } #[tokio::test] -#[serial] async fn test_sts_query_rate_limit_error_is_aws_sdk_compatible() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/tier_transition_regression_test.rs b/crates/e2e_test/src/tier_transition_regression_test.rs index 5349e3084..26c2922e1 100644 --- a/crates/e2e_test/src/tier_transition_regression_test.rs +++ b/crates/e2e_test/src/tier_transition_regression_test.rs @@ -30,7 +30,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, admin_ok, init_logging}; use serde_json::Value; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -42,7 +41,6 @@ mod tests { /// validates that an expiration-only rule (the persistence path) survives /// a server restart. #[tokio::test] - #[serial] async fn test_lifecycle_rule_persists_after_restart() -> TestResult { init_logging(); info!("RT-13: lifecycle rule persists after restart"); @@ -105,7 +103,6 @@ mod tests { /// Regression pattern: tier add/verify/delete API fails or the tier /// configuration is not persisted (rustfs#5218). #[tokio::test] - #[serial] async fn test_admin_tier_list_endpoint_returns_json() -> TestResult { init_logging(); info!("RT-13b: admin tier list endpoint returns JSON"); @@ -135,7 +132,6 @@ mod tests { /// is not persisted (rustfs#5013), causing the scanner to not run or /// use stale settings. #[tokio::test] - #[serial] async fn test_scanner_config_persists_after_restart() -> TestResult { init_logging(); info!("RT-13c: scanner config persists after restart"); diff --git a/crates/e2e_test/src/tls_hot_reload_test.rs b/crates/e2e_test/src/tls_hot_reload_test.rs index 59c5894ce..b8f752627 100644 --- a/crates/e2e_test/src/tls_hot_reload_test.rs +++ b/crates/e2e_test/src/tls_hot_reload_test.rs @@ -34,7 +34,6 @@ use rcgen::generate_simple_self_signed; use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; use rustls::{ClientConfig, ClientConnection, DigitallySignedStruct, Error as RustlsError, SignatureScheme, StreamOwned}; -use serial_test::serial; use sha2::{Digest, Sha256}; use std::error::Error; use std::io::{Read, Write}; @@ -242,7 +241,6 @@ async fn roundtrip_and_return(mut session: TlsSession) -> Result TestResult { init_logging(); // Install the process-wide rustls crypto provider (idempotent). diff --git a/crates/e2e_test/src/version_id_regression_test.rs b/crates/e2e_test/src/version_id_regression_test.rs index 1ff2e34ef..75d19e5db 100644 --- a/crates/e2e_test/src/version_id_regression_test.rs +++ b/crates/e2e_test/src/version_id_regression_test.rs @@ -27,7 +27,6 @@ mod tests { use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration}; - use serial_test::serial; use tracing::info; fn create_s3_client(env: &RustFSTestEnvironment) -> Client { @@ -86,7 +85,6 @@ mod tests { /// Test 1: PutObject should return version_id when versioning is enabled /// This directly addresses the Veeam issue from #1066 #[tokio::test] - #[serial] async fn test_put_object_returns_version_id_with_versioning() { init_logging(); info!("๐Ÿงช TEST: PutObject returns version_id with versioning enabled"); @@ -130,7 +128,6 @@ mod tests { /// Test 2: CopyObject should return version_id when versioning is enabled #[tokio::test] - #[serial] async fn test_copy_object_returns_version_id_with_versioning() { init_logging(); info!("๐Ÿงช TEST: CopyObject returns version_id with versioning enabled"); @@ -185,7 +182,6 @@ mod tests { /// Test 3: CompleteMultipartUpload should return version_id when versioning is enabled #[tokio::test] - #[serial] async fn test_multipart_upload_returns_version_id_with_versioning() { init_logging(); info!("๐Ÿงช TEST: CompleteMultipartUpload returns version_id with versioning enabled"); @@ -260,7 +256,6 @@ mod tests { /// Test 4: PutObject should NOT return version_id when versioning is NOT enabled /// This ensures we didn't break non-versioned buckets #[tokio::test] - #[serial] async fn test_put_object_without_versioning() { init_logging(); info!("๐Ÿงช TEST: PutObject behavior without versioning (no regression)"); @@ -296,7 +291,6 @@ mod tests { /// Test 5: Basic S3 operations still work correctly (no regression) #[tokio::test] - #[serial] async fn test_basic_s3_operations_no_regression() { init_logging(); info!("๐Ÿงช TEST: Basic S3 operations work correctly (no regression)"); @@ -363,7 +357,6 @@ mod tests { /// Test 6: Veeam-specific scenario simulation /// Simulates the exact workflow that Veeam uses when backing up data #[tokio::test] - #[serial] async fn test_veeam_backup_workflow_simulation() { init_logging(); info!("๐Ÿงช TEST: Veeam VBR backup workflow simulation (Issue #1066)"); @@ -413,7 +406,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_terraform_put_after_delete() -> Result<(), Box> { init_logging(); @@ -456,7 +448,6 @@ mod tests { /// Test 7: PutObject should omit version_id when versioning is Suspended #[tokio::test] - #[serial] async fn test_put_object_omits_version_id_with_suspended_versioning() { init_logging(); info!("๐Ÿงช TEST: PutObject omits version_id with versioning suspended"); @@ -500,7 +491,6 @@ mod tests { /// Test 8: CopyObject should omit version_id when versioning is Suspended #[tokio::test] - #[serial] async fn test_copy_object_omits_version_id_with_suspended_versioning() { init_logging(); info!("๐Ÿงช TEST: CopyObject omits version_id with versioning suspended"); @@ -551,7 +541,6 @@ mod tests { /// Test 9: CompleteMultipartUpload should omit version_id when versioning is Suspended #[tokio::test] - #[serial] async fn test_multipart_upload_omits_version_id_with_suspended_versioning() { init_logging(); info!("๐Ÿงช TEST: CompleteMultipartUpload omits version_id with versioning suspended"); diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 26e9f899a..ed8772f53 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -41,6 +41,10 @@ use bytes::Bytes; use futures::lock::Mutex; use metrics::counter; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; +use rustfs_io_metrics::internode_metrics::{ + INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, + INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, +}; use rustfs_protos::ChannelClass; use rustfs_protos::evict_failed_connection; use rustfs_protos::proto_gen::node_service::RenamePartRequest; @@ -64,7 +68,7 @@ use std::{ atomic::{AtomicBool, AtomicU32, Ordering}, }, task::{Context, Poll}, - time::Duration, + time::{Duration, Instant}, }; use tokio::time; use tokio::{ @@ -1790,6 +1794,16 @@ fn decode_msgpack_or_json(binary: &[u8], json: &str, value_ } } +fn read_version_stage_timer(attribution_enabled: bool) -> Option { + attribution_enabled.then(Instant::now) +} + +fn record_read_version_stage(stage: &'static str, started_at: Option) { + if let Some(started_at) = started_at { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_stage(stage, started_at.elapsed()); + } +} + /// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads /// and falling back to the JSON compatibility strings. Used to size the RPC for the payload /// histogram / large-payload alerting (grpc-optimization P0 instrumentation). @@ -2705,8 +2719,11 @@ impl DiskAPI for RemoteDisk { state = "started", "Remote disk RPC started" ); - let opts_str = compat_json(opts)?; - let opts_bin = encode_msgpack(opts)?; + let read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let encode_started = read_version_stage_timer(read_version_attribution_enabled); + let encoded_opts = compat_json(opts).and_then(|opts_str| encode_msgpack(opts).map(|opts_bin| (opts_str, opts_bin))); + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, encode_started); + let (opts_str, opts_bin) = encoded_opts?; // Idempotent version read: eligible for the bounded transient-network retry so a single // reset-by-peer during the read-after-write window does not erode the metadata read @@ -2722,6 +2739,14 @@ impl DiskAPI for RemoteDisk { .get_client() .await .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; + let request_payload_bytes = read_version_attribution_enabled.then(|| { + disk.len() + .saturating_add(volume.len()) + .saturating_add(path.len()) + .saturating_add(version_id.len()) + .saturating_add(opts_str.len()) + .saturating_add(opts_bin.len()) + }); let request = Request::new(ReadVersionRequest { disk, volume: volume.to_string(), @@ -2731,14 +2756,47 @@ impl DiskAPI for RemoteDisk { opts_bin: opts_bin.into(), }); - let response = client.read_version(request).await?.into_inner(); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_request(); + if let Some(request_payload_bytes) = request_payload_bytes { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_sent_bytes(request_payload_bytes); + } + let rpc_started = read_version_stage_timer(read_version_attribution_enabled); + let response = match client.read_version(request).await { + Ok(response) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, rpc_started); + response.into_inner() + } + Err(err) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, rpc_started); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error(); + return Err(err.into()); + } + }; if !response.success { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error(); return Err(response.error.unwrap_or_default().into()); } - let file_info = decode_msgpack_or_json::(&response.file_info_bin, &response.file_info, "FileInfo")?; - validate_decoded_file_info(&file_info)?; + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_recv_bytes( + response.file_info.len().saturating_add(response.file_info_bin.len()), + ); + let decode_started = read_version_stage_timer(read_version_attribution_enabled); + let file_info = match decode_msgpack_or_json::(&response.file_info_bin, &response.file_info, "FileInfo") + .and_then(|file_info| { + validate_decoded_file_info(&file_info)?; + Ok(file_info) + }) { + Ok(file_info) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, decode_started); + file_info + } + Err(err) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, decode_started); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error(); + return Err(err); + } + }; Ok(file_info) }, @@ -7931,12 +7989,17 @@ mod tests { } #[tokio::test] + #[serial] async fn read_version_uses_the_metadata_timeout_on_a_stalled_peer() { runtime_sources::ensure_test_rpc_secret(); let Some((base_addr, accept_task)) = spawn_stalled_grpc_peer().await else { return; }; let remote_disk = remote_disk_for_addr(&base_addr).await; + let metrics = rustfs_io_metrics::internode_metrics::global_internode_metrics(); + let previous_stage_metrics = rustfs_io_metrics::get_stage_metrics_enabled(); + metrics.reset_for_test(); + rustfs_io_metrics::set_get_stage_metrics_enabled(true); temp_env::async_with_vars( [ @@ -7960,6 +8023,18 @@ mod tests { ) .await; + rustfs_io_metrics::set_get_stage_metrics_enabled(previous_stage_metrics); + let snapshot = metrics.snapshot(); + assert!( + snapshot.outgoing_requests_total >= 1, + "ReadVersion call site should record outgoing attempts when attribution is enabled" + ); + assert!( + snapshot.sent_bytes_total > 0, + "ReadVersion call site should record request payload bytes when attribution is enabled" + ); + metrics.reset_for_test(); + remote_disk.cancel_token.cancel(); accept_task.abort(); } diff --git a/crates/ecstore/src/cluster/rpc/runtime_sources.rs b/crates/ecstore/src/cluster/rpc/runtime_sources.rs index 03d19f60f..0c8393a2e 100644 --- a/crates/ecstore/src/cluster/rpc/runtime_sources.rs +++ b/crates/ecstore/src/cluster/rpc/runtime_sources.rs @@ -14,10 +14,11 @@ use rustfs_io_metrics::internode_metrics::{ INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE, - INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_WRITE_ALL, - INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, - INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics, + INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, + INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics, }; +use std::time::Duration; #[cfg(test)] use rustfs_io_metrics::internode_metrics::InternodeMetricsSnapshot; @@ -82,6 +83,59 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() { .record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC); } +pub(crate) fn record_remote_disk_grpc_read_version_request() { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_outgoing_request_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); +} + +pub(crate) fn record_remote_disk_grpc_read_version_error() { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics() + .record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC); +} + +pub(crate) fn record_remote_disk_grpc_read_version_sent_bytes(bytes: usize) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_sent_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + bytes, + ); +} + +pub(crate) fn record_remote_disk_grpc_read_version_recv_bytes(bytes: usize) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_recv_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + bytes, + ); + record_grpc_payload_size(INTERNODE_OPERATION_GRPC_READ_VERSION, bytes); +} + +pub(crate) fn record_remote_disk_grpc_read_version_stage(stage: &'static str, duration: Duration) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_stage_duration_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + stage, + duration, + ); +} + pub(crate) fn record_remote_disk_grpc_read_all_recv_bytes(bytes: usize) { global_internode_metrics().record_recv_bytes_for_operation_and_backend( INTERNODE_OPERATION_GRPC_READ_ALL, diff --git a/crates/ecstore/src/disk/error.rs b/crates/ecstore/src/disk/error.rs index 51fb04daa..9c2f1706c 100644 --- a/crates/ecstore/src/disk/error.rs +++ b/crates/ecstore/src/disk/error.rs @@ -853,13 +853,32 @@ mod tests { #[test] fn test_error_conversions() { - // Test From implementations + // A plain io::Error carries no typed payload to recover, so it lands in + // `Io` rather than being guessed at from its kind โ€” `NotFound` here must + // not silently become `FileNotFound`, which quorum aggregation counts as + // a different error (rustfs/backlog#1836). let io_error = std::io::Error::new(std::io::ErrorKind::NotFound, "test"); - let _disk_error: DiskError = io_error.into(); + let disk_error: DiskError = io_error.into(); + match &disk_error { + DiskError::Io(inner) => assert_eq!(inner.kind(), std::io::ErrorKind::NotFound), + other => panic!("a plain io::Error must stay typed as Io, got {other:?}"), + } - let json_str = r#"{"invalid": json}"#; // Invalid JSON + // A typed DiskError boxed through io::Error round-trips back to itself + // instead of degrading to `Io`. + let boxed: std::io::Error = std::io::Error::other(DiskError::VolumeNotFound); + assert_eq!(DiskError::from(boxed), DiskError::VolumeNotFound); + + // serde_json errors have no dedicated variant and fold into `other`, + // keeping the original message. + let json_str = r#"{"invalid": json}"#; let json_error = serde_json::from_str::(json_str).unwrap_err(); - let _disk_error: DiskError = json_error.into(); + let json_message = json_error.to_string(); + let disk_error: DiskError = json_error.into(); + assert!( + disk_error.to_string().contains(&json_message), + "the json error message must survive the conversion: {disk_error}" + ); } #[test] diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index 94392fa81..acc3a5e5e 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -1098,6 +1098,10 @@ pub(crate) async fn sync_dir_files_with_limiter(dir: impl AsRef, disk_perm let files = run_file_sync_blocking(disk_permits.clone(), move || { let files = regular_files(&scan_dir)?; if files.len() < PARALLEL_FILE_SYNC_THRESHOLD { + rustfs_io_metrics::record_put_rename_fdatasync_batch( + rustfs_io_metrics::PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL, + files.len(), + ); sync_files(&files)?; let fsync_started = rustfs_io_metrics::put_stage_timer(); let result = fsync_dir_std(scan_dir); @@ -1115,6 +1119,10 @@ pub(crate) async fn sync_dir_files_with_limiter(dir: impl AsRef, disk_perm let Some(files) = files else { return Ok(()); }; + rustfs_io_metrics::record_put_rename_fdatasync_batch( + rustfs_io_metrics::PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL, + files.len(), + ); futures::stream::iter(files.into_iter().map(Ok::<_, io::Error>)) .try_for_each_concurrent(MAX_PARALLEL_FILE_SYNCS, |path| { let disk_permits = disk_permits.clone(); diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index dcf5d11e3..784969051 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -3417,6 +3417,25 @@ impl SetDisks { quorum_wait_started, ); let (results, mut file_infos) = fanout_result.map_err(|_| DiskError::Unexpected)?; + if rustfs_io_metrics::put_stage_metrics_enabled() { + let mut fanout_success = 0; + let mut fanout_error = 0; + let mut fanout_panic = 0; + for result in &results { + match result { + Ok(Ok(_)) => fanout_success += 1, + Ok(Err(_)) => fanout_error += 1, + Err(_) => fanout_panic += 1, + } + } + rustfs_io_metrics::record_put_rename_quorum_wait_fanout( + results.len(), + write_quorum, + fanout_success, + fanout_error, + fanout_panic, + ); + } for (idx, result) in results.iter().enumerate() { match result { diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index 8f4ab9e8a..edfcf2613 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -759,7 +759,7 @@ impl HealChannelProcessor { #[cfg(test)] mod tests { - use super::super::{DiskStore, Endpoint}; + use super::super::DiskStore; use super::*; use crate::heal::manager::HealConfig; use crate::heal::storage::{HealObjectInfo, HealStorageAPI}; @@ -776,45 +776,18 @@ mod tests { async fn get_object_meta(&self, _bucket: &str, _object: &str) -> crate::Result> { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> crate::Result>> { - Ok(None) - } - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> crate::Result<()> { - Ok(()) - } - async fn delete_object(&self, _bucket: &str, _object: &str) -> crate::Result<()> { - Ok(()) - } - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> crate::Result { - Ok(true) - } async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> crate::Result> { Ok(vec![]) } - async fn get_disk_status(&self, _endpoint: &Endpoint) -> crate::Result { - Ok(crate::heal::storage::DiskStatus::Ok) - } - async fn format_disk(&self, _endpoint: &Endpoint) -> crate::Result<()> { - Ok(()) - } async fn get_bucket_info(&self, _bucket: &str) -> crate::Result> { Ok(None) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> crate::Result<()> { - Ok(()) - } async fn list_buckets(&self) -> crate::Result> { Ok(vec![]) } async fn object_exists(&self, _bucket: &str, _object: &str) -> crate::Result { Ok(false) } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> crate::Result> { - Ok(None) - } - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> crate::Result> { - Ok(None) - } async fn heal_object( &self, _bucket: &str, @@ -837,13 +810,6 @@ mod tests { ) -> crate::Result<(rustfs_madmin::heal_commands::HealResultItem, Option)> { Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None)) } - async fn list_objects_for_heal( - &self, - _bucket: &str, - _prefix: &str, - ) -> crate::Result> { - Ok(vec![]) - } async fn list_objects_for_heal_page( &self, _bucket: &str, diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 04075837c..920fffcf5 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -1267,7 +1267,7 @@ mod resume_loop_tests { CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils, compose_key, }; - use crate::heal::storage::{DiskStatus, HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI}; + use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI}; use crate::heal::storage_api::status::BucketInfo; use crate::heal::{ BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk, @@ -1448,36 +1448,15 @@ mod resume_loop_tests { async fn get_object_meta(&self, _b: &str, _o: &str) -> Result> { Ok(None) } - async fn get_object_data(&self, _b: &str, _o: &str) -> Result>> { - Ok(None) - } - async fn put_object_data(&self, _b: &str, _o: &str, _d: &[u8]) -> Result<()> { - Ok(()) - } - async fn delete_object(&self, _b: &str, _o: &str) -> Result<()> { - Ok(()) - } - async fn verify_object_integrity(&self, _b: &str, _o: &str) -> Result { - Ok(true) - } async fn ec_decode_rebuild(&self, _b: &str, _o: &str) -> Result> { Ok(Vec::new()) } - async fn get_disk_status(&self, _e: &Endpoint) -> Result { - Ok(DiskStatus::Ok) - } - async fn format_disk(&self, _e: &Endpoint) -> Result<()> { - Ok(()) - } async fn get_bucket_info(&self, bucket: &str) -> Result> { Ok(Some(BucketInfo { name: bucket.to_string(), ..Default::default() })) } - async fn heal_bucket_metadata(&self, _b: &str) -> Result<()> { - Ok(()) - } async fn list_buckets(&self) -> Result> { Ok(Vec::new()) } @@ -1485,12 +1464,6 @@ mod resume_loop_tests { // Must never be consulted: the resume loop always goes through heal_object. panic!("object_exists must not be called by the resume heal loop"); } - async fn get_object_size(&self, _b: &str, _o: &str) -> Result> { - Ok(None) - } - async fn get_object_checksum(&self, _b: &str, _o: &str) -> Result> { - Ok(None) - } async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result> { Ok((!self.lifecycle_expired.lock().unwrap().is_empty()).then(HealLifecycleExpiryContext::test)) } @@ -1556,9 +1529,6 @@ mod resume_loop_tests { ReplacementCommitEvidence::Error(message) => Err(Error::other(message)), } } - async fn list_objects_for_heal(&self, _b: &str, _p: &str) -> Result> { - Ok(Vec::new()) - } async fn list_objects_for_heal_page( &self, _bucket: &str, diff --git a/crates/heal/src/heal/event.rs b/crates/heal/src/heal/event.rs deleted file mode 100644 index 0d46b859b..000000000 --- a/crates/heal/src/heal/event.rs +++ /dev/null @@ -1,683 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::heal::{HealOptions, HealPriority, HealRequest, HealType}; -use crate::{Error, Result}; -use serde::{Deserialize, Serialize}; -use std::time::SystemTime; - -use super::Endpoint; - -/// Corruption type -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum CorruptionType { - /// Data corruption - DataCorruption, - /// Metadata corruption - MetadataCorruption, - /// Partial corruption - PartialCorruption, - /// Complete corruption - CompleteCorruption, -} - -/// Severity level -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub enum Severity { - /// Low severity - Low = 0, - /// Medium severity - Medium = 1, - /// High severity - High = 2, - /// Critical severity - Critical = 3, -} - -/// Heal event -#[derive(Debug, Clone)] -pub enum HealEvent { - /// Object corruption event - ObjectCorruption { - bucket: String, - object: String, - version_id: Option, - corruption_type: CorruptionType, - severity: Severity, - }, - /// Object missing event - ObjectMissing { - bucket: String, - object: String, - version_id: Option, - expected_locations: Vec, - available_locations: Vec, - }, - /// Metadata corruption event - MetadataCorruption { - bucket: String, - object: String, - corruption_type: CorruptionType, - }, - /// Disk status change event - DiskStatusChange { - endpoint: Endpoint, - old_status: String, - new_status: String, - }, - /// EC decode failure event - ECDecodeFailure { - bucket: String, - object: String, - version_id: Option, - missing_shards: Vec, - available_shards: Vec, - }, - /// Checksum mismatch event - ChecksumMismatch { - bucket: String, - object: String, - version_id: Option, - expected_checksum: String, - actual_checksum: String, - }, - /// Bucket metadata corruption event - BucketMetadataCorruption { - bucket: String, - corruption_type: CorruptionType, - }, - /// MRF metadata corruption event - MRFMetadataCorruption { - meta_path: String, - corruption_type: CorruptionType, - }, -} - -impl HealEvent { - /// Convert HealEvent to HealRequest - pub fn to_heal_request(&self) -> Result { - match self { - HealEvent::ObjectCorruption { - bucket, - object, - version_id, - severity, - .. - } => Ok(HealRequest::new( - HealType::Object { - bucket: bucket.clone(), - object: object.clone(), - version_id: version_id.clone(), - }, - HealOptions::default(), - Self::severity_to_priority(severity), - )), - HealEvent::ObjectMissing { - bucket, - object, - version_id, - .. - } => Ok(HealRequest::new( - HealType::Object { - bucket: bucket.clone(), - object: object.clone(), - version_id: version_id.clone(), - }, - HealOptions::default(), - HealPriority::High, - )), - HealEvent::MetadataCorruption { bucket, object, .. } => Ok(HealRequest::new( - HealType::Metadata { - bucket: bucket.clone(), - object: object.clone(), - }, - HealOptions::default(), - HealPriority::High, - )), - HealEvent::DiskStatusChange { endpoint, .. } => { - // Convert disk status change to erasure set heal - // Note: This requires access to storage to get bucket list, which is not available here - // The actual bucket list will need to be provided by the caller or retrieved differently - let set_disk_id = crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx) - .ok_or_else(|| Error::InvalidHealType { - heal_type: format!("erasure-set(pool={}, set={})", endpoint.pool_idx, endpoint.set_idx), - })?; - Ok(HealRequest::new( - HealType::ErasureSet { - buckets: vec![], // Empty bucket list - caller should populate this - set_disk_id, - }, - HealOptions::default(), - HealPriority::High, - )) - } - HealEvent::ECDecodeFailure { - bucket, - object, - version_id, - .. - } => Ok(HealRequest::new( - HealType::ECDecode { - bucket: bucket.clone(), - object: object.clone(), - version_id: version_id.clone(), - }, - HealOptions::default(), - HealPriority::Urgent, - )), - HealEvent::ChecksumMismatch { - bucket, - object, - version_id, - .. - } => Ok(HealRequest::new( - HealType::Object { - bucket: bucket.clone(), - object: object.clone(), - version_id: version_id.clone(), - }, - HealOptions::default(), - HealPriority::High, - )), - HealEvent::BucketMetadataCorruption { bucket, .. } => Ok(HealRequest::new( - HealType::Bucket { bucket: bucket.clone() }, - HealOptions::default(), - HealPriority::High, - )), - HealEvent::MRFMetadataCorruption { meta_path, .. } => Ok(HealRequest::new( - HealType::MRF { - meta_path: meta_path.clone(), - }, - HealOptions::default(), - HealPriority::High, - )), - } - } - - /// Convert severity to priority - fn severity_to_priority(severity: &Severity) -> HealPriority { - match severity { - Severity::Low => HealPriority::Low, - Severity::Medium => HealPriority::Normal, - Severity::High => HealPriority::High, - Severity::Critical => HealPriority::Urgent, - } - } - - /// Get event description - pub fn description(&self) -> String { - match self { - HealEvent::ObjectCorruption { - bucket, - object, - corruption_type, - .. - } => { - format!("Object corruption detected: {bucket}/{object} - {corruption_type:?}") - } - HealEvent::ObjectMissing { bucket, object, .. } => { - format!("Object missing: {bucket}/{object}") - } - HealEvent::MetadataCorruption { - bucket, - object, - corruption_type, - .. - } => { - format!("Metadata corruption: {bucket}/{object} - {corruption_type:?}") - } - HealEvent::DiskStatusChange { - endpoint, - old_status, - new_status, - .. - } => { - format!("Disk status changed: {endpoint:?} {old_status} -> {new_status}") - } - HealEvent::ECDecodeFailure { - bucket, - object, - missing_shards, - .. - } => { - format!("EC decode failure: {bucket}/{object} - missing shards: {missing_shards:?}") - } - HealEvent::ChecksumMismatch { - bucket, - object, - expected_checksum, - actual_checksum, - .. - } => { - format!("Checksum mismatch: {bucket}/{object} - expected: {expected_checksum}, actual: {actual_checksum}") - } - HealEvent::BucketMetadataCorruption { - bucket, corruption_type, .. - } => { - format!("Bucket metadata corruption: {bucket} - {corruption_type:?}") - } - HealEvent::MRFMetadataCorruption { - meta_path, - corruption_type, - .. - } => { - format!("MRF metadata corruption: {meta_path} - {corruption_type:?}") - } - } - } - - /// Get event severity - pub fn severity(&self) -> Severity { - match self { - HealEvent::ObjectCorruption { severity, .. } => severity.clone(), - HealEvent::ObjectMissing { .. } => Severity::High, - HealEvent::MetadataCorruption { .. } => Severity::High, - HealEvent::DiskStatusChange { .. } => Severity::High, - HealEvent::ECDecodeFailure { .. } => Severity::Critical, - HealEvent::ChecksumMismatch { .. } => Severity::High, - HealEvent::BucketMetadataCorruption { .. } => Severity::High, - HealEvent::MRFMetadataCorruption { .. } => Severity::High, - } - } - - /// Get event timestamp - pub fn timestamp(&self) -> SystemTime { - SystemTime::now() - } -} - -/// Heal event handler -pub struct HealEventHandler { - /// Event queue - events: Vec, - /// Maximum number of events - max_events: usize, -} - -impl HealEventHandler { - pub fn new(max_events: usize) -> Self { - Self { - events: Vec::new(), - max_events, - } - } - - /// Add event - pub fn add_event(&mut self, event: HealEvent) { - if self.events.len() >= self.max_events { - // Remove oldest event - self.events.remove(0); - } - self.events.push(event); - } - - /// Get all events - pub fn get_events(&self) -> &[HealEvent] { - &self.events - } - - /// Clear events - pub fn clear_events(&mut self) { - self.events.clear(); - } - - /// Get event count - pub fn event_count(&self) -> usize { - self.events.len() - } - - /// Filter events by severity - pub fn filter_by_severity(&self, min_severity: Severity) -> Vec<&HealEvent> { - self.events.iter().filter(|event| event.severity() >= min_severity).collect() - } - - /// Filter events by type - pub fn filter_by_type(&self, event_type: &str) -> Vec<&HealEvent> { - self.events - .iter() - .filter(|event| match event { - HealEvent::ObjectCorruption { .. } => event_type == "ObjectCorruption", - HealEvent::ObjectMissing { .. } => event_type == "ObjectMissing", - HealEvent::MetadataCorruption { .. } => event_type == "MetadataCorruption", - HealEvent::DiskStatusChange { .. } => event_type == "DiskStatusChange", - HealEvent::ECDecodeFailure { .. } => event_type == "ECDecodeFailure", - HealEvent::ChecksumMismatch { .. } => event_type == "ChecksumMismatch", - HealEvent::BucketMetadataCorruption { .. } => event_type == "BucketMetadataCorruption", - HealEvent::MRFMetadataCorruption { .. } => event_type == "MRFMetadataCorruption", - }) - .collect() - } -} - -impl Default for HealEventHandler { - fn default() -> Self { - Self::new(1000) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::heal::task::{HealPriority, HealType}; - - #[test] - fn test_heal_event_object_corruption_to_request() { - let event = HealEvent::ObjectCorruption { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::Object { .. })); - assert_eq!(request.priority, HealPriority::High); - } - - #[test] - fn test_heal_event_object_missing_to_request() { - let event = HealEvent::ObjectMissing { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: Some("v1".to_string()), - expected_locations: vec![0, 1], - available_locations: vec![2, 3], - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::Object { .. })); - assert_eq!(request.priority, HealPriority::High); - } - - #[test] - fn test_heal_event_metadata_corruption_to_request() { - let event = HealEvent::MetadataCorruption { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - corruption_type: CorruptionType::MetadataCorruption, - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::Metadata { .. })); - assert_eq!(request.priority, HealPriority::High); - } - - #[test] - fn test_heal_event_ec_decode_failure_to_request() { - let event = HealEvent::ECDecodeFailure { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: None, - missing_shards: vec![0, 1], - available_shards: vec![2, 3, 4], - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::ECDecode { .. })); - assert_eq!(request.priority, HealPriority::Urgent); - } - - #[test] - fn test_heal_event_checksum_mismatch_to_request() { - let event = HealEvent::ChecksumMismatch { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: None, - expected_checksum: "abc123".to_string(), - actual_checksum: "def456".to_string(), - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::Object { .. })); - assert_eq!(request.priority, HealPriority::High); - } - - #[test] - fn test_heal_event_bucket_metadata_corruption_to_request() { - let event = HealEvent::BucketMetadataCorruption { - bucket: "test-bucket".to_string(), - corruption_type: CorruptionType::MetadataCorruption, - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::Bucket { .. })); - assert_eq!(request.priority, HealPriority::High); - } - - #[test] - fn test_heal_event_mrf_metadata_corruption_to_request() { - let event = HealEvent::MRFMetadataCorruption { - meta_path: "test-bucket/test-object".to_string(), - corruption_type: CorruptionType::MetadataCorruption, - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::MRF { .. })); - assert_eq!(request.priority, HealPriority::High); - } - - #[test] - fn test_heal_event_severity_to_priority() { - let event_low = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::Low, - }; - let request = event_low.to_heal_request().unwrap(); - assert_eq!(request.priority, HealPriority::Low); - - let event_medium = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::Medium, - }; - let request = event_medium.to_heal_request().unwrap(); - assert_eq!(request.priority, HealPriority::Normal); - - let event_high = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - let request = event_high.to_heal_request().unwrap(); - assert_eq!(request.priority, HealPriority::High); - - let event_critical = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::Critical, - }; - let request = event_critical.to_heal_request().unwrap(); - assert_eq!(request.priority, HealPriority::Urgent); - } - - #[test] - fn test_heal_event_description() { - let event = HealEvent::ObjectCorruption { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - - let desc = event.description(); - assert!(desc.contains("Object corruption detected")); - assert!(desc.contains("test-bucket/test-object")); - assert!(desc.contains("DataCorruption")); - } - - #[test] - fn test_heal_event_severity() { - let event = HealEvent::ECDecodeFailure { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - missing_shards: vec![], - available_shards: vec![], - }; - assert_eq!(event.severity(), Severity::Critical); - - let event = HealEvent::ObjectMissing { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - expected_locations: vec![], - available_locations: vec![], - }; - assert_eq!(event.severity(), Severity::High); - } - - #[test] - fn test_heal_event_handler_new() { - let handler = HealEventHandler::new(10); - assert_eq!(handler.event_count(), 0); - assert_eq!(handler.max_events, 10); - } - - #[test] - fn test_heal_event_handler_default() { - let handler = HealEventHandler::default(); - assert_eq!(handler.max_events, 1000); - } - - #[test] - fn test_heal_event_handler_add_event() { - let mut handler = HealEventHandler::new(3); - let event = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - - handler.add_event(event.clone()); - assert_eq!(handler.event_count(), 1); - - handler.add_event(event.clone()); - handler.add_event(event); - assert_eq!(handler.event_count(), 3); - } - - #[test] - fn test_heal_event_handler_max_events() { - let mut handler = HealEventHandler::new(2); - let event = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - - handler.add_event(event.clone()); - handler.add_event(event.clone()); - handler.add_event(event); // Should remove oldest - - assert_eq!(handler.event_count(), 2); - } - - #[test] - fn test_heal_event_handler_get_events() { - let mut handler = HealEventHandler::new(10); - let event = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - - handler.add_event(event.clone()); - handler.add_event(event); - - let events = handler.get_events(); - assert_eq!(events.len(), 2); - } - - #[test] - fn test_heal_event_handler_clear_events() { - let mut handler = HealEventHandler::new(10); - let event = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - - handler.add_event(event); - assert_eq!(handler.event_count(), 1); - - handler.clear_events(); - assert_eq!(handler.event_count(), 0); - } - - #[test] - fn test_heal_event_handler_filter_by_severity() { - let mut handler = HealEventHandler::new(10); - handler.add_event(HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::Low, - }); - handler.add_event(HealEvent::ECDecodeFailure { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - missing_shards: vec![], - available_shards: vec![], - }); - - let high_severity = handler.filter_by_severity(Severity::High); - assert_eq!(high_severity.len(), 1); // Only ECDecodeFailure is Critical >= High - } - - #[test] - fn test_heal_event_handler_filter_by_type() { - let mut handler = HealEventHandler::new(10); - handler.add_event(HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }); - handler.add_event(HealEvent::ObjectMissing { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - expected_locations: vec![], - available_locations: vec![], - }); - - let corruption_events = handler.filter_by_type("ObjectCorruption"); - assert_eq!(corruption_events.len(), 1); - - let missing_events = handler.filter_by_type("ObjectMissing"); - assert_eq!(missing_events.len(), 1); - } -} diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 3a243c88e..2f03b30d4 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -86,7 +86,7 @@ fn unblock_replacement_recovery_sets_after_validation( } } -// Admission/scheduler outcomes for per-object requests (Object/Metadata/MRF/ +// Admission/scheduler outcomes for per-object requests (Object/Metadata/ // ECDecode) log via demote_to_debug_when! โ€” MRF, autoheal, and scanner // recovery loops submit those per object, so a full queue or a retry storm // would otherwise emit one warn! per object (rustfs/rustfs#5716). The @@ -143,6 +143,16 @@ async fn pause_duplicate_admission_after_active_lock(request_id: &str) { type WorkloadSnapshotProviderRef = Arc; +/// Per-key bookkeeping for the queued-request dedup index: how many queued +/// requests hold the key, and the id of the first request that opened it โ€” +/// the O(1) stand-in for the former heap scan when a merge receipt needs to +/// name a queued representative. +#[derive(Debug)] +struct DedupKeyEntry { + refcount: usize, + representative_request_id: String, +} + /// Priority queue wrapper for heal requests /// Uses BinaryHeap for priority-based ordering while maintaining FIFO for same-priority items #[derive(Debug)] @@ -151,8 +161,8 @@ struct PriorityHealQueue { heap: BinaryHeap, /// Sequence counter for FIFO ordering within same priority sequence: u64, - /// Deduplication key reference counts for queued requests - dedup_keys: HashMap, + /// Deduplication index for queued requests + dedup_keys: HashMap, } /// Wrapper for heap items to implement proper ordering @@ -402,8 +412,16 @@ impl PriorityHealQueue { return QueuePushOutcome::Merged; } // Track dedup keys for both normal and forced requests so queued forced work - // also reserves the dedup key for later non-forced duplicates. - *self.dedup_keys.entry(key).or_insert(0) += 1; + // also reserves the dedup key for later non-forced duplicates. The first + // request that opens the key becomes the named representative for merge + // receipts (taken before `request` moves into the heap). + self.dedup_keys + .entry(key) + .or_insert_with(|| DedupKeyEntry { + refcount: 0, + representative_request_id: request.id.clone(), + }) + .refcount += 1; self.sequence += 1; self.heap.push(PriorityQueueItem { priority: request.priority, @@ -447,6 +465,7 @@ impl PriorityHealQueue { let displaced = displaced.map(|item| { let key = Self::make_dedup_key(&item.request); Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); + self.refresh_dedup_representative(&key); item.request }); @@ -559,9 +578,6 @@ impl PriorityHealQueue { HealType::Metadata { bucket, object } => { format!("metadata:{bucket}:{object}") } - HealType::MRF { meta_path } => { - format!("mrf:{meta_path}") - } HealType::ECDecode { bucket, object, @@ -572,12 +588,12 @@ impl PriorityHealQueue { } } - fn decrement_or_remove_dedup_key(dedup_keys: &mut HashMap, key: &str) { - if let Some(count) = dedup_keys.get_mut(key) { - if *count <= 1 { + fn decrement_or_remove_dedup_key(dedup_keys: &mut HashMap, key: &str) { + if let Some(entry) = dedup_keys.get_mut(key) { + if entry.refcount <= 1 { dedup_keys.remove(key); } else { - *count -= 1; + entry.refcount -= 1; } } } @@ -610,10 +626,32 @@ impl PriorityHealQueue { .any(|item| item.request.id == request_id && heal_type_matches_path(&item.request.heal_type, heal_path)) } - fn request_for_dedup_key(&self, key: &str) -> Option<&HealRequest> { - self.heap + fn queued_request_id_for_dedup_key(&self, key: &str) -> Option<&str> { + self.dedup_keys.get(key).map(|entry| entry.representative_request_id.as_str()) + } + + /// Re-elect the representative for `key` from the queue entries holding + /// it. Needed after a holder leaves the queue *without* becoming active + /// (canceled by id, or displaced): the former opener may be the request + /// that just left, and a merge receipt must never name an id that + /// resolves nowhere. The scheduler pop path does not need this โ€” the + /// popped request surfaces in `active_heals` under the same id and the + /// duplicate pre-check consults active heals before the queue. No-op for + /// released keys; the survivor scan only runs when a key still has + /// holders, which under forced duplicates is the rare admin path. + fn refresh_dedup_representative(&mut self, key: &str) { + if !self.dedup_keys.contains_key(key) { + return; + } + if let Some(id) = self + .heap .iter() - .find_map(|item| (Self::make_dedup_key(&item.request) == key).then_some(&item.request)) + .find(|item| Self::make_dedup_key(&item.request) == key) + .map(|item| item.request.id.clone()) + && let Some(entry) = self.dedup_keys.get_mut(key) + { + entry.representative_request_id = id; + } } fn contains_matching(&self, mut matches: F) -> bool @@ -638,6 +676,9 @@ impl PriorityHealQueue { } self.heap = retained; + if let Some(removed) = removed.as_ref() { + self.refresh_dedup_representative(&Self::make_dedup_key(removed)); + } removed } @@ -647,11 +688,13 @@ impl PriorityHealQueue { { let mut retained = BinaryHeap::new(); let mut removed = Vec::new(); + let mut affected_keys = Vec::new(); while let Some(item) = self.heap.pop() { if should_remove(&item.request) { let key = Self::make_dedup_key(&item.request); Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); + affected_keys.push(key); removed.push(item.request); } else { retained.push(item); @@ -659,6 +702,9 @@ impl PriorityHealQueue { } self.heap = retained; + for key in &affected_keys { + self.refresh_dedup_representative(key); + } removed } } @@ -686,7 +732,6 @@ fn heal_type_matches_path(heal_type: &HealType, heal_path: &str) -> bool { HealType::Bucket { bucket } => heal_path == bucket, HealType::Prefix { bucket, prefix } => heal_path_matches_bucket_child(heal_path, bucket, prefix), HealType::ErasureSet { set_disk_id, .. } => heal_path == set_disk_id, - HealType::MRF { meta_path } => heal_path == meta_path.trim_matches('/'), } } @@ -781,9 +826,6 @@ fn heal_type_path_view(heal_type: &HealType) -> (Option<&str>, &str) { HealType::Object { bucket, object, .. } | HealType::Metadata { bucket, object } | HealType::ECDecode { bucket, object, .. } => (Some(bucket), object), - // MRF/MetaPath heal keys on a meta path; treat the whole set of - // buckets as one namespace so it only overlaps itself exactly. - HealType::MRF { meta_path } => (Some("\u{0}mrf"), meta_path), // Erasure-set heal: the set id is the overlap dimension. HealType::ErasureSet { set_disk_id, .. } => (Some("\u{0}set"), set_disk_id), } @@ -1974,8 +2016,8 @@ impl HealManager { .map(|(task_id, _)| (task_id, "active")) .or_else(|| { queue - .request_for_dedup_key(&dedup_key) - .map(|queued| (queued.id.clone(), "queued")) + .queued_request_id_for_dedup_key(&dedup_key) + .map(|queued_id| (queued_id.to_string(), "queued")) }) .or_else(|| retrying_heal_for_dedup_key(&retrying_heals, &dedup_key).map(|(task_id, _)| (task_id, "retrying"))) }); @@ -2093,9 +2135,9 @@ impl HealManager { let mut task_id = request.id.clone(); let admission = Self::admit_request_to_queue(&mut queue, request, &config, "submit"); if admission == HealAdmissionResult::Merged - && let Some(queued) = queue.request_for_dedup_key(&dedup_key) + && let Some(queued_id) = queue.queued_request_id_for_dedup_key(&dedup_key) { - task_id.clone_from(&queued.id); + task_id = queued_id.to_owned(); } let should_notify = matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable; drop(retrying_heals); @@ -3712,7 +3754,6 @@ fn heal_request_type_label(request: &HealRequest) -> &'static str { HealType::Prefix { .. } => "prefix", HealType::ErasureSet { .. } => "erasure_set", HealType::Metadata { .. } => "metadata", - HealType::MRF { .. } => "mrf", HealType::ECDecode { .. } => "ec_decode", } } @@ -3882,42 +3923,14 @@ mod tests { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> Result>> { - Ok(None) - } - - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> Result<()> { - Ok(()) - } - - async fn delete_object(&self, _bucket: &str, _object: &str) -> Result<()> { - Ok(()) - } - - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> Result { - Ok(true) - } - async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result> { Ok(Vec::new()) } - async fn get_disk_status(&self, _endpoint: &Endpoint) -> Result { - Ok(crate::heal::storage::DiskStatus::Ok) - } - - async fn format_disk(&self, _endpoint: &Endpoint) -> Result<()> { - Ok(()) - } - async fn get_bucket_info(&self, _bucket: &str) -> Result> { Ok(None) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> { - Ok(()) - } - async fn list_buckets(&self) -> Result> { if let Some(hook) = manager_recovery_test_hook() { *hook.listed.lock().expect("manager recovery listed lock should not poison") = true; @@ -3929,14 +3942,6 @@ mod tests { Ok(bucket == "retry-transition") } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> Result> { - Ok(None) - } - - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> Result> { - Ok(None) - } - async fn heal_object( &self, bucket: &str, @@ -3998,10 +4003,6 @@ mod tests { Ok((HealResultItem::default(), None)) } - async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> Result> { - Ok(Vec::new()) - } - async fn list_objects_for_heal_page( &self, _bucket: &str, @@ -4144,6 +4145,46 @@ mod tests { assert_eq!(queue.len(), 0); } + #[test] + fn queued_request_id_for_dedup_key_tracks_the_representative() { + let mut queue = PriorityHealQueue::new(); + + let first = HealRequest::object("bucket".to_string(), "object".to_string(), None); + let first_id = first.id.clone(); + let first_key = PriorityHealQueue::make_dedup_key(&first); + assert_eq!(queue.push(first), QueuePushOutcome::Accepted); + + // A forced duplicate of the same target opens a second entry under + // the same key; the representative stays the request that opened it. + let mut second = HealRequest::object("bucket".to_string(), "object".to_string(), None); + second.force_start = true; + let second_id = second.id.clone(); + assert_eq!(queue.push(second), QueuePushOutcome::Accepted); + + let representative = queue + .queued_request_id_for_dedup_key(&first_key) + .expect("key must be reserved while either request is queued"); + assert_eq!(representative, first_id); + + // A holder leaving WITHOUT becoming active (canceled by id) must + // re-elect the representative to the surviving queued request, or a + // later merge receipt would name an id that resolves nowhere. The + // scheduler pop path needs no re-election: the popped request + // surfaces in active_heals under the same id and the duplicate + // pre-check consults active heals before the queue. + queue.remove_request_id(&first_id); + assert_eq!( + queue.queued_request_id_for_dedup_key(&first_key), + Some(second_id.as_str()), + "canceling the opener must re-elect the surviving queued holder" + ); + + // Pop the last holder: the key is released entirely. + let last = queue.pop_next().expect("second request must be queued"); + assert_eq!(last.id, second_id); + assert!(queue.queued_request_id_for_dedup_key(&first_key).is_none()); + } + #[test] fn test_priority_queue_ordering() { let mut queue = PriorityHealQueue::new(); diff --git a/crates/heal/src/heal/mod.rs b/crates/heal/src/heal/mod.rs index ff910ed70..35080e8e9 100644 --- a/crates/heal/src/heal/mod.rs +++ b/crates/heal/src/heal/mod.rs @@ -14,7 +14,6 @@ pub mod channel; pub mod erasure_healer; -pub mod event; pub mod manager; pub mod mrf_queue; pub mod progress; diff --git a/crates/heal/src/heal/mrf_queue.rs b/crates/heal/src/heal/mrf_queue.rs index c1144435e..5101a5e12 100644 --- a/crates/heal/src/heal/mrf_queue.rs +++ b/crates/heal/src/heal/mrf_queue.rs @@ -276,20 +276,26 @@ async fn read_journal() -> Option> { None } -async fn write_journal(data: &[u8]) { +/// Write the snapshot to every local disk; returns true when at least one +/// disk accepted it, so a total write failure keeps the runtime dirty and +/// the next tick retries the persist. +async fn write_journal(data: &[u8]) -> bool { let payload = bytes::Bytes::copy_from_slice(data); + let mut any_persisted = false; for disk in journal_disks().await { - if let Err(err) = disk + match disk .write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone()) .await { - warn_mrf_journal_write(&err); + Ok(()) => any_persisted = true, + Err(err) => warn_mrf_journal_write(&err), } } if !data.is_empty() { counter!("rustfs_heal_mrf_journal_fsync_total").increment(1); } gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64); + any_persisted } async fn delete_journal() { @@ -351,6 +357,12 @@ struct MrfRuntime { queue: MrfQueue, config: MrfConsumerConfig, new_since_flush: usize, + /// True while the in-memory pending set has changed since the last + /// journal flush (push, pop, or an attempts bump that alters the encoded + /// bytes). Only a dirty state rewrites the snapshot: a steady backlog + /// waiting out an admission backoff must not re-fsync every local disk + /// twice a second. + dirty: bool, /// True while a journal snapshot exists on disk that no longer reflects /// an all-consumed pending set; the next idle tick removes it (MinIO /// deletes its `list.bin` after replay for the same reason). @@ -360,11 +372,6 @@ struct MrfRuntime { } impl MrfRuntime { - fn record_accept(&mut self) { - // Accepted intents leave the pending set; the next flush persists the - // smaller snapshot, which is the journal's compaction. - } - fn snapshot(&self) -> Vec { let mut buf = Vec::new(); for intent in self.queue.intents() { @@ -374,8 +381,14 @@ impl MrfRuntime { } async fn flush(&mut self) { - write_journal(&self.snapshot()).await; + let persisted = write_journal(&self.snapshot()).await; self.new_since_flush = 0; + // Keep the dirty flag when every disk write failed: a clean backlog + // would otherwise never rewrite, losing the periodic persist retry a + // non-empty queue used to provide. + if persisted { + self.dirty = false; + } self.journal_on_disk = true; } @@ -389,9 +402,15 @@ impl MrfRuntime { self.backoff_until = None; } while let Some(mut intent) = self.queue.pop_front() { + // Leaving the pending set (consumed or re-queued with a bumped + // attempts counter) changes the encoded snapshot; mark it dirty + // either way. + self.dirty = true; let request = build_heal_request(&intent); match manager.submit_heal_request(request).await { - Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => self.record_accept(), + // Accepted intents leave the pending set; the next flush persists the + // smaller snapshot, which is the journal's compaction. + Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {} Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => { intent.attempts = intent.attempts.saturating_add(1); if intent.attempts >= MRF_MAX_ATTEMPTS { @@ -519,6 +538,7 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes), config: config.clone(), new_since_flush: 0, + dirty: false, journal_on_disk: false, backoff_until: None, }; @@ -526,6 +546,10 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive // Replay: read the journal, re-arm intents (duplicates are merged by the // manager's dedup key), then drop the file so the next flush starts clean. replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await; + // The replay deleted the journal file; anything still pending (e.g. the + // manager was full and backoff armed) must be re-persisted by the next + // flush or a crash before it would lose those intents. + runtime.dirty = runtime.queue.depth() > 0; let mut flush_tick = tokio::time::interval(runtime.config.flush_interval); flush_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); @@ -535,8 +559,13 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive tokio::select! { received = receiver.recv_many(&mut batch, runtime.config.replay_batch) => { if received == 0 { - // Channel closed: flush once more and stop. - runtime.flush().await; + // Channel closed: flush once more unless the snapshot is + // provably current AND idle (a dirty or pending state + // gets one last persist attempt, matching the shutdown + // retry the unconditional flush used to provide). + if runtime.dirty || runtime.queue.depth() > 0 { + runtime.flush().await; + } tracing::info!( target: "rustfs::heal::mrf", "MRF channel closed; consumer stopped after final flush" @@ -544,8 +573,10 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive return; } for intent in batch.drain(..) { - runtime.queue.try_push(intent); - runtime.new_since_flush += 1; + if runtime.queue.try_push(intent) { + runtime.new_since_flush += 1; + runtime.dirty = true; + } } runtime.dispatch(manager.as_ref()).await; if runtime.new_since_flush >= runtime.config.flush_threshold { @@ -553,15 +584,26 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive } } _ = flush_tick.tick() => { - if runtime.new_since_flush > 0 || runtime.queue.depth() > 0 { - runtime.flush().await; - runtime.dispatch(manager.as_ref()).await; - } else if runtime.journal_on_disk { - // All intents consumed: remove the journal so a restart - // replays nothing (mirrors MinIO's post-replay unlink). - delete_journal().await; - runtime.journal_on_disk = false; - gauge!("rustfs_heal_mrf_journal_bytes").set(0.0); + match tick_action(runtime.dirty, runtime.queue.depth(), runtime.journal_on_disk) { + TickAction::Flush => { + runtime.flush().await; + runtime.dispatch(manager.as_ref()).await; + } + TickAction::Retry => { + // Pending set unchanged since the last flush (a + // backlog waiting out an admission backoff): skip the + // rewrite but keep dispatching so the retry fires on + // time. + runtime.dispatch(manager.as_ref()).await; + } + TickAction::DeleteJournal => { + // All intents consumed: remove the journal so a restart + // replays nothing (mirrors MinIO's post-replay unlink). + delete_journal().await; + runtime.journal_on_disk = false; + gauge!("rustfs_heal_mrf_journal_bytes").set(0.0); + } + TickAction::Idle => {} } gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64); } @@ -569,6 +611,33 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive } } +/// What the periodic tick should do, as a pure function of the runtime state +/// so the decision table is unit-testable. +enum TickAction { + /// The pending set changed since the last snapshot: rewrite it, then + /// drain. + Flush, + /// Pending intents exist but the snapshot is current: only drain (an + /// admission backoff may have expired). + Retry, + /// Nothing pending and a stale journal file remains: remove it. + DeleteJournal, + /// Quiescent: nothing to do. + Idle, +} + +fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool) -> TickAction { + if dirty { + TickAction::Flush + } else if depth > 0 { + TickAction::Retry + } else if journal_on_disk { + TickAction::DeleteJournal + } else { + TickAction::Idle + } +} + #[cfg(test)] mod tests { use super::*; @@ -586,6 +655,27 @@ mod tests { } } + #[test] + fn tick_action_table() { + use TickAction::*; + + // Dirty dominates: a changed pending set flushes even when idle + // otherwise. + assert!(matches!(tick_action(true, 0, false), Flush)); + assert!(matches!(tick_action(true, 3, true), Flush)); + + // Clean backlog: no rewrite, but keep draining so an expired + // admission backoff retries on time. + assert!(matches!(tick_action(false, 1, false), Retry)); + assert!(matches!(tick_action(false, 2, true), Retry)); + + // Quiescent with a stale journal file on disk: remove it. + assert!(matches!(tick_action(false, 0, true), DeleteJournal)); + + // Fully quiescent: nothing to do. + assert!(matches!(tick_action(false, 0, false), Idle)); + } + #[test] fn queue_enforces_count_and_byte_ceilings() { let mut queue = MrfQueue::new(2, usize::MAX); diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index fbe51f050..63ca002e4 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -27,7 +27,7 @@ use super::storage_api::storage::{ BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _, ObjectOperations as _, StorageAdminApi, }; -use super::{DiskStore, ECStore, Endpoint, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity}; +use super::{DiskStore, ECStore, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity}; pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader}; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -65,7 +65,6 @@ const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_STORAGE: &str = "storage"; const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io"; const EVENT_HEAL_STORAGE_OBJECT_READ_LIMIT: &str = "heal_storage_object_read_limit"; -const EVENT_HEAL_STORAGE_OBJECT_VERIFY: &str = "heal_storage_object_verify"; const EVENT_HEAL_STORAGE_ADMIN_OP: &str = "heal_storage_admin_op"; const EVENT_HEAL_STORAGE_REPAIR_OP: &str = "heal_storage_repair_op"; @@ -312,56 +311,23 @@ pub struct HealListItem { pub is_delete_marker: bool, } -/// Disk status for heal operations -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DiskStatus { - /// Ok - Ok, - /// Offline - Offline, - /// Corrupt - Corrupt, - /// Missing - Missing, - /// Permission denied - PermissionDenied, - /// Faulty - Faulty, - /// Root mount - RootMount, - /// Unknown - Unknown, - /// Unformatted - Unformatted, -} - /// Heal storage layer interface #[async_trait] pub trait HealStorageAPI: Send + Sync { /// Get object meta + /// + /// Reserved for HS-01 MRF wiring (rustfs/backlog#1865): MRF intents + /// currently execute through `heal_object`; keep this entry point for the + /// metadata-corruption variant that must inspect metadata first. async fn get_object_meta(&self, bucket: &str, object: &str) -> Result>; - /// Get object data - async fn get_object_data(&self, bucket: &str, object: &str) -> Result>>; - - /// Put object data - async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()>; - - /// Delete object - async fn delete_object(&self, bucket: &str, object: &str) -> Result<()>; - - /// Check object integrity - async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result; - /// EC decode rebuild + /// + /// Reserved for HS-01 MRF wiring (rustfs/backlog#1865): urgent ECDecode + /// requests currently execute through `heal_object`; keep the explicit + /// rebuild-and-read path for the decode-failure fast variant. async fn ec_decode_rebuild(&self, bucket: &str, object: &str) -> Result>; - /// Get disk status - async fn get_disk_status(&self, endpoint: &Endpoint) -> Result; - - /// Format disk - async fn format_disk(&self, endpoint: &Endpoint) -> Result<()>; - /// Get bucket info async fn get_bucket_info(&self, bucket: &str) -> Result>; @@ -387,21 +353,12 @@ pub trait HealStorageAPI: Send + Sync { Ok(false) } - /// Fix bucket metadata - async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()>; - /// Get all buckets async fn list_buckets(&self) -> Result>; /// Check object exists async fn object_exists(&self, bucket: &str, object: &str) -> Result; - /// Get object size - async fn get_object_size(&self, bucket: &str, object: &str) -> Result>; - - /// Get object checksum - async fn get_object_checksum(&self, bucket: &str, object: &str) -> Result>; - /// Heal object using ecstore async fn heal_object( &self, @@ -453,12 +410,6 @@ pub trait HealStorageAPI: Send + Sync { Ok(false) } - /// List object versions for healing (returns all versions, may use significant memory for large buckets) - /// - /// WARNING: This method loads all object versions into memory at once. For buckets with many - /// objects/versions, consider using `list_objects_for_heal_page` instead to process versions in pages. - async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result>; - /// List object versions for healing with pagination (returns one page and continuation token) /// Returns (versions, next_continuation_token, is_truncated). The continuation token is an /// opaque composite `(marker, version_marker)` value โ€” see `encode_heal_token`/`decode_heal_token`. @@ -527,89 +478,11 @@ impl ECStoreHealStorage { pub fn new(ecstore: Arc) -> Self { Self { ecstore } } -} - -fn is_transient_object_exists_message(message: &str) -> bool { - let message = message.to_ascii_lowercase(); - - [ - "failed to acquire read lock", - "lock acquisition failed", - "lock acquisition timeout", - "quorum not reached", - "deadline has elapsed", - "timed out", - "network error", - "transport error", - "connection refused", - ] - .iter() - .any(|pattern| message.contains(pattern)) -} - -fn is_transient_object_exists_error(err: &StorageError) -> bool { - if err.is_quorum_error() { - return true; - } - - match err { - StorageError::Lock(lock_err) => lock_err.is_retryable() || is_transient_object_exists_message(&lock_err.to_string()), - StorageError::Io(io_err) => is_transient_object_exists_message(&io_err.to_string()), - StorageError::SlowDown | StorageError::OperationCanceled => true, - _ => false, - } -} - -#[async_trait] -impl HealStorageAPI for ECStoreHealStorage { - async fn get_object_meta(&self, bucket: &str, object: &str) -> Result> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_object_meta", - bucket, - object, - "Heal storage request started" - ); - - match self.ecstore.get_object_info(bucket, object, &Default::default()).await { - Ok(info) => Ok(Some(info)), - Err(e) => { - // Map ObjectNotFound to None to align with Option return type - if matches!(e, StorageError::ObjectNotFound(_, _)) { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_object_meta", - bucket, - object, - result = "not_found", - "Heal storage object metadata missing" - ); - Ok(None) - } else { - error!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_object_meta", - bucket, - object, - result = "failed", - error = %e, - "Heal storage request failed" - ); - Err(Error::other(e)) - } - } - } - } + /// Read back an object's bytes, capped to bound memory. + /// + /// Private support for the reserved `ec_decode_rebuild` (HS-01); not part + /// of the storage trait surface. async fn get_object_data(&self, bucket: &str, object: &str) -> Result>> { debug!( target: "rustfs::heal::storage", @@ -695,196 +568,85 @@ impl HealStorageAPI for ECStoreHealStorage { } Ok(Some(buf)) } +} - async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()> { +fn is_transient_object_exists_message(message: &str) -> bool { + let message = message.to_ascii_lowercase(); + + [ + "failed to acquire read lock", + "lock acquisition failed", + "lock acquisition timeout", + "quorum not reached", + "deadline has elapsed", + "timed out", + "network error", + "transport error", + "connection refused", + ] + .iter() + .any(|pattern| message.contains(pattern)) +} + +fn is_transient_object_exists_error(err: &StorageError) -> bool { + if err.is_quorum_error() { + return true; + } + + match err { + StorageError::Lock(lock_err) => lock_err.is_retryable() || is_transient_object_exists_message(&lock_err.to_string()), + StorageError::Io(io_err) => is_transient_object_exists_message(&io_err.to_string()), + StorageError::SlowDown | StorageError::OperationCanceled => true, + _ => false, + } +} + +#[async_trait] +impl HealStorageAPI for ECStoreHealStorage { + async fn get_object_meta(&self, bucket: &str, object: &str) -> Result> { debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_OBJECT_IO, component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "put_object_data", - bucket, - object, - bytes = data.len(), - "Heal storage request started" - ); - - let mut reader = HealPutObjReader::from_vec(data.to_vec()); - match (*self.ecstore) - .put_object(bucket, object, &mut reader, &Default::default()) - .await - { - Ok(_) => { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "put_object_data", - bucket, - object, - result = "ok", - "Heal storage object write completed" - ); - Ok(()) - } - Err(e) => { - error!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "put_object_data", - bucket, - object, - result = "failed", - error = %e, - "Heal storage request failed" - ); - Err(Error::other(e)) - } - } - } - - async fn delete_object(&self, bucket: &str, object: &str) -> Result<()> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "delete_object", + operation = "get_object_meta", bucket, object, "Heal storage request started" ); - match self.ecstore.delete_object(bucket, object, Default::default()).await { - Ok(_) => { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "delete_object", - bucket, - object, - result = "ok", - "Heal storage object delete completed" - ); - Ok(()) - } + match self.ecstore.get_object_info(bucket, object, &Default::default()).await { + Ok(info) => Ok(Some(info)), Err(e) => { - error!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "delete_object", - bucket, - object, - result = "failed", - error = %e, - "Heal storage request failed" - ); - Err(Error::other(e)) - } - } - } - - async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - bucket, - object, - state = "started", - "Heal storage object verification started" - ); - - // Check object metadata first - match self.get_object_meta(bucket, object).await? { - Some(obj_info) => { - if obj_info.size < 0 { - warn!( + // Map ObjectNotFound to None to align with Option return type + if matches!(e, StorageError::ObjectNotFound(_, _)) { + debug!( target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, + event = EVENT_HEAL_STORAGE_OBJECT_IO, component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_STORAGE, + operation = "get_object_meta", bucket, object, - state = "invalid_size", - "Heal storage object verification failed" + result = "not_found", + "Heal storage object metadata missing" ); - return Ok(false); + Ok(None) + } else { + error!( + target: "rustfs::heal::storage", + event = EVENT_HEAL_STORAGE_OBJECT_IO, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_STORAGE, + operation = "get_object_meta", + bucket, + object, + result = "failed", + error = %e, + "Heal storage request failed" + ); + Err(Error::other(e)) } - - // Stream-read the object to a sink to avoid loading into memory - match (*self.ecstore) - .get_object_reader(bucket, object, None, Default::default(), &Default::default()) - .await - { - Ok(reader) => { - let mut stream = reader.stream; - match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await { - Ok(_) => { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - bucket, - object, - state = "ok", - "Heal storage object verified" - ); - Ok(true) - } - Err(e) => { - warn!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - bucket, - object, - state = "stream_read_failed", - error = %e, - "Heal storage object verification failed" - ); - Ok(false) - } - } - } - Err(e) => { - warn!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - bucket, - object, - state = "reader_open_failed", - error = %e, - "Heal storage object verification failed" - ); - Ok(false) - } - } - } - None => { - warn!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - bucket, - object, - state = "metadata_missing", - "Heal storage object verification failed" - ); - Ok(false) } } } @@ -976,81 +738,6 @@ impl HealStorageAPI for ECStoreHealStorage { } } - async fn get_disk_status(&self, endpoint: &Endpoint) -> Result { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_disk_status", - endpoint = ?endpoint, - state = "started", - "Heal storage admin operation started" - ); - - // TODO: implement disk status check using ecstore - // For now, return Ok status - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_disk_status", - endpoint = ?endpoint, - result = "ok", - disk_status = "ok", - "Heal storage disk status resolved" - ); - Ok(DiskStatus::Ok) - } - - async fn format_disk(&self, endpoint: &Endpoint) -> Result<()> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "format_disk", - endpoint = ?endpoint, - state = "started", - "Heal storage admin operation started" - ); - - // Use ecstore's heal_format - match self.heal_format(false).await { - Ok((_, error)) => { - if error.is_some() { - return Err(Error::other(format!("Format failed: {error:?}"))); - } - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "format_disk", - endpoint = ?endpoint, - result = "ok", - "Heal storage disk format completed" - ); - Ok(()) - } - Err(e) => { - error!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "format_disk", - endpoint = ?endpoint, - result = "failed", - error = %e, - "Heal storage admin operation failed" - ); - Err(e) - } - } - } - async fn get_bucket_info(&self, bucket: &str) -> Result> { debug!( target: "rustfs::heal::storage", @@ -1161,61 +848,6 @@ impl HealStorageAPI for ECStoreHealStorage { } } - async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_REPAIR_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "heal_bucket_metadata", - bucket, - state = "started", - "Heal storage repair started" - ); - - let heal_opts = HealOpts { - recursive: true, - dry_run: false, - remove: false, - recreate: false, - scan_mode: HealScanMode::Normal, - update_parity: false, - no_lock: false, - pool: None, - set: None, - }; - - match self.heal_bucket(bucket, &heal_opts).await { - Ok(_) => { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_REPAIR_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "heal_bucket_metadata", - bucket, - result = "ok", - "Heal storage bucket metadata repaired" - ); - Ok(()) - } - Err(e) => { - error!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_REPAIR_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "heal_bucket_metadata", - bucket, - result = "failed", - error = %e, - "Heal storage repair failed" - ); - Err(e) - } - } - } - async fn list_buckets(&self) -> Result> { debug!( target: "rustfs::heal::storage", @@ -1315,48 +947,6 @@ impl HealStorageAPI for ECStoreHealStorage { } } - async fn get_object_size(&self, bucket: &str, object: &str) -> Result> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_object_size", - bucket, - object, - "Heal storage request started" - ); - - match self.get_object_meta(bucket, object).await { - Ok(Some(obj_info)) => Ok(Some(obj_info.size as u64)), - Ok(None) => Ok(None), - Err(e) => Err(e), - } - } - - async fn get_object_checksum(&self, bucket: &str, object: &str) -> Result> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_object_checksum", - bucket, - object, - "Heal storage request started" - ); - - match self.get_object_meta(bucket, object).await { - Ok(Some(obj_info)) => { - // Convert checksum bytes to hex string - let checksum = obj_info.checksum.iter().map(|b| format!("{b:02x}")).collect::(); - Ok(Some(checksum)) - } - Ok(None) => Ok(None), - Err(e) => Err(e), - } - } - async fn heal_object( &self, bucket: &str, @@ -1547,65 +1137,6 @@ impl HealStorageAPI for ECStoreHealStorage { .map_err(Error::Storage) } - async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "list_objects_for_heal", - bucket, - prefix, - state = "started", - "Heal storage admin operation started" - ); - warn!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "list_objects_for_heal", - bucket, - prefix, - state = "memory_heavy", - "Heal storage version listing loads all versions into memory (footprint is per-version, not per-object)" - ); - - let mut all_objects: Vec = Vec::new(); - let mut continuation_token: Option = None; - - loop { - let (page_objects, next_token, is_truncated) = self - .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false) - .await?; - - all_objects.extend(page_objects); - - if !is_truncated { - break; - } - - continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?; - if continuation_token.is_none() { - break; - } - } - - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "list_objects_for_heal", - bucket, - prefix, - object_count = all_objects.len(), - result = "ok", - "Heal storage object listing completed" - ); - Ok(all_objects) - } - async fn list_objects_for_heal_page( &self, bucket: &str, diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index 2c0626d8f..4db62f271 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -54,7 +54,7 @@ const MAX_BUCKET_OBJECT_HEAL_RETRIES: u32 = 3; const MAX_BUCKET_FAILURE_LOG_SAMPLES: u64 = 5; /// Emits at `$level`, demoted to `debug!` when `$demote` is true. Keeps -/// per-object heal work โ€” Object/Metadata/MRF/ECDecode tasks queued per +/// per-object heal work โ€” Object/Metadata/ECDecode tasks queued per /// object by MRF/autoheal/scanner loops, and per-object sweep failures past /// a sample cap โ€” from amplifying into one info!/warn!/error! line per /// object during mass recovery (rustfs/rustfs#5716). Aggregate task kinds @@ -75,8 +75,6 @@ const EVENT_HEAL_BUCKET_STAGE: &str = "heal_bucket_stage"; const EVENT_HEAL_BUCKET_RESULT: &str = "heal_bucket_result"; const EVENT_HEAL_METADATA_STAGE: &str = "heal_metadata_stage"; const EVENT_HEAL_METADATA_RESULT: &str = "heal_metadata_result"; -const EVENT_HEAL_MRF_STAGE: &str = "heal_mrf_stage"; -const EVENT_HEAL_MRF_RESULT: &str = "heal_mrf_result"; const EVENT_HEAL_EC_DECODE_STAGE: &str = "heal_ec_decode_stage"; const EVENT_HEAL_EC_DECODE_RESULT: &str = "heal_ec_decode_result"; const EVENT_HEAL_ERASURE_SET_STAGE: &str = "heal_erasure_set_stage"; @@ -101,8 +99,6 @@ pub enum HealType { ErasureSet { buckets: Vec, set_disk_id: String }, /// Metadata heal Metadata { bucket: String, object: String }, - /// MRF heal - MRF { meta_path: String }, /// EC decode heal ECDecode { bucket: String, @@ -120,21 +116,18 @@ impl HealType { Self::Prefix { .. } => "prefix", Self::ErasureSet { .. } => "erasure_set", Self::Metadata { .. } => "metadata", - Self::MRF { .. } => "mrf", Self::ECDecode { .. } => "ec_decode", } } /// Task kinds enqueued at per-object granularity (MRF, autoheal, scanner, - /// read-repair loops). Their lifecycle and admission logs stay at `debug!` + /// read-repair loops; the MRF loop queues Object/ECDecode/Metadata + /// tasks). Their lifecycle and admission logs stay at `debug!` /// so a recovery loop queuing hundreds of thousands of object heal tasks /// cannot amplify into per-object `info!`/`warn!` lines; aggregate kinds /// (cluster/bucket/prefix/erasure-set) keep operator-visible levels. pub(crate) fn is_per_object(&self) -> bool { - matches!( - self, - Self::Object { .. } | Self::Metadata { .. } | Self::MRF { .. } | Self::ECDecode { .. } - ) + matches!(self, Self::Object { .. } | Self::Metadata { .. } | Self::ECDecode { .. }) } } @@ -504,7 +497,6 @@ impl HealTask { HealType::Prefix { .. } => "prefix", HealType::ErasureSet { .. } => "erasure_set", HealType::Metadata { .. } => "metadata", - HealType::MRF { .. } => "mrf", HealType::ECDecode { .. } => "ec_decode", } } @@ -579,7 +571,6 @@ impl HealTask { None => event, } } - HealType::MRF { meta_path } => event.with_object(meta_path.as_str()), }; match error { @@ -821,7 +812,6 @@ impl HealTask { HealType::Prefix { bucket, prefix } => self.heal_prefix(bucket, prefix).await, HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await, - HealType::MRF { meta_path } => self.heal_mrf(meta_path).await, HealType::ECDecode { bucket, object, @@ -2020,139 +2010,6 @@ impl HealTask { } } - async fn heal_mrf(&self, meta_path: &str) -> Result<()> { - debug!( - target: "rustfs::heal::task", - event = EVENT_HEAL_MRF_STAGE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - meta_path, - stage = "start", - "Heal MRF started" - ); - - // update progress - { - let mut progress = self.progress.write().await; - progress.set_current_object(Some(format!("mrf: {meta_path}"))); - progress.update_progress(0, 2, 0, 0); - } - - // Parse meta_path to extract bucket and object - let parts: Vec<&str> = meta_path.split('/').collect(); - if parts.len() < 2 { - return Err(Error::TaskExecutionFailed { - message: format!("Invalid meta path format: {meta_path}"), - }); - } - - let bucket = parts[0]; - let object = parts[1..].join("/"); - - // Step 1: Perform MRF heal using ecstore - debug!( - target: "rustfs::heal::task", - event = EVENT_HEAL_MRF_STAGE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - meta_path, - bucket, - object = %object, - stage = "heal_with_ecstore", - "Heal MRF stage entered" - ); - let heal_opts = HealOpts { - recursive: true, - dry_run: self.options.dry_run, - remove: self.options.remove_corrupted, - recreate: self.options.recreate_missing, - scan_mode: HealScanMode::Deep, - update_parity: true, - no_lock: self.options.no_lock, - pool: None, - set: None, - }; - - let heal_result = self - .await_with_control(self.storage.heal_object(bucket, &object, None, &heal_opts)) - .await; - - match heal_result { - Ok((result, error)) => { - if let Some(e) = error { - error!( - target: "rustfs::heal::task", - event = EVENT_HEAL_MRF_RESULT, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - meta_path, - bucket, - object = %object, - result = "failed", - error = %e, - "Heal MRF failed" - ); - { - let mut progress = self.progress.write().await; - progress.update_progress(2, 2, 0, 0); - } - return Err(Error::TaskExecutionFailed { - message: format!("Failed to heal MRF {meta_path}: {e}"), - }); - } - - debug!( - target: "rustfs::heal::task", - event = EVENT_HEAL_MRF_RESULT, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - meta_path, - bucket, - object = %object, - drives_healed = result.drives_healed(), - drives_total = result.drives_reported(), - result = "ok", - "Heal MRF repaired" - ); - - { - let mut progress = self.progress.write().await; - progress.update_progress(2, 2, 0, 0); - } - self.record_result_item(result).await; - Ok(()) - } - Err(Error::TaskCancelled) => Err(Error::TaskCancelled), - Err(Error::TaskTimeout) => Err(Error::TaskTimeout), - Err(e) => { - error!( - target: "rustfs::heal::task", - event = EVENT_HEAL_MRF_RESULT, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - meta_path, - bucket, - object = %object, - result = "failed", - error = %e, - "Heal MRF failed" - ); - { - let mut progress = self.progress.write().await; - progress.update_progress(2, 2, 0, 0); - } - Err(Error::TaskExecutionFailed { - message: format!("Failed to heal MRF {meta_path}: {e}"), - }) - } - } - } - async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { debug!( target: "rustfs::heal::task", @@ -2822,7 +2679,7 @@ impl std::fmt::Debug for HealTask { mod tests { use super::super::{DiskOption, DiskStore, Endpoint, HealDiskExt as _, new_disk}; use super::*; - use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo}; + use crate::heal::storage::{HealListItem, HealObjectInfo}; use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events}; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; use std::collections::{HashMap, VecDeque}; @@ -3354,7 +3211,6 @@ mod tests { object_exists_by_name: Mutex>, heal_object_outcome: Mutex>, heal_object_outcomes: Mutex>>, - deleted_objects: Mutex>, format_no_heal_required: Mutex, global_format_calls: Mutex, replacement_format_calls: Mutex)>>, @@ -3390,12 +3246,6 @@ mod tests { } .is_per_object() ); - assert!( - HealType::MRF { - meta_path: "p".to_string(), - } - .is_per_object() - ); assert!( HealType::ECDecode { bucket: "b".to_string(), @@ -3547,35 +3397,10 @@ mod tests { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> Result>> { - Ok(None) - } - - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> Result<()> { - Ok(()) - } - - async fn delete_object(&self, _bucket: &str, object: &str) -> Result<()> { - self.deleted_objects.lock().unwrap().push(object.to_string()); - Ok(()) - } - - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> Result { - Ok(true) - } - async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result> { Ok(Vec::new()) } - async fn get_disk_status(&self, _endpoint: &Endpoint) -> Result { - Ok(DiskStatus::Ok) - } - - async fn format_disk(&self, _endpoint: &Endpoint) -> Result<()> { - Ok(()) - } - async fn get_bucket_info(&self, bucket: &str) -> Result> { Ok(Some(BucketInfo { name: bucket.to_string(), @@ -3590,10 +3415,6 @@ mod tests { Ok(*self.usage_baseline.lock().unwrap()) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> { - Ok(()) - } - async fn list_buckets(&self) -> Result> { let buckets = self .listed_buckets @@ -3621,14 +3442,6 @@ mod tests { Ok(self.object_exists.lock().unwrap().unwrap_or(true)) } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> Result> { - Ok(None) - } - - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> Result> { - Ok(None) - } - async fn heal_object( &self, bucket: &str, @@ -3764,10 +3577,6 @@ mod tests { Ok(*self.replacement_targets_ready.lock().unwrap()) } - async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> Result> { - Ok(vec![heal_item("object-a"), heal_item("object-b")]) - } - async fn list_objects_for_heal_page( &self, bucket: &str, @@ -4858,7 +4667,7 @@ mod tests { } #[tokio::test] - async fn test_heal_failure_with_remove_corrupted_does_not_delete_object() { + async fn test_heal_failure_with_remove_corrupted_propagates_remove_flag() { let storage = Arc::new(MockStorage { object_exists: Mutex::new(Some(true)), heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::OkWithOtherError( @@ -4884,7 +4693,6 @@ mod tests { let err = task.execute().await.expect_err("heal failure should still be reported"); assert!(matches!(err, Error::TaskExecutionFailed { .. })); - assert!(storage.deleted_objects.lock().unwrap().is_empty()); assert!(storage.object_heal_opts.lock().unwrap()[0].remove); } diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index f1ec4cebe..29444d0f5 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -352,8 +352,8 @@ pub(crate) fn set_heal_queue_length(count: usize) { mod tests { use super::{ Error, HEAL_RUNTIME_INIT_TEST_HOOK, HealRuntimeInitTestHook, get_heal_channel_processor, get_heal_manager, - heal::DiskStore, heal::Endpoint, heal::manager::HealConfig, heal::storage::DiskStatus, heal::storage::HealListItem, - heal::storage::HealObjectInfo, heal::storage::HealStorageAPI, init_heal_manager, run_owned_initialization, + heal::DiskStore, heal::manager::HealConfig, heal::storage::HealListItem, heal::storage::HealObjectInfo, + heal::storage::HealStorageAPI, init_heal_manager, run_owned_initialization, }; use crate::heal::storage_api::status::BucketInfo; use rustfs_common::heal_channel::HealOpts; @@ -370,42 +370,14 @@ mod tests { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> Result>, Error> { - Ok(None) - } - - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> Result<(), Error> { - Ok(()) - } - - async fn delete_object(&self, _bucket: &str, _object: &str) -> Result<(), Error> { - Ok(()) - } - - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> Result { - Ok(true) - } - async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result, Error> { Ok(Vec::new()) } - async fn get_disk_status(&self, _endpoint: &Endpoint) -> Result { - Ok(DiskStatus::Ok) - } - - async fn format_disk(&self, _endpoint: &Endpoint) -> Result<(), Error> { - Ok(()) - } - async fn get_bucket_info(&self, _bucket: &str) -> Result, Error> { Ok(None) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<(), Error> { - Ok(()) - } - async fn list_buckets(&self) -> Result, Error> { Ok(Vec::new()) } @@ -414,14 +386,6 @@ mod tests { Ok(false) } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> Result, Error> { - Ok(None) - } - - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> Result, Error> { - Ok(None) - } - async fn heal_object( &self, _bucket: &str, @@ -440,10 +404,6 @@ mod tests { Ok((HealResultItem::default(), None)) } - async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> Result, Error> { - Ok(Vec::new()) - } - async fn list_objects_for_heal_page( &self, _bucket: &str, diff --git a/crates/heal/tests/heal_bug_fixes_test.rs b/crates/heal/tests/heal_bug_fixes_test.rs index ba59142fd..59ff1bb3f 100644 --- a/crates/heal/tests/heal_bug_fixes_test.rs +++ b/crates/heal/tests/heal_bug_fixes_test.rs @@ -13,93 +13,13 @@ // limitations under the License. use rustfs_heal::heal::{ - event::{HealEvent, Severity}, task::{HealPriority, HealType}, utils, }; mod storage_api; -use storage_api::bug_fixes::{BucketInfo, DiskStore, Endpoint}; - -#[test] -fn test_heal_event_to_heal_request_no_panic() { - // Test that invalid pool/set indices don't cause panic - // Create endpoint using try_from or similar method - let endpoint_result = Endpoint::try_from("http://localhost:9000"); - if let Ok(mut endpoint) = endpoint_result { - endpoint.pool_idx = -1; - endpoint.set_idx = -1; - endpoint.disk_idx = 0; - - let event = HealEvent::DiskStatusChange { - endpoint, - old_status: "ok".to_string(), - new_status: "offline".to_string(), - }; - - // Should return error instead of panicking - let result = event.to_heal_request(); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("Invalid heal type")); - } -} - -#[test] -fn test_heal_event_to_heal_request_valid_indices() { - // Test that valid indices work correctly - let endpoint_result = Endpoint::try_from("http://localhost:9000"); - if let Ok(mut endpoint) = endpoint_result { - endpoint.pool_idx = 0; - endpoint.set_idx = 1; - endpoint.disk_idx = 0; - - let event = HealEvent::DiskStatusChange { - endpoint, - old_status: "ok".to_string(), - new_status: "offline".to_string(), - }; - - let result = event.to_heal_request(); - assert!(result.is_ok()); - let request = result.unwrap(); - assert!(matches!(request.heal_type, HealType::ErasureSet { .. })); - } -} - -#[test] -fn test_heal_event_object_corruption() { - let event = HealEvent::ObjectCorruption { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: None, - corruption_type: rustfs_heal::heal::event::CorruptionType::DataCorruption, - severity: Severity::High, - }; - - let result = event.to_heal_request(); - assert!(result.is_ok()); - let request = result.unwrap(); - assert!(matches!(request.heal_type, HealType::Object { .. })); - assert_eq!(request.priority, HealPriority::High); -} - -#[test] -fn test_heal_event_ec_decode_failure() { - let event = HealEvent::ECDecodeFailure { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: None, - missing_shards: vec![0, 1], - available_shards: vec![2, 3], - }; - - let result = event.to_heal_request(); - assert!(result.is_ok()); - let request = result.unwrap(); - assert!(matches!(request.heal_type, HealType::ECDecode { .. })); - assert_eq!(request.priority, HealPriority::Urgent); -} +use storage_api::bug_fixes::{BucketInfo, DiskStore}; #[test] fn test_format_set_disk_id_from_i32_negative() { @@ -117,13 +37,16 @@ fn test_format_set_disk_id_from_i32_valid() { assert_eq!(result.unwrap(), "pool_0_set_1"); } +/// A wall-clock lower bound for "the timestamp was actually read from the +/// clock": 2020-01-01. `unwrap_or_default()` on a pre-epoch clock yields 0, and +/// the old versions of these tests bound the fields to `_` and so could not tell +/// that apart from a real reading (rustfs/backlog#1836). +const SANE_EPOCH_SECS: u64 = 1_577_836_800; + #[test] fn test_resume_state_timestamp_handling() { use rustfs_heal::heal::resume::ResumeState; - // Test that ResumeState creation doesn't panic even if system time is before epoch - // This is a theoretical test - in practice, system time should never be before epoch - // But we want to ensure unwrap_or_default handles edge cases let state = ResumeState::new( "test-task".to_string(), "test-type".to_string(), @@ -131,22 +54,30 @@ fn test_resume_state_timestamp_handling() { vec!["bucket1".to_string()], ); - // Verify fields are initialized (u64 is always >= 0) - // The important thing is that unwrap_or_default prevents panic - let _ = state.start_time; - let _ = state.last_update; + assert!( + state.start_time > SANE_EPOCH_SECS, + "start_time fell back to the default instead of reading the clock: {}", + state.start_time + ); + assert!( + state.last_update >= state.start_time, + "last_update {} must not predate start_time {}", + state.last_update, + state.start_time + ); } #[test] fn test_resume_checkpoint_timestamp_handling() { use rustfs_heal::heal::resume::ResumeCheckpoint; - // Test that ResumeCheckpoint creation doesn't panic let checkpoint = ResumeCheckpoint::new("test-task".to_string()); - // Verify field is initialized (u64 is always >= 0) - // The important thing is that unwrap_or_default prevents panic - let _ = checkpoint.checkpoint_time; + assert!( + checkpoint.checkpoint_time > SANE_EPOCH_SECS, + "checkpoint_time fell back to the default instead of reading the clock: {}", + checkpoint.checkpoint_time + ); } #[test] @@ -173,45 +104,18 @@ fn test_heal_task_status_atomic_update() { async fn get_object_meta(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result>> { - Ok(None) - } - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> rustfs_heal::Result<()> { - Ok(()) - } - async fn delete_object(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<()> { - Ok(()) - } - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result { - Ok(true) - } async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { Ok(vec![]) } - async fn get_disk_status(&self, _endpoint: &Endpoint) -> rustfs_heal::Result { - Ok(rustfs_heal::heal::storage::DiskStatus::Ok) - } - async fn format_disk(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<()> { - Ok(()) - } async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result> { Ok(None) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> { - Ok(()) - } async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(vec![]) } async fn object_exists(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result { Ok(false) } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { - Ok(None) - } - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { - Ok(None) - } async fn heal_object( &self, _bucket: &str, @@ -234,9 +138,6 @@ fn test_heal_task_status_atomic_update() { ) -> rustfs_heal::Result<(rustfs_madmin::heal_commands::HealResultItem, Option)> { Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None)) } - async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> rustfs_heal::Result> { - Ok(vec![]) - } async fn list_objects_for_heal_page( &self, _bucket: &str, @@ -278,7 +179,7 @@ fn test_heal_task_status_atomic_update() { #[tokio::test] async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { - use rustfs_heal::heal::storage::{DiskStatus, HealListItem, HealObjectInfo, HealStorageAPI}; + use rustfs_heal::heal::storage::{HealListItem, HealObjectInfo, HealStorageAPI}; use rustfs_heal::heal::task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType}; use std::sync::{ Arc, @@ -296,42 +197,14 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result>> { - Ok(None) - } - - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> rustfs_heal::Result<()> { - Ok(()) - } - - async fn delete_object(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<()> { - Ok(()) - } - - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result { - Ok(true) - } - async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { Ok(Vec::new()) } - async fn get_disk_status(&self, _endpoint: &Endpoint) -> rustfs_heal::Result { - Ok(DiskStatus::Ok) - } - - async fn format_disk(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<()> { - Ok(()) - } - async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result> { Ok(None) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> { - Ok(()) - } - async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(Vec::new()) } @@ -343,14 +216,6 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { )) } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { - Ok(None) - } - - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { - Ok(None) - } - async fn heal_object( &self, _bucket: &str, @@ -377,10 +242,6 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None)) } - async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> rustfs_heal::Result> { - Ok(Vec::new()) - } - async fn list_objects_for_heal_page( &self, _bucket: &str, diff --git a/crates/io-core/src/io_profile.rs b/crates/io-core/src/io_profile.rs index 86cd79448..46b894c49 100644 --- a/crates/io-core/src/io_profile.rs +++ b/crates/io-core/src/io_profile.rs @@ -436,30 +436,45 @@ mod tests { assert_eq!(unknown_profile.sequential_boost_multiplier, 1.0); } - #[cfg(target_os = "linux")] + // What platform probing returns depends on the machine, so these pin the two + // rules that do not: the override wins over probing, and probing that is + // switched off reports Unknown rather than guessing (rustfs/backlog#1836). #[test] - fn test_linux_storage_detection_exists() { - // This test just verifies the detection function exists and doesn't panic - // The actual result depends on the system it's running on - let result = detect_storage_media(true, ""); - // We should get some result (not panic) - match result { - StorageMedia::Nvme | StorageMedia::Ssd | StorageMedia::Hdd | StorageMedia::Unknown => { - // All valid results - } + fn storage_media_override_wins_over_platform_detection() { + for (override_value, expected) in [ + ("nvme", StorageMedia::Nvme), + ("ssd", StorageMedia::Ssd), + ("hdd", StorageMedia::Hdd), + ] { + assert_eq!(detect_storage_media(true, override_value), expected); + assert_eq!( + detect_storage_media(false, override_value), + expected, + "an override must be honoured even with detection disabled" + ); } } - #[cfg(target_os = "macos")] #[test] - fn test_macos_storage_detection_exists() { - // This test just verifies the detection function exists and doesn't panic - let result = detect_storage_media(true, ""); - // We should get some result (not panic) - match result { - StorageMedia::Nvme | StorageMedia::Ssd | StorageMedia::Hdd | StorageMedia::Unknown => { - // All valid results - } - } + fn disabled_detection_reports_unknown_instead_of_guessing() { + assert_eq!(detect_storage_media(false, ""), StorageMedia::Unknown); + assert_eq!( + detect_storage_media(false, "not-a-medium"), + StorageMedia::Unknown, + "an unparseable override falls through to the disabled path" + ); + } + + #[test] + fn enabled_detection_returns_a_medium_for_this_platform() { + // Whatever this machine reports, it must be one of the known variants and + // it must be stable across calls โ€” a probe that flapped would make the + // scheduler's profile depend on when it asked. + let first = detect_storage_media(true, ""); + assert!(matches!( + first, + StorageMedia::Nvme | StorageMedia::Ssd | StorageMedia::Hdd | StorageMedia::Unknown + )); + assert_eq!(detect_storage_media(true, ""), first); } } diff --git a/crates/io-metrics/Cargo.toml b/crates/io-metrics/Cargo.toml index 6124de8ca..0591aa4e7 100644 --- a/crates/io-metrics/Cargo.toml +++ b/crates/io-metrics/Cargo.toml @@ -58,7 +58,7 @@ sysinfo = { workspace = true } [dev-dependencies] criterion = { workspace = true, features = ["html_reports"] } -metrics-util = { version = "0.20", features = ["debugging"] } +metrics-util = { workspace = true, features = ["debugging"] } tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] } [lints] diff --git a/crates/io-metrics/src/internode_metrics.rs b/crates/io-metrics/src/internode_metrics.rs index a12a8edfe..fb12a9911 100644 --- a/crates/io-metrics/src/internode_metrics.rs +++ b/crates/io-metrics/src/internode_metrics.rs @@ -47,6 +47,13 @@ pub const INTERNODE_MSGPACK_DIRECTION_REQUEST: &str = "request"; pub const INTERNODE_MSGPACK_DIRECTION_RESPONSE: &str = "response"; pub const INTERNODE_MSGPACK_CODEC_MSGPACK: &str = "msgpack"; pub const INTERNODE_MSGPACK_CODEC_JSON: &str = "json"; +pub const INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE: &str = "read_version_request_encode"; +pub const INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE: &str = "read_version_request_decode"; +pub const INTERNODE_STAGE_READ_VERSION_DISK_READ: &str = "read_version_disk_read"; +pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "read_version_response_json_encode"; +pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "read_version_response_msgpack_encode"; +pub const INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP: &str = "read_version_rpc_roundtrip"; +pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE: &str = "read_version_response_decode"; const OPERATION_LABEL: &str = "operation"; const BACKEND_LABEL: &str = "backend"; @@ -67,6 +74,7 @@ const INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network const INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_incoming_total"; const INTERNODE_OPERATION_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_errors_total"; const INTERNODE_OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_operation_duration_ms"; +const INTERNODE_OPERATION_STAGE_DURATION_MS: &str = "rustfs_system_network_internode_operation_stage_duration_ms"; const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total"; const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total"; const INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL: &str = "rustfs_system_network_internode_operation_retry_successes_total"; @@ -105,6 +113,7 @@ const SERVER_OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[SERVER_LABEL, OP const SERVER_OPERATION_BACKEND_FAILURE_REASON_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL]; const SERVER_OPERATION_BACKEND_RPC_PATH_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL]; +const SERVER_OPERATION_BACKEND_STAGE_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, STAGE_LABEL]; const SERVER_LABELS: &[&str] = &[SERVER_LABEL]; const SERVER_REASON_LABELS: &[&str] = &[SERVER_LABEL, REASON_LABEL]; const SERVER_QUORUM_FAILURE_LABELS: &[&str] = &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]; @@ -134,6 +143,10 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = & name: INTERNODE_OPERATION_DURATION_MS, labels: SERVER_OPERATION_BACKEND_LABELS, }, + InternodeOperationMetricDescriptor { + name: INTERNODE_OPERATION_STAGE_DURATION_MS, + labels: SERVER_OPERATION_BACKEND_STAGE_LABELS, + }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL, labels: SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS, @@ -394,6 +407,24 @@ impl InternodeMetrics { .record(duration_ms); } + pub fn record_stage_duration_for_operation_and_backend( + &self, + operation: &'static str, + backend: &'static str, + stage: &'static str, + duration: Duration, + ) { + let duration_ms = duration.as_secs_f64() * 1000.0; + metrics::histogram!( + INTERNODE_OPERATION_STAGE_DURATION_MS, + SERVER_LABEL => current_server_label(), + OPERATION_LABEL => operation, + BACKEND_LABEL => backend, + STAGE_LABEL => stage + ) + .record(duration_ms); + } + pub fn record_classified_error_for_operation_and_backend( &self, operation: &'static str, @@ -988,42 +1019,90 @@ mod tests { assert_eq!(snapshot.replay_cache_evictions_total, 3); } + #[test] + fn operation_stage_duration_records_low_cardinality_stage_labels() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let metrics = InternodeMetrics::default(); + + with_local_recorder(&recorder, || { + metrics.record_stage_duration_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, + Duration::from_micros(125), + ); + }); + + let entries: Vec<_> = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter(|(composite, _, _, _)| composite.key().name() == INTERNODE_OPERATION_STAGE_DURATION_MS) + .collect(); + assert_eq!(entries.len(), 1); + let labels: HashMap<_, _> = entries[0] + .0 + .key() + .labels() + .map(|label| (label.key().to_string(), label.value().to_string())) + .collect(); + assert_eq!( + labels.get(OPERATION_LABEL).map(String::as_str), + Some(INTERNODE_OPERATION_GRPC_READ_VERSION) + ); + assert_eq!(labels.get(BACKEND_LABEL).map(String::as_str), Some(INTERNODE_TRANSPORT_BACKEND_GRPC)); + assert_eq!( + labels.get(STAGE_LABEL).map(String::as_str), + Some(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP) + ); + assert!(labels.get(SERVER_LABEL).is_some_and(|value| !value.is_empty())); + match &entries[0].3 { + DebugValue::Histogram(samples) => assert_eq!(samples.iter().map(|sample| sample.0).collect::>(), vec![0.125]), + other => panic!("{INTERNODE_OPERATION_STAGE_DURATION_MS} must be a histogram, got {other:?}"), + } + } + #[test] fn operation_metric_descriptors_include_backend_and_operation_labels() { - assert_eq!(INTERNODE_OPERATION_METRICS.len(), 21); + assert_eq!(INTERNODE_OPERATION_METRICS.len(), 22); for metric in &INTERNODE_OPERATION_METRICS[..6] { assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); } - for metric in &INTERNODE_OPERATION_METRICS[6..9] { + assert_eq!( + INTERNODE_OPERATION_METRICS[6].labels, + &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, STAGE_LABEL] + ); + for metric in &INTERNODE_OPERATION_METRICS[7..10] { assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]); } assert_eq!( - INTERNODE_OPERATION_METRICS[9].labels, + INTERNODE_OPERATION_METRICS[10].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL] ); - for metric in &INTERNODE_OPERATION_METRICS[10..12] { + for metric in &INTERNODE_OPERATION_METRICS[11..13] { assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); } - assert_eq!( - INTERNODE_OPERATION_METRICS[12].labels, - &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL] - ); assert_eq!( INTERNODE_OPERATION_METRICS[13].labels, - &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL] + &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL] ); assert_eq!( INTERNODE_OPERATION_METRICS[14].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL] ); - for metric in &INTERNODE_OPERATION_METRICS[15..17] { + assert_eq!( + INTERNODE_OPERATION_METRICS[15].labels, + &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL] + ); + for metric in &INTERNODE_OPERATION_METRICS[16..18] { assert_eq!(metric.labels, &[SERVER_LABEL]); } - assert_eq!(INTERNODE_OPERATION_METRICS[17].labels, &[SERVER_LABEL, REASON_LABEL]); - assert_eq!(INTERNODE_OPERATION_METRICS[18].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]); + assert_eq!(INTERNODE_OPERATION_METRICS[18].labels, &[SERVER_LABEL, REASON_LABEL]); + assert_eq!(INTERNODE_OPERATION_METRICS[19].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]); // Payload histogram + large-payload counter carry operation+backend labels. - assert_eq!(INTERNODE_OPERATION_METRICS[19].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); assert_eq!(INTERNODE_OPERATION_METRICS[20].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); + assert_eq!(INTERNODE_OPERATION_METRICS[21].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); } #[test] @@ -1054,62 +1133,66 @@ mod tests { ); assert_eq!( INTERNODE_OPERATION_METRICS[6].name, - "rustfs_system_network_internode_operation_classified_errors_total" + "rustfs_system_network_internode_operation_stage_duration_ms" ); assert_eq!( INTERNODE_OPERATION_METRICS[7].name, - "rustfs_system_network_internode_operation_retries_total" + "rustfs_system_network_internode_operation_classified_errors_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[8].name, - "rustfs_system_network_internode_operation_retry_successes_total" + "rustfs_system_network_internode_operation_retries_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[9].name, - "rustfs_system_network_internode_operation_http_versions_total" + "rustfs_system_network_internode_operation_retry_successes_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[10].name, - "rustfs_system_network_internode_operation_stall_timeouts_total" + "rustfs_system_network_internode_operation_http_versions_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[11].name, - "rustfs_system_network_internode_operation_write_shutdown_errors_total" + "rustfs_system_network_internode_operation_stall_timeouts_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[12].name, - "rustfs_system_network_internode_rpc_auth_failures_total" + "rustfs_system_network_internode_operation_write_shutdown_errors_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[13].name, - "rustfs_system_network_internode_replay_cache_overflow_by_operation_total" + "rustfs_system_network_internode_rpc_auth_failures_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[14].name, - "rustfs_system_network_internode_replay_cache_records_total" + "rustfs_system_network_internode_replay_cache_overflow_by_operation_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[15].name, - "rustfs_system_network_internode_replay_cache_entries" + "rustfs_system_network_internode_replay_cache_records_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[16].name, - "rustfs_system_network_internode_replay_cache_capacity" + "rustfs_system_network_internode_replay_cache_entries" ); assert_eq!( INTERNODE_OPERATION_METRICS[17].name, - "rustfs_system_network_internode_replay_cache_evictions_total" + "rustfs_system_network_internode_replay_cache_capacity" ); assert_eq!( INTERNODE_OPERATION_METRICS[18].name, - "rustfs_system_storage_erasure_write_quorum_failures_total" + "rustfs_system_network_internode_replay_cache_evictions_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[19].name, - "rustfs_system_network_internode_operation_payload_bytes" + "rustfs_system_storage_erasure_write_quorum_failures_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[20].name, + "rustfs_system_network_internode_operation_payload_bytes" + ); + assert_eq!( + INTERNODE_OPERATION_METRICS[21].name, "rustfs_system_network_internode_operation_large_payloads_total" ); assert_eq!(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, "grpc_read_multiple"); @@ -1129,6 +1212,16 @@ mod tests { assert_eq!(INTERNODE_MSGPACK_DIRECTION_RESPONSE, "response"); assert_eq!(INTERNODE_MSGPACK_CODEC_MSGPACK, "msgpack"); assert_eq!(INTERNODE_MSGPACK_CODEC_JSON, "json"); + assert_eq!(INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, "read_version_request_encode"); + assert_eq!(INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, "read_version_request_decode"); + assert_eq!(INTERNODE_STAGE_READ_VERSION_DISK_READ, "read_version_disk_read"); + assert_eq!(INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE, "read_version_response_json_encode"); + assert_eq!( + INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE, + "read_version_response_msgpack_encode" + ); + assert_eq!(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, "read_version_rpc_roundtrip"); + assert_eq!(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, "read_version_response_decode"); assert_eq!( INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL, "rustfs_system_network_internode_signature_v1_fallback_total" diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index 785cea246..50a660383 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -120,6 +120,14 @@ pub const PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC: &str = "set_disk_rename_ba pub const PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC: &str = "set_disk_rename_ancestor_dir_fsync"; pub const PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL: &str = "set_disk_rename_rename_syscall"; +pub const PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL: &str = "serial"; +pub const PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL: &str = "parallel"; +pub const PUT_RENAME_QUORUM_FANOUT_STATE_SCHEDULED: &str = "scheduled"; +pub const PUT_RENAME_QUORUM_FANOUT_STATE_WRITE_QUORUM: &str = "write_quorum"; +pub const PUT_RENAME_QUORUM_FANOUT_STATE_SUCCESS: &str = "success"; +pub const PUT_RENAME_QUORUM_FANOUT_STATE_ERROR: &str = "error"; +pub const PUT_RENAME_QUORUM_FANOUT_STATE_PANIC: &str = "panic"; + #[inline(always)] pub fn get_stage_metrics_enabled() -> bool { GET_STAGE_METRICS_ENABLED.load(Ordering::Relaxed) @@ -2042,6 +2050,44 @@ pub fn record_put_object_stage_duration_from(stage: &'static str, started_at: Op } } +#[inline(always)] +fn put_stage_count_value(value: usize) -> f64 { + match u32::try_from(value) { + Ok(value) => f64::from(value), + Err(_) => f64::from(u32::MAX), + } +} + +#[inline(always)] +pub fn record_put_rename_fdatasync_batch(mode: &'static str, files: usize) { + if !put_stage_metrics_enabled() { + return; + } + histogram!("rustfs_s3_put_object_rename_fdatasync_batch_files", "mode" => mode).record(put_stage_count_value(files)); +} + +#[inline(always)] +pub fn record_put_rename_quorum_wait_fanout( + scheduled: usize, + write_quorum: usize, + success: usize, + error: usize, + panicked: usize, +) { + if !put_stage_metrics_enabled() { + return; + } + for (state, count) in [ + (PUT_RENAME_QUORUM_FANOUT_STATE_SCHEDULED, scheduled), + (PUT_RENAME_QUORUM_FANOUT_STATE_WRITE_QUORUM, write_quorum), + (PUT_RENAME_QUORUM_FANOUT_STATE_SUCCESS, success), + (PUT_RENAME_QUORUM_FANOUT_STATE_ERROR, error), + (PUT_RENAME_QUORUM_FANOUT_STATE_PANIC, panicked), + ] { + histogram!("rustfs_s3_put_object_rename_quorum_wait_fanout_disks", "state" => state).record(put_stage_count_value(count)); + } +} + /// Record generic internal operation stage duration (non-PUT paths). /// Use this for metacache walks, listing, lifecycle, and other background /// operations that are NOT part of the PUT object hot path. @@ -3122,6 +3168,70 @@ mod tests { assert!(stages.iter().all(|stage| recorded.contains(*stage))); } + #[test] + fn put_rename_code_level_metrics_are_static_and_gated() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + set_put_stage_metrics_enabled(false); + record_put_rename_fdatasync_batch(PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL, 2); + record_put_rename_quorum_wait_fanout(4, 3, 3, 1, 0); + + set_put_stage_metrics_enabled(true); + record_put_rename_fdatasync_batch(PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL, 9); + record_put_rename_quorum_wait_fanout(4, 3, 3, 1, 0); + set_put_stage_metrics_enabled(false); + }); + + let rows = snapshotter.snapshot().into_vec(); + assert_eq!(histogram_samples(&rows, "rustfs_s3_put_object_rename_fdatasync_batch_files"), vec![9.0]); + let batch_modes = rows + .iter() + .filter(|(composite, _, _, _)| { + composite.kind() == MetricKind::Histogram + && composite.key().name() == "rustfs_s3_put_object_rename_fdatasync_batch_files" + }) + .flat_map(|(composite, _, _, _)| { + composite + .key() + .labels() + .filter(|label| label.key() == "mode") + .map(|label| label.value().to_string()) + .collect::>() + }) + .collect::>(); + assert_eq!(batch_modes, HashSet::from([PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL.to_string()])); + + let quorum_samples = histogram_samples(&rows, "rustfs_s3_put_object_rename_quorum_wait_fanout_disks"); + assert_eq!(quorum_samples, vec![0.0, 1.0, 3.0, 3.0, 4.0]); + let quorum_states = rows + .iter() + .filter(|(composite, _, _, _)| { + composite.kind() == MetricKind::Histogram + && composite.key().name() == "rustfs_s3_put_object_rename_quorum_wait_fanout_disks" + }) + .flat_map(|(composite, _, _, _)| { + composite + .key() + .labels() + .filter(|label| label.key() == "state") + .map(|label| label.value().to_string()) + .collect::>() + }) + .collect::>(); + assert_eq!( + quorum_states, + HashSet::from([ + PUT_RENAME_QUORUM_FANOUT_STATE_SCHEDULED.to_string(), + PUT_RENAME_QUORUM_FANOUT_STATE_WRITE_QUORUM.to_string(), + PUT_RENAME_QUORUM_FANOUT_STATE_SUCCESS.to_string(), + PUT_RENAME_QUORUM_FANOUT_STATE_ERROR.to_string(), + PUT_RENAME_QUORUM_FANOUT_STATE_PANIC.to_string(), + ]) + ); + } + #[test] fn test_put_object_diagnostic_buckets() { assert_eq!(put_object_size_bucket(0), "unknown"); diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index 2883667c1..f1a41c888 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -94,7 +94,7 @@ aws-smithy-types = { workspace = true } [dev-dependencies] anyhow = { workspace = true } # Debugging recorder for asserting emitted metrics in tests. -metrics-util = { version = "0.20", features = ["debugging"] } +metrics-util = { workspace = true, features = ["debugging"] } insta = { workspace = true, features = ["yaml", "json"] } tempfile = { workspace = true } temp-env = { workspace = true } diff --git a/crates/lifecycle/Cargo.toml b/crates/lifecycle/Cargo.toml index 8f5f63556..ae8232f20 100644 --- a/crates/lifecycle/Cargo.toml +++ b/crates/lifecycle/Cargo.toml @@ -67,7 +67,7 @@ url.workspace = true uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] } [dev-dependencies] -metrics-util = { version = "0.20", features = ["debugging"] } +metrics-util = { workspace = true, features = ["debugging"] } proptest = "1" serial_test.workspace = true temp-env.workspace = true diff --git a/crates/lock/src/client/local.rs b/crates/lock/src/client/local.rs index 3b2a185aa..84ef32a88 100644 --- a/crates/lock/src/client/local.rs +++ b/crates/lock/src/client/local.rs @@ -47,6 +47,8 @@ pub struct LocalClient { #[derive(Debug)] struct LocalGuardEntry { guard: FastLockGuard, + acquired_at: SystemTime, + last_refreshed: SystemTime, expires_at: SystemTime, deadline: Instant, ttl: Duration, @@ -54,11 +56,13 @@ struct LocalGuardEntry { impl LocalGuardEntry { fn new(guard: FastLockGuard, ttl: Duration) -> Self { - let now = SystemTime::now(); + let acquired_at = SystemTime::now(); let monotonic_now = Instant::now(); Self { guard, - expires_at: now.checked_add(ttl).unwrap_or(now), + acquired_at, + last_refreshed: acquired_at, + expires_at: acquired_at.checked_add(ttl).unwrap_or(acquired_at), deadline: monotonic_now.checked_add(ttl).unwrap_or(monotonic_now), ttl, } @@ -72,6 +76,7 @@ impl LocalGuardEntry { let now = SystemTime::now(); let monotonic_now = Instant::now(); self.expires_at = now.checked_add(self.ttl).unwrap_or(now); + self.last_refreshed = now; self.deadline = monotonic_now.checked_add(self.ttl).unwrap_or(monotonic_now); } } @@ -231,13 +236,14 @@ impl LockClient for LocalClient { match lock_manager.acquire_lock(build_lock_request(remaining)).await { Ok(guard) => { let lock_id = request.lock_id.clone(); - let acquired_at = SystemTime::now(); - let expires_at = acquired_at.checked_add(request.ttl).unwrap_or(acquired_at); + let entry = LocalGuardEntry::new(guard, request.ttl); + let acquired_at = entry.acquired_at; + let expires_at = entry.expires_at; { let shard = self.get_shard(&lock_id); let mut guards = shard.write().await; - guards.insert(lock_id.clone(), LocalGuardEntry::new(guard, request.ttl)); + guards.insert(lock_id.clone(), entry); } let lock_info = LockInfo { @@ -342,9 +348,9 @@ impl LockClient for LocalClient { lock_type, status, owner: entry.guard.owner().to_string(), - acquired_at: SystemTime::now(), + acquired_at: entry.acquired_at, expires_at: entry.expires_at, - last_refreshed: SystemTime::now(), + last_refreshed: entry.last_refreshed, metadata: LockMetadata::default(), priority: LockPriority::Normal, wait_start_time: None, @@ -354,6 +360,26 @@ impl LockClient for LocalClient { } } + async fn list_lock_leases(&self) -> Vec { + let mut leases = Vec::new(); + for shard in self.guard_storage.iter() { + let guards = shard.read().await; + leases.reserve(guards.len()); + leases.extend(guards.iter().map(|(lock_id, entry)| crate::LockLeaseInfo { + resource: lock_id.resource.clone(), + lock_type: match entry.guard.mode() { + crate::LockMode::Shared => LockType::Shared, + crate::LockMode::Exclusive => LockType::Exclusive, + }, + owner: entry.guard.owner().to_string(), + acquired_at: entry.acquired_at, + remaining_ttl: entry.deadline.saturating_duration_since(Instant::now()), + guard_id: (!entry.guard.is_disabled()).then(|| entry.guard.guard_id()), + })); + } + leases + } + async fn get_stats(&self) -> Result { Ok(LockStats::default()) } @@ -403,6 +429,10 @@ mod tests { assert!(client.check_status(&lock_id).await.unwrap().is_some()); tokio::time::sleep(Duration::from_millis(15)).await; wait_until_reaped(&client, &lock_id).await; + assert!( + client.list_lock_leases().await.is_empty(), + "reaped guards must disappear from lease diagnostics" + ); let direct = manager .acquire_lock(crate::ObjectLockRequest::new_write(request.resource.clone(), "owner-b")) @@ -442,6 +472,71 @@ mod tests { wait_until_reaped(&client, &lock_id).await; } + #[tokio::test(start_paused = true)] + async fn lease_snapshot_tracks_refresh_without_resetting_acquisition_time() { + let manager = Arc::new(GlobalLockManager::new()); + let client = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_secs(60)); + client.reaper_started.store(true, Ordering::Release); + let lock_request = request(crate::ObjectKey::new("bucket", "lease-snapshot"), "owner-a", Duration::from_secs(30)); + let lock_id = lock_request.lock_id.clone(); + + assert!( + client + .acquire_lock(&lock_request) + .await + .expect("lease-backed lock should acquire") + .success + ); + let initial = client.list_lock_leases().await.pop().expect("acquired lock should be listed"); + let initial_status = client + .check_status(&lock_id) + .await + .expect("initial lock status should be readable") + .expect("newly acquired lock should remain held"); + assert_eq!(initial_status.last_refreshed, initial_status.acquired_at); + + tokio::time::advance(Duration::from_secs(20)).await; + let aging = client + .list_lock_leases() + .await + .pop() + .expect("held lock should remain listed before refresh"); + assert_eq!(aging.remaining_ttl, Duration::from_secs(10)); + let aging_status = client + .check_status(&lock_id) + .await + .expect("aging lock status should be readable") + .expect("aging lock should remain held"); + assert_eq!(aging_status.last_refreshed, initial_status.last_refreshed); + + assert!(client.refresh(&lock_id).await.expect("refresh should return a result")); + + let refreshed = client + .list_lock_leases() + .await + .pop() + .expect("refreshed lock should be listed"); + let status = client + .check_status(&lock_id) + .await + .expect("lock status should be readable") + .expect("refreshed lock should remain held"); + + assert_eq!(refreshed.acquired_at, initial.acquired_at); + assert_eq!(refreshed.guard_id, initial.guard_id); + assert_eq!(status.acquired_at, initial.acquired_at); + assert!(status.last_refreshed > initial_status.last_refreshed); + assert_eq!(refreshed.remaining_ttl, Duration::from_secs(30)); + + tokio::time::advance(Duration::from_secs(30)).await; + let expired = client + .list_lock_leases() + .await + .pop() + .expect("unreaped lease should remain listed"); + assert_eq!(expired.remaining_ttl, Duration::ZERO); + } + #[tokio::test(start_paused = true)] async fn refresh_after_expiry_releases_guard_without_reviving_it() { let manager = Arc::new(GlobalLockManager::new()); diff --git a/crates/lock/src/client/mod.rs b/crates/lock/src/client/mod.rs index d16db4bb6..bf378c3bf 100644 --- a/crates/lock/src/client/mod.rs +++ b/crates/lock/src/client/mod.rs @@ -15,7 +15,7 @@ pub mod local; // pub mod remote; -use crate::{LockId, LockInfo, LockRequest, LockResponse, LockStats, Result}; +use crate::{LockId, LockInfo, LockLeaseInfo, LockRequest, LockResponse, LockStats, Result}; use async_trait::async_trait; use futures::future::join_all; use std::sync::Arc; @@ -54,6 +54,13 @@ pub trait LockClient: Send + Sync + std::fmt::Debug { /// Check lock status async fn check_status(&self, lock_id: &LockId) -> Result>; + /// Return authoritative lease information when this client owns lease state. + /// + /// Clients that do not manage renewable leases return an empty snapshot. + async fn list_lock_leases(&self) -> Vec { + Vec::new() + } + /// Get statistics async fn get_stats(&self) -> Result; diff --git a/crates/lock/src/fast_lock/manager.rs b/crates/lock/src/fast_lock/manager.rs index b24f96d6f..fecb2d92f 100644 --- a/crates/lock/src/fast_lock/manager.rs +++ b/crates/lock/src/fast_lock/manager.rs @@ -100,7 +100,7 @@ impl FastObjectLockManager { Ok(()) => { let guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone()); // Register guard to prevent premature cleanup - shard.register_guard(guard.guard_id()); + shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner()); Ok(guard) } Err(err) => Err(err), @@ -223,7 +223,7 @@ impl FastObjectLockManager { if acquired { let guard = FastLockGuard::new(key.clone(), mode, owner.clone(), shard.clone()); - shard.register_guard(guard.guard_id()); + shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner()); all_successful.push(key); guards.push(guard); } @@ -252,7 +252,7 @@ impl FastObjectLockManager { match shard.acquire_lock(request).await { Ok(()) => { let guard = FastLockGuard::new(request.key.clone(), request.mode, request.owner.clone(), shard.clone()); - shard.register_guard(guard.guard_id()); + shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner()); acquired_guards.push(guard); } Err(err) => { @@ -295,9 +295,26 @@ impl FastObjectLockManager { /// Powers the admin "top locks" view. Order is shard-then-insertion and is /// not otherwise stable across calls. pub fn list_locks(&self) -> Vec { + self.list_locks_with_holder_counts() + .into_iter() + .map(|(info, _)| info) + .collect() + } + + /// Enumerate held locks with the number of guards represented by each owner. + pub fn list_locks_with_holder_counts(&self) -> Vec<(crate::fast_lock::types::ObjectLockInfo, u32)> { let mut infos = Vec::new(); for shard in &self.shards { - infos.extend(shard.list_locks()); + infos.extend(shard.list_locks_with_holder_counts()); + } + infos + } + + /// Enumerate held locks with holder counts and stable holder identities. + pub fn list_locks_with_holder_generations(&self) -> Vec<(crate::fast_lock::types::ObjectLockInfo, u32, Option>)> { + let mut infos = Vec::new(); + for shard in &self.shards { + infos.extend(shard.list_locks_with_holder_generations()); } infos } @@ -548,14 +565,18 @@ mod tests { let write_key = ObjectKey::new("bucket", "write-object"); let read_key = ObjectKey::new("bucket", "read-object"); - let _write_guard = manager + let write_guard = manager .acquire_write_lock(write_key.clone(), "writer") .await .expect("write lock should acquire"); - let _read_guard = manager + let read_guard = manager .acquire_read_lock(read_key.clone(), "reader") .await .expect("read lock should acquire"); + let second_read_guard = manager + .acquire_read_lock(read_key.clone(), "reader") + .await + .expect("second read lock should acquire"); let mut locks = manager.list_locks(); locks.sort_by(|a, b| a.key.object.cmp(&b.key.object)); @@ -569,6 +590,42 @@ mod tests { assert_eq!(write.mode, LockMode::Exclusive); assert_eq!(write.owner.as_ref(), "writer"); + let counts = manager.list_locks_with_holder_counts(); + let (_, read_holder_count) = counts + .iter() + .find(|(info, _)| info.key == read_key) + .expect("read holder count listed"); + assert_eq!(*read_holder_count, 2); + let (_, write_holder_count) = counts + .iter() + .find(|(info, _)| info.key == write_key) + .expect("write holder count listed"); + assert_eq!(*write_holder_count, 1); + + let generations = manager.list_locks_with_holder_generations(); + let (_, _, read_generations) = generations + .iter() + .find(|(info, _, _)| info.key == read_key) + .expect("read holder generations listed"); + let mut expected_read_generations = vec![read_guard.guard_id(), second_read_guard.guard_id()]; + expected_read_generations.sort_unstable(); + assert_eq!(read_generations.as_ref(), Some(&expected_read_generations)); + + let (_, _, write_generations) = generations + .iter() + .find(|(info, _, _)| info.key == write_key) + .expect("write holder generation listed"); + assert_eq!(write_generations.as_ref(), Some(&vec![write_guard.guard_id()])); + + drop(read_guard); + let remaining = manager.list_locks_with_holder_generations(); + let (_, remaining_count, remaining_generations) = remaining + .iter() + .find(|(info, _, _)| info.key == read_key) + .expect("remaining read holder generation listed"); + assert_eq!(*remaining_count, 1); + assert_eq!(remaining_generations.as_ref(), Some(&vec![second_read_guard.guard_id()])); + manager.shutdown().await; } diff --git a/crates/lock/src/fast_lock/shard.rs b/crates/lock/src/fast_lock/shard.rs index a0ca7eaa7..5da4d355b 100644 --- a/crates/lock/src/fast_lock/shard.rs +++ b/crates/lock/src/fast_lock/shard.rs @@ -24,7 +24,20 @@ use crate::fast_lock::{ state::ObjectLockState, types::{LockMode, LockResult, ObjectKey, ObjectLockRequest}, }; -use std::collections::HashSet; + +#[derive(Debug)] +struct ActiveGuardInfo { + key: ObjectKey, + mode: LockMode, + owner: Arc, +} + +#[derive(Debug, PartialEq, Eq, Hash)] +struct GuardHolderKey { + key: ObjectKey, + mode: LockMode, + owner: Arc, +} /// Lock shard to reduce global contention #[derive(Debug)] @@ -38,7 +51,7 @@ pub struct LockShard { /// Shard ID for debugging _shard_id: usize, /// Active guard IDs to prevent cleanup of locks with live guards - active_guards: parking_lot::Mutex>, + active_guards: parking_lot::Mutex>>, } /// Cancellation-safe waiter counter ticket. @@ -84,7 +97,7 @@ impl LockShard { object_pool: ObjectStatePool::new(), metrics: ShardMetrics::new(), _shard_id: shard_id, - active_guards: parking_lot::Mutex::new(HashSet::new()), + active_guards: parking_lot::Mutex::new(HashMap::new()), } } @@ -327,7 +340,7 @@ impl LockShard { // First, try to remove the guard from active set let guard_was_active = { let mut guards = self.active_guards.lock(); - guards.remove(&guard_id) + guards.remove(&guard_id).is_some() }; // If guard was not active, this is a double-release attempt @@ -375,8 +388,19 @@ impl LockShard { /// Register a guard to prevent premature cleanup pub fn register_guard(&self, guard_id: u64) { + self.active_guards.lock().insert(guard_id, None); + } + + pub(crate) fn register_guard_with_info(&self, guard_id: u64, key: &ObjectKey, mode: LockMode, owner: &Arc) { let mut guards = self.active_guards.lock(); - guards.insert(guard_id); + guards.insert( + guard_id, + Some(ActiveGuardInfo { + key: key.clone(), + mode, + owner: owner.clone(), + }), + ); } /// Unregister a guard (called when guard is dropped) @@ -396,7 +420,7 @@ impl LockShard { #[cfg(test)] pub fn is_guard_active(&self, guard_id: u64) -> bool { let guards = self.active_guards.lock(); - guards.contains(&guard_id) + guards.contains_key(&guard_id) } /// Calculate adaptive timeout based on current system load and request priority @@ -544,6 +568,13 @@ impl LockShard { /// holder. Entries for objects that are tracked but not currently locked /// (e.g. pooled-but-idle state) are skipped. pub fn list_locks(&self) -> Vec { + self.list_locks_with_holder_counts() + .into_iter() + .map(|(info, _)| info) + .collect() + } + + pub(crate) fn list_locks_with_holder_counts(&self) -> Vec<(crate::fast_lock::types::ObjectLockInfo, u32)> { let objects = self.objects.read(); let mut infos = Vec::new(); for (key, state) in objects.iter() { @@ -558,14 +589,17 @@ impl LockShard { .acquired_at .checked_add(info.lock_timeout) .unwrap_or_else(|| info.acquired_at + crate::fast_lock::DEFAULT_LOCK_TIMEOUT); - infos.push(crate::fast_lock::types::ObjectLockInfo { - key: key.clone(), - mode, - owner: info.owner, - acquired_at: info.acquired_at, - expires_at, - priority, - }); + infos.push(( + crate::fast_lock::types::ObjectLockInfo { + key: key.clone(), + mode, + owner: info.owner, + acquired_at: info.acquired_at, + expires_at, + priority, + }, + 1, + )); } } LockMode::Shared => { @@ -574,14 +608,17 @@ impl LockShard { .acquired_at .checked_add(entry.lock_timeout) .unwrap_or_else(|| entry.acquired_at + crate::fast_lock::DEFAULT_LOCK_TIMEOUT); - infos.push(crate::fast_lock::types::ObjectLockInfo { - key: key.clone(), - mode, - owner: entry.owner.clone(), - acquired_at: entry.acquired_at, - expires_at, - priority, - }); + infos.push(( + crate::fast_lock::types::ObjectLockInfo { + key: key.clone(), + mode, + owner: entry.owner.clone(), + acquired_at: entry.acquired_at, + expires_at, + priority, + }, + entry.count, + )); } } } @@ -589,6 +626,50 @@ impl LockShard { infos } + pub(crate) fn list_locks_with_holder_generations( + &self, + ) -> Vec<(crate::fast_lock::types::ObjectLockInfo, u32, Option>)> { + // Snapshot lock state before guard registrations. Acquires register after + // mutating state, while releases unregister before mutating state, so a + // concurrent transition can only make the cohort mismatch and fall back. + let infos = self.list_locks_with_holder_counts(); + let guards = self.active_guards.lock(); + let mut guard_ids_by_holder: HashMap> = HashMap::with_capacity(guards.len()); + for (&guard_id, guard) in guards + .iter() + .filter_map(|(guard_id, guard)| guard.as_ref().map(|guard| (guard_id, guard))) + { + let key = GuardHolderKey { + key: guard.key.clone(), + mode: guard.mode, + owner: guard.owner.clone(), + }; + guard_ids_by_holder + .entry(key) + .and_modify(|guard_ids| guard_ids.push(guard_id)) + .or_insert_with(|| vec![guard_id]); + } + drop(guards); + for guard_ids in guard_ids_by_holder.values_mut() { + guard_ids.sort_unstable(); + } + + infos + .into_iter() + .map(|(info, holder_count)| { + let key = GuardHolderKey { + key: info.key.clone(), + mode: info.mode, + owner: info.owner.clone(), + }; + let generation = guard_ids_by_holder + .remove(&key) + .filter(|guard_ids| u32::try_from(guard_ids.len()).ok() == Some(holder_count)); + (info, holder_count, generation) + }) + .collect() + } + /// Force-release every holder of a lock on `key`, regardless of owner. /// /// Returns the number of owners that were released. Used by the admin diff --git a/crates/lock/src/fast_lock/types.rs b/crates/lock/src/fast_lock/types.rs index 8337971fe..fbf51f9ae 100644 --- a/crates/lock/src/fast_lock/types.rs +++ b/crates/lock/src/fast_lock/types.rs @@ -257,7 +257,7 @@ impl std::fmt::Display for ObjectKey { } /// Lock type for object operations -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum LockMode { /// Shared lock for read operations Shared, diff --git a/crates/lock/src/lib.rs b/crates/lock/src/lib.rs index 6562f8675..be1f1c68d 100644 --- a/crates/lock/src/lib.rs +++ b/crates/lock/src/lib.rs @@ -51,8 +51,8 @@ pub use crate::{ namespace::{NamespaceLock, NamespaceLockGuard, NamespaceLockWrapper}, // Core types types::{ - HealthInfo, HealthStatus, LockId, LockInfo, LockMetadata, LockPriority, LockRequest, LockResponse, LockStats, LockStatus, - LockType, + HealthInfo, HealthStatus, LockId, LockInfo, LockLeaseInfo, LockMetadata, LockPriority, LockRequest, LockResponse, + LockStats, LockStatus, LockType, }, }; diff --git a/crates/lock/src/types.rs b/crates/lock/src/types.rs index eebe158ff..77a344dea 100644 --- a/crates/lock/src/types.rs +++ b/crates/lock/src/types.rs @@ -79,6 +79,23 @@ pub struct LockInfo { pub wait_start_time: Option, } +/// Point-in-time lease information exposed by lock clients for diagnostics. +#[derive(Debug, Clone)] +pub struct LockLeaseInfo { + /// Resource protected by the lock. + pub resource: ObjectKey, + /// Shared or exclusive lock mode. + pub lock_type: LockType, + /// Lock owner recorded by the local lock backend. + pub owner: String, + /// Original acquisition time. Refreshes do not change this value. + pub acquired_at: SystemTime, + /// Opaque guard identity used to reject stale diagnostic snapshots. + pub guard_id: Option, + /// Remaining lease duration derived from the monotonic lease deadline. + pub remaining_ttl: Duration, +} + impl LockInfo { /// Check if the lock has expired pub fn has_expired(&self) -> bool { diff --git a/crates/madmin/src/site_replication.rs b/crates/madmin/src/site_replication.rs index 740924405..1dcee9350 100644 --- a/crates/madmin/src/site_replication.rs +++ b/crates/madmin/src/site_replication.rs @@ -83,6 +83,16 @@ pub struct SiteReplicationInfo { pub service_account_access_key: String, #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] pub api_version: Option, + /// Outstanding peer deliveries. Absent when the retry queue is empty, so a + /// healthy site serializes exactly as it did before this field existed. + /// Present means peer operations are failing even if `enabled` is true. + #[serde(rename = "retryStats", default, skip_serializing_if = "Option::is_none")] + pub retry_stats: Option, + /// A multi-step lifecycle operation this site has not finished โ€” most + /// importantly a removal that could not reach its peers, which makes the + /// site reject peer operations while `enabled` may still read true. + #[serde(rename = "pendingOperation", default, skip_serializing_if = "Option::is_none")] + pub pending_operation: Option, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] diff --git a/crates/notify/src/runtime_facade.rs b/crates/notify/src/runtime_facade.rs index 683a01ba6..f692880c5 100644 --- a/crates/notify/src/runtime_facade.rs +++ b/crates/notify/src/runtime_facade.rs @@ -527,9 +527,19 @@ mod tests { } #[tokio::test] - async fn runtime_facade_stops_empty_replay_workers() { + async fn stopping_replay_workers_is_a_no_op_when_there_are_none() { let (facade, _, _) = build_facade(); + facade.stop_replay_workers().await; + + // The stop path takes the worker list and hands it to the adapter, so an + // empty facade must come back with the list still empty and dispatch + // released rather than left paused (rustfs/backlog#1836). + assert!(facade.replay_workers.read().await.is_empty()); + + // Calling it twice must stay harmless: shutdown paths do exactly that. + facade.stop_replay_workers().await; + assert!(facade.replay_workers.read().await.is_empty()); } #[tokio::test] diff --git a/crates/object-data-cache/Cargo.toml b/crates/object-data-cache/Cargo.toml index 704f3df6f..152b73992 100644 --- a/crates/object-data-cache/Cargo.toml +++ b/crates/object-data-cache/Cargo.toml @@ -58,7 +58,7 @@ tracing = { workspace = true, optional = true } [dev-dependencies] criterion = { workspace = true, features = ["html_reports"] } -metrics-util = { version = "0.20", features = ["debugging"] } +metrics-util = { workspace = true, features = ["debugging"] } # `rt-multi-thread` lets the concurrency stress tests run tasks on real worker # threads, so they exercise true parallelism on the shared singleflight/index # state rather than only cooperative interleaving. diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index eeec4ee02..17ba113ed 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -163,4 +163,5 @@ libc = { workspace = true } [dev-dependencies] tempfile = { workspace = true } temp-env = { workspace = true } -log = "0.4" +log.workspace = true +metrics-util = { workspace = true, features = ["debugging"] } diff --git a/crates/obs/src/metrics/collectors/scanner.rs b/crates/obs/src/metrics/collectors/scanner.rs index db220ca96..56f1db556 100644 --- a/crates/obs/src/metrics/collectors/scanner.rs +++ b/crates/obs/src/metrics/collectors/scanner.rs @@ -465,6 +465,57 @@ fn bool_metric_value(enabled: bool) -> f64 { mod tests { use super::*; use crate::metrics::report::report_metrics; + use metrics_util::debugging::DebuggingRecorder; + use rustfs_common::metrics::{Metric, Metrics}; + + fn prometheus_counter_name(name: &str) -> String { + if name.ends_with("_total") { + name.to_string() + } else { + format!("{name}_total") + } + } + + #[test] + fn scanner_lifetime_counters_have_one_prometheus_producer() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let scanner_metrics = collect_scanner_metrics(&ScannerStats { + directories_scanned: 3, + objects_scanned: 7, + ..Default::default() + }); + + metrics::with_local_recorder(&recorder, || { + Metrics::time(Metric::ScanObject)(); + Metrics::time(Metric::ScanFolder)(); + report_metrics(&scanner_metrics); + }); + + let normalized_counter_names: Vec<_> = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter_map(|(composite, _, _, value)| { + matches!(value, metrics_util::debugging::DebugValue::Counter(_)) + .then(|| prometheus_counter_name(composite.key().name())) + }) + .collect(); + + for name in [ + "rustfs_scanner_objects_scanned_total", + "rustfs_scanner_directories_scanned_total", + ] { + assert_eq!( + normalized_counter_names + .iter() + .filter(|candidate| candidate.as_str() == name) + .count(), + 1, + "scanner lifetime counter must have exactly one producer after Prometheus name normalization" + ); + } + } #[test] fn test_collect_scanner_metrics() { diff --git a/crates/protocols/src/sftp/lifecycle.rs b/crates/protocols/src/sftp/lifecycle.rs index 4bd3f1aad..64c05a950 100644 --- a/crates/protocols/src/sftp/lifecycle.rs +++ b/crates/protocols/src/sftp/lifecycle.rs @@ -84,11 +84,11 @@ const TCP_STATE_RADIX: u32 = 16; /// and the SftpDriver, registered weakly into the SessionRegistry so an /// outside observer can enumerate live sessions without holding their /// lifetime. -#[allow(dead_code)] pub struct SessionDiag { pub session_id: u64, pub local: SocketAddr, pub peer: SocketAddr, + #[allow(dead_code, reason = "written at accept time but never read back (backlog#1823)")] pub accepted_at: Instant, pub last_activity_ms: AtomicU64, } diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index d23224b93..a9e59b8a0 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -2433,7 +2433,7 @@ mod tests { json_field: "opts", bin_field: "opts_bin", }, - json_encoder: "let opts_str = compat_json(opts)?;", + json_encoder: "let encoded_opts = compat_json(opts).and_then(|opts_str| encode_msgpack(opts).map(|opts_bin| (opts_str, opts_bin)));", policy: RequestJsonPolicy::MsgpackOnlyEligible, }, RequestCompatSendSite { diff --git a/crates/s3-types/src/event_name.rs b/crates/s3-types/src/event_name.rs index 659b88b46..dee252dbd 100644 --- a/crates/s3-types/src/event_name.rs +++ b/crates/s3-types/src/event_name.rs @@ -873,9 +873,16 @@ mod tests { /// now return a finite, non-panicking mask. #[test] fn test_mask_never_recurses_for_any_variant() { - for ev in ALL_EVENT_NAMES { - // Must terminate (no infinite recursion / stack overflow). - let _ = ev.mask(); + // Terminating is the point โ€” a regression here overflows the stack rather + // than failing an assertion โ€” but the masks are collected and checked so + // the loop cannot be optimised into nothing and so a variant that starts + // returning an empty mask is caught too (rustfs/backlog#1836). + let masks: Vec = ALL_EVENT_NAMES.iter().map(|ev| ev.mask()).collect(); + + assert_eq!(masks.len(), ALL_EVENT_NAMES.len()); + for (ev, mask) in ALL_EVENT_NAMES.iter().zip(&masks) { + assert_ne!(*mask, 0, "{ev:?} must carry at least one bit"); + assert_eq!(ev.mask(), *mask, "{ev:?} must return the same mask every call"); } } diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index fe743a1c6..eb2dc9d4b 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1425,10 +1425,6 @@ fn maintenance_inspection_decision(generation: u64, current_generation: u64, att } } -fn single_disk_default_cycle_secs(_features: ScannerMaintenanceFeatures) -> Option { - None -} - fn single_disk_default_speed() -> ScannerSpeed { ScannerSpeed::Default } @@ -1592,9 +1588,12 @@ async fn configure_scanner_defaults( scanner_maintenance_generation(), ) }); - let default_cycle_secs = single_disk_default_cycle_secs(features); + // Single-disk keeps the speed-preset-derived default cycle (60s at the + // `default` preset) instead of a special shorter cycle: no measured + // cold-start ILM latency basis for an override, and clean-idle backoff + // already stretches idle cadence. Decision record: backlog#1878 (HS-16). set_scanner_default_speed(single_disk_default_speed()); - set_scanner_default_cycle_secs(default_cycle_secs); + set_scanner_default_cycle_secs(None); info!( target: "rustfs::scanner", event = EVENT_SCANNER_RUNTIME_CONFIG, @@ -1603,7 +1602,6 @@ async fn configure_scanner_defaults( env_speed = ENV_SCANNER_SPEED, env_cycle = ENV_SCANNER_CYCLE, env_start_delay = ENV_SCANNER_START_DELAY_SECS, - ?default_cycle_secs, lifecycle_active = features.lifecycle, replication_active = features.replication, feature_inspection_failed = features.inspection_failed, @@ -6950,11 +6948,6 @@ mod tests { }); } - #[test] - fn test_single_disk_default_cycle_uses_speed_based_interval_without_maintenance_features() { - assert_eq!(single_disk_default_cycle_secs(ScannerMaintenanceFeatures::default()), None); - } - #[test] fn test_single_disk_default_speed_uses_regular_scanner_default() { assert_eq!(single_disk_default_speed(), ScannerSpeed::Default); @@ -7415,39 +7408,6 @@ mod tests { }); } - #[test] - fn test_single_disk_default_cycle_preserves_regular_cycle_for_lifecycle() { - assert_eq!( - single_disk_default_cycle_secs(ScannerMaintenanceFeatures { - lifecycle: true, - ..Default::default() - }), - None - ); - } - - #[test] - fn test_single_disk_default_cycle_preserves_regular_cycle_for_replication() { - assert_eq!( - single_disk_default_cycle_secs(ScannerMaintenanceFeatures { - replication: true, - ..Default::default() - }), - None - ); - } - - #[test] - fn test_single_disk_default_cycle_preserves_regular_cycle_on_inspection_failure() { - assert_eq!( - single_disk_default_cycle_secs(ScannerMaintenanceFeatures { - inspection_failed: true, - ..Default::default() - }), - None - ); - } - #[test] #[serial] fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() { diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index c573065f7..c15ed6c41 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -71,7 +71,6 @@ const EVENT_SCANNER_METADATA_CORRUPT: &str = "scanner_metadata_corrupt"; const EVENT_SCANNER_LIFECYCLE_ACTION: &str = "scanner_lifecycle_action"; const EVENT_SCANNER_HEAL_ADMISSION: &str = "scanner_heal_admission"; const EVENT_SCANNER_ALERT_STATE: &str = "scanner_alert_state"; -const EVENT_SCANNER_COMPAT_STATE: &str = "scanner_compat_state"; const DATA_USAGE_UPDATE_DIR_CYCLES: u32 = 16; const DATA_SCANNER_COMPACT_LEAST_OBJECT: usize = 500; @@ -92,7 +91,6 @@ const ENV_FAILED_OBJECTS_MAX: &str = "RUSTFS_DATA_USAGE_FAILED_OBJECTS_MAX"; const DEFAULT_FAILED_OBJECT_TTL_SECS: u32 = 86_400; const DEFAULT_FAILED_OBJECTS_MAX: u32 = 10_000; const DEFAULT_SCANNER_DEEP_VERIFY_COOLDOWN_SECS: u64 = 60; -const METRIC_SCANNER_INLINE_HEAL_TOTAL: &str = "rustfs_scanner_inline_heal_total"; const METRIC_SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL: &str = "rustfs_scanner_excess_object_versions_total"; const METRIC_SCANNER_EXCESS_OBJECT_VERSION_SIZE_TOTAL: &str = "rustfs_scanner_excess_object_version_size_total"; const METRIC_SCANNER_EXCESS_FOLDERS_TOTAL: &str = "rustfs_scanner_excess_folders_total"; @@ -196,8 +194,6 @@ fn emit_scanner_alert_event(event_name: &str, bucket: &str, object: &str, size: } const MAX_PENDING_SCANNER_HEALS_PER_BUCKET: usize = 10_000; -static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new(); -static SCANNER_INLINE_HEAL_METRICS_ONCE: Once = Once::new(); static SCANNER_ALERT_METRICS_ONCE: Once = Once::new(); #[cfg(test)] @@ -251,27 +247,6 @@ fn effective_object_heal_scan_mode(heal_bitrot: bool, mod_time: Option bool { - scanner_inline_heal_enabled_from_value(std::env::var(rustfs_config::ENV_SCANNER_INLINE_HEAL_ENABLE).ok().as_deref()) -} - -fn scanner_inline_heal_enabled_from_value(value: Option<&str>) -> bool { - match value { - Some(value) => matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "true" | "on" | "yes"), - None => rustfs_config::DEFAULT_SCANNER_INLINE_HEAL_ENABLE, - } -} - -fn ensure_scanner_inline_heal_metric_registered() { - SCANNER_INLINE_HEAL_METRICS_ONCE.call_once(|| { - describe_counter!( - METRIC_SCANNER_INLINE_HEAL_TOTAL, - "Total number of inline heal operations executed directly by scanner." - ); - counter!(METRIC_SCANNER_INLINE_HEAL_TOTAL).increment(0); - }); -} - fn ensure_scanner_alert_metrics_registered() { SCANNER_ALERT_METRICS_ONCE.call_once(|| { describe_counter!( @@ -505,24 +480,6 @@ fn should_alert_excessive_versions(remaining_versions: usize, cumulative_size: i (too_many_versions, too_large_versions) } -fn warn_inline_heal_compat_requested() { - if !scanner_inline_heal_enabled() { - return; - } - - SCANNER_INLINE_HEAL_WARN_ONCE.call_once(|| { - warn!( - target: "rustfs::scanner::folder", - event = EVENT_SCANNER_COMPAT_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_HEAL, - env = rustfs_config::ENV_SCANNER_INLINE_HEAL_ENABLE, - state = "inline_heal_rollback_unsupported", - "Scanner inline-heal rollback is unsupported; using async heal admission" - ); - }); -} - fn non_negative_i64_to_u64(value: i64) -> u64 { value.max(0) as u64 } @@ -1282,7 +1239,6 @@ impl ScannerItem { async fn heal_actions(&mut self, oi: &ObjectInfo, actual_size: i64, size_summary: &mut SizeSummary) -> i64 { if self.heal_enabled { - warn_inline_heal_compat_requested(); self.enqueue_heal(oi).await; } @@ -3231,8 +3187,6 @@ pub async fn scan_data_folder( ) -> Result { use crate::data_usage_define::DATA_USAGE_ROOT; - ensure_scanner_inline_heal_metric_registered(); - // Check that we're not trying to scan the root if cache.info.name.is_empty() || cache.info.name == DATA_USAGE_ROOT { return Err(ScannerError::Other("internal error: root scan attempted".to_string())); @@ -4325,19 +4279,6 @@ mod tests { assert!(!scanner.new_cache.info.failed_objects.contains_key("expired")); } - #[test] - fn test_scanner_inline_heal_enabled_defaults_to_false() { - assert!(!scanner_inline_heal_enabled_from_value(None)); - } - - #[test] - fn test_scanner_inline_heal_enabled_reads_env_override() { - assert!(scanner_inline_heal_enabled_from_value(Some("true"))); - assert!(scanner_inline_heal_enabled_from_value(Some("YES"))); - assert!(scanner_inline_heal_enabled_from_value(Some("1"))); - assert!(!scanner_inline_heal_enabled_from_value(Some("false"))); - } - #[test] fn test_build_object_heal_request_omits_nil_version_id() { let request = build_object_heal_request( diff --git a/docs/operations/heal-scanner-parity-notes-zh.md b/docs/operations/heal-scanner-parity-notes-zh.md new file mode 100644 index 000000000..027926747 --- /dev/null +++ b/docs/operations/heal-scanner-parity-notes-zh.md @@ -0,0 +1,109 @@ +# Heal/Scanner ้…็ฝฎไธŽ่ฏญไน‰ๅฏน็…ง๏ผˆMinIO parity ๅ†ณ็ญ–่ฎฐๅฝ•๏ผ‰ + +ๅฏนๅบ” backlog rustfs/backlog#1878๏ผˆ็ˆถ #1862๏ผŒๆ‰น HS-14/HS-16/HS-18๏ผ‰ใ€‚ๆœฌ้กตๆฒ‰ๆท€ไธ‰้กน"ๅ†ณ็ญ– + ๆ–‡ๆกฃๅŒ–"็ป“่ฎบ๏ผšscanner idle ่Š‚ๆต่ฏญไน‰ๅฏน็…งไธŽ่ฟ็งป่ญฆๅ‘Š๏ผˆHS-14๏ผ‰ใ€ๅ•ๆœบ้ป˜่ฎคๆ‰ซๆๅ‘จๆœŸๅ†ณ็ญ–๏ผˆHS-16๏ผ‰ใ€stale multipart ไธŽ tmp/.trash ๆธ…็†ไธ‰ๆฎตๆ ธๅฏน๏ผˆHS-18๏ผ‰๏ผŒๅนถ้กบๅธฆๆ”ถๅฝ• bitrot_cycle ไธŽ alert_excess_folders ไธค้กนๅทฒ็กฎ่ฎค็š„้ป˜่ฎคๅ€ผๅทฎๅผ‚ใ€‚ๆ‰€ๆœ‰ MinIO ไพง็ป“่ฎบๅ‡ไบŽ 2026-08 ๆŒ‰ minio/minio master ้€ๆบ็ ๆ ธๅฏน๏ผˆๅผ•็”จๆ–‡ไปถไธบไธŠๆธธ่ทฏๅพ„๏ผ‰๏ผŒไธ่ฝฌ่ฟฐไบŒๆ‰‹่ต„ๆ–™ใ€‚ + +่ฟ่กŒๆ—ถๆ—‹้’ฎ็š„ๅฎŒๆ•ดๆธ…ๅ•ใ€็Šถๆ€็ซฏ็‚นไธŽ่ฐƒๅ‚ๆต็จ‹่ง [Scanner Runtime Controls](scanner-runtime-controls.md)๏ผ›excess ๅ‘Š่ญฆ้˜ˆๅ€ผๅทฎๅผ‚่ง [Scanner Excess Alerts](scanner-excess-alerts_zh.md)๏ผ›heal ๅนถๅ‘ๆจกๅž‹ๅฏน็…ง่ง [Heal ๅนถๅ‘ๅฎ‰ๅ…จ่ฏดๆ˜Ž](heal-concurrency-safety-notes-zh.md)ใ€‚ + +## 1. HS-14๏ผšscanner idle ่Š‚ๆต่ฏญไน‰ๅฏน็…ง + +### RustFS ๅฝ“ๅ‰่ฏญไน‰๏ผˆไธ‰ๅ› ๅญ๏ผ‰ + +RustFS ็š„ scanner ๆญฅ่ฟ›่Š‚ๆต็”ฑไธ‰ไธชๅ› ๅญๅ…ฑๅŒๅ†ณๅฎš๏ผˆcrates/scanner/src/sleeper.rs๏ผ‰๏ผš + +1. **ๆ€ป้—ธ `scanner.idle_mode` / `RUSTFS_SCANNER_IDLE_MODE`๏ผˆ้ป˜่ฎค `true`๏ผ‰**๏ผš`false` ๆ—ถๆ‰€ๆœ‰่Š‚ๆต sleep ๅ…จ้ƒจ่ทณ่ฟ‡๏ผŒscanner ๅ…จ้€ŸๆŽจ่ฟ›๏ผ›`true` ๆ—ถๆŒ‰ไธ‹้ขไธคๅ› ๅญ่ฎก็ฎ— sleepใ€‚ +2. **้€Ÿๅบฆๆกฃ**๏ผˆ`scanner.speed` / `RUSTFS_SCANNER_SPEED`๏ผŒ้ป˜่ฎค `default`๏ผ‰๏ผšๆกฃไฝ่กจไธŽ MinIO ๅฎŒๅ…จไธ€่‡ด๏ผˆ่งไธ‹่กจ๏ผ‰ใ€‚็›ฎๅฝ•็บง sleep = `1ms ร— factor`๏ผˆไธŠ้™ `max_wait`๏ผ‰๏ผ›ๅฏน่ฑก็บง sleep = `ๆœฌๅฏน่ฑกๅค„็†่€—ๆ—ถ ร— factor`๏ผŒไธ‹้™ 1msใ€ไธŠ้™ `max_wait`ใ€‚ +3. **ๅ‰ๅฐ่ฏป้€€้ฟไธ‹้™**๏ผš`current_foreground_read_activity()` ๅ–ๅนถๅ‘ GetObject ่ฏทๆฑ‚ๆ•ฐ๏ผˆrustfs/src/storage/concurrency/request_guard.rs ็š„ `GetObjectGuard`๏ผ‰ไธŽๆตๅผ่ฏป่ฎกๆ•ฐ๏ผˆ`ForegroundReadGuard`๏ผ‰็š„่พƒๅคงๅ€ผ๏ผŒๆข็ฎ—ไธบ `10ms ร— ๆดป่ทƒ่ฏปๆ•ฐ`ใ€ๅฐ้กถ 250ms ็š„ไธ‹้™๏ผ›่ฏฅไธ‹้™ๅฏน็›ฎๅฝ•็บงไธŽๅฏน่ฑก็บง sleep ้ƒฝ็”Ÿๆ•ˆ๏ผˆ`.max(foreground_sleep)`๏ผ‰๏ผŒไธ”**ๅฏไปฅ่ถ…่ฟ‡้€Ÿๅบฆๆกฃ็š„ `max_wait`**๏ผˆ่‡ช่บซๅฐ้กถ 250ms๏ผ‰ใ€‚้€Ÿๅบฆๆกฃไธบ `fastest`๏ผˆfactor=0๏ผ‰ๆ—ถ้ข„่ฎพ sleep ไธบ 0๏ผŒไฝ†ๅช่ฆ `idle_mode=true`๏ผŒๅ‰ๅฐ่ฏปไธ‹้™ไป็„ถ็”Ÿๆ•ˆใ€‚ + +| ้€Ÿๅบฆๆกฃ | sleep factor | ๅ•ๆฌก sleep ไธŠ้™ | ๅ‘จๆœŸ้—ด้š” | +|---|---:|---:|---:| +| `fastest` | 0 | 0 | 1s | +| `fast` | 1ร— | 100ms | 1m | +| `default` | 2ร— | 1s | 1m | +| `slow` | 10ร— | 15s | 1m | +| `slowest` | 100ร— | 15s | 30m | + +ๅฎž้™…่กŒไธบ็Ÿฉ้˜ต๏ผˆRustFS๏ผ‰๏ผš + +| `idle_mode` | ้€Ÿๅบฆๆกฃ | ๅ‰ๅฐๅนถๅ‘่ฏป = 0 | ๅ‰ๅฐๅนถๅ‘่ฏป > 0 | +|---|---|---|---| +| `false` | ไปปๆ„ | ๅฎŒๅ…จไธไผ‘็œ ๏ผŒๅ…จ้€Ÿ | ๅฎŒๅ…จไธไผ‘็œ ๏ผŒๅ…จ้€Ÿ๏ผˆๅ‰ๅฐ้€€้ฟไนŸ่ขซๆ€ป้—ธๅ…ณ้—ญ๏ผ‰ | +| `true` | `fastest` | ้ข„่ฎพ sleep = 0๏ผŒ็ญ‰ๆ•ˆๅ…จ้€Ÿ | ๆฏๆญฅ sleep = ๅ‰ๅฐ่ฏปไธ‹้™๏ผˆ10msร—่ฏปๆ•ฐ๏ผŒๅฐ้กถ 250ms๏ผ‰ | +| `true` | ๅ…ถไฝ™ๆกฃ | ๆฏๆญฅ sleep = ้ข„่ฎพๅ€ผ๏ผˆ1ms~15s ๅฐ้กถ๏ผ‰ | ๆฏๆญฅ sleep = max(้ข„่ฎพๅ€ผ, ๅ‰ๅฐ่ฏปไธ‹้™) | + +ๅ‘จๆœŸ้—ด้š”็š„่งฃๆžไผ˜ๅ…ˆ็บงไธบ env `RUSTFS_SCANNER_CYCLE` > ๆŒไน…ๅŒ– `scanner.cycle` > `scanner.start_delay` > ๅฏๅŠจๆœŸ้ป˜่ฎค่ฆ†็›–๏ผˆๅฝ“ๅ‰ๆ’ๆ— ๏ผ‰> ้€Ÿๅบฆๆกฃๆดพ็”Ÿ๏ผˆcrates/scanner/src/runtime_config.rs๏ผ‰ใ€‚ๅฆๆœ‰ `scanner.yield_every_n_objects`๏ผˆ้ป˜่ฎค 128๏ผ‰็š„ๅไฝœๅผ่ฎฉๅ‡บ๏ผŒไธŽ่Š‚ๆต sleep ็›ธไบ’็‹ฌ็ซ‹ใ€‚ + +### MinIO ๅฝ“ๅ‰่ฏญไน‰๏ผˆmaster ้€ๆบ็ ๆ ธๅฏน๏ผ‰ + +MinIO ็š„ๅฏนๅบ”ๅผ€ๅ…ณๆ˜ฏ `scanner:idle_speed` / `MINIO_SCANNER_IDLE_SPEED`๏ผˆinternal/config/scanner/scanner.go๏ผ‰๏ผšๅ–ๅ€ผไธบ็ฉบไธฒๆˆ– `on`๏ผˆ้ป˜่ฎค๏ผ‰ๆ—ถ `IdleMode=0`๏ผŒๅ–ๅ€ผ `off` ๆ—ถ `IdleMode=1`ใ€‚ๅฏๅŠจ/้…็ฝฎๅŠ ่ฝฝๆ—ถไธ€ๆฌกๆ€งๅ†™ๅ…ฅ `scannerIdleMode`๏ผˆcmd/config-current.go๏ผ‰๏ผŒๆ‰ซๆไพง้—ญๅŒ… `weSleep = scannerIdleMode.Load() == 0`๏ผˆcmd/xl-storage-disk-id-check.go๏ผ‰๏ผš**`on`๏ผˆ้ป˜่ฎค๏ผ‰= ็›ฎๅฝ•็บงไธŽๅฏน่ฑก็บง่Š‚ๆต sleep ๅง‹็ปˆๆ’ๅ…ฅ๏ผˆๆŒ‰้€Ÿๅบฆๆกฃ factor๏ผŒminSleep 100ยตs๏ผ‰๏ผ›`off` = ไธคๆก่Š‚ๆต่ทฏๅพ„ๅฎŒๅ…จไธ sleep๏ผŒๅ…จ้€Ÿๆ‰ซๆ**ใ€‚ๅฝ“ๅ‰ไธŠๆธธๆฒกๆœ‰ไปปไฝ•ๆŒ‰ S3 ่ฏทๆฑ‚/็ฃ็›˜ๆดปๅŠจๅŠจๆ€่ฐƒๆ•ด่Š‚ๆต็š„้€ป่พ‘โ€”โ€”่ฟ™ๆ˜ฏ้™ๆ€ๅผ€ๅ…ณใ€‚ + +ๅ‘ฝๅๅ…ทๆœ‰่ฏฏๅฏผๆ€ง๏ผŒๆ˜ฏๅŽ†ๅฒๆฎ‹็•™๏ผš2024-01 ไน‹ๅ‰ `weSleep` ็”ฑ็ฃ็›˜ๆดปๅŠจ้ฉฑๅŠจ๏ผˆ"Entire queue is full, so we sleep"๏ผŒๅณๆœ‰ๅนถๅ‘ S3/heal ๆดปๅŠจๆ‰ sleep๏ผ‰๏ผŒminio/minio#18734๏ผˆcommit 7705605b๏ผ‰ๆŠŠ่ฏฅๆดปๅŠจ้—จๆ›ฟๆขไธบไธŠ่ฟฐ้™ๆ€้…็ฝฎ๏ผˆๅˆ็‰ˆๅ–ๅ€ผ `throttled`/`full`๏ผŒๅŽๆ”นไธบ `on`/`off`๏ผ‰๏ผŒไธŠๆธธๆฎ‹็•™ๆณจ้‡Š "default is throttled when idle"ใ€"Sleep always or based on incoming S3 requests" ๅ‡ๆ˜ฏๆ›ฟๆขๅ‰็š„่ฏญไน‰ๆ่ฟฐ๏ผŒไธŽ็Žฐ่กŒไปฃ็ ไธ็ฌฆใ€‚ + +### ๅฏน็…งไธŽ่ฟ็งป่ญฆๅ‘Š + +| ็ปดๅบฆ | RustFS | MinIO๏ผˆmaster๏ผ‰ | +|---|---|---| +| ๅผ€ๅ…ณๅ | `scanner.idle_mode` / `RUSTFS_SCANNER_IDLE_MODE` | `scanner:idle_speed` / `MINIO_SCANNER_IDLE_SPEED` | +| ๅ–ๅ€ผ | ๅธƒๅฐ” `true`/`false` | `on`/`off` | +| ้ป˜่ฎค | `true`๏ผˆ่Š‚ๆตๅผ€ๅฏ๏ผ‰ | `on`๏ผˆ่Š‚ๆตๅผ€ๅฏ๏ผ‰ | +| ๅผ€ = | ่Š‚ๆตๆ€ป้—ธๅผ€๏ผš้€Ÿๅบฆๆกฃ sleep + ๅ‰ๅฐ่ฏปไธ‹้™ | ่Š‚ๆตๆ€ป้—ธๅผ€๏ผš้€Ÿๅบฆๆกฃ sleep | +| ๅ…ณ = | ๅฎŒๅ…จไธไผ‘็œ ๏ผˆๅซๅ‰ๅฐ่ฏปไธ‹้™ไธ€ๅนถๅคฑๆ•ˆ๏ผ‰ | ๅฎŒๅ…จไธไผ‘็œ  | +| ๆดปๅŠจ่€ฆๅˆ | ๆœ‰๏ผšๅ‰ๅฐๅนถๅ‘่ฏปๆŠฌ้ซ˜ sleep ไธ‹้™๏ผˆ10msร—่ฏปๆ•ฐ๏ผŒๅฐ้กถ 250ms๏ผ‰ | ๆ— ๏ผˆ2024-01 ่ตทไธบ้™ๆ€ๅผ€ๅ…ณ๏ผ‰ | +| ้€Ÿๅบฆๆกฃ่กจ | ไธค่พนๅฎŒๅ…จไธ€่‡ด๏ผˆไธŠ่กจ๏ผ‰ | ๅŒๅทฆ | + +่ฟ็งป่ญฆๅ‘Š๏ผš + +- **็Žฏๅขƒๅ˜้‡ๅไธๅฏ็…งๆฌ**๏ผšRustFS ๅช่ฏปๅ– `RUSTFS_*` ๅ‰็ผ€๏ผŒไธ่งฃๆž `MINIO_SCANNER_*` ไปปไฝ•ๅˆซๅ๏ผˆcrates/scannerใ€crates/utils ็š„ env ่ฏปๅ–ๆ— ๅˆซๅ้“พ๏ผŒๆต‹่ฏ•่ฟ˜ไธ“้—จๆ–ญ่จ€ `MINIO_SCANNER_SPEED`/`MINIO_SCANNER_CYCLE` ไธๆณ„ๆผ็”Ÿๆ•ˆ๏ผ‰ใ€‚็…งๆฌ `MINIO_SCANNER_IDLE_SPEED=off` ๅˆฐ RustFS ไผš้™้ป˜ๆ— ๆ•ˆ๏ผŒๅฟ…้กปๆ”นๅ†™ๆˆ `RUSTFS_SCANNER_IDLE_MODE=false`ใ€‚ +- **ๅ–ๅ€ผ่ฏ่กจไธๅŒ**๏ผš`on/off` vs `true/false`๏ผŒไธ่ƒฝๅŽŸๆ ทๅคๅˆถใ€‚ +- **ๆ–นๅ‘ๆพ„ๆธ…๏ผˆไฟฎๆญฃ็ˆถ issue ็š„้ข„่ฎพ๏ผ‰**๏ผšๆŒ‰ๅฝ“ๅ‰ไธŠๆธธๆบ็ ๏ผŒMinIO `idle_speed` ไธŽ RustFS `idle_mode` ๅœจ"ๅผ€=่Š‚ๆตใ€ๅ…ณ=ๅ…จ้€Ÿ"ๆ–นๅ‘ไธŠๆ˜ฏไธ€่‡ด็š„๏ผŒๅนถ้žๅๅ‘๏ผ›็ˆถ issue ไธญ"MinIO on=้›†็พค็ฉบ้—ฒๆ‰่Š‚ๆตใ€off=ๅง‹็ปˆๆŒ‰ delay ่Š‚ๆต"็š„็Ÿฉ้˜ตๆ่ฟฐ็š„ๆ˜ฏ 2024-01 ไน‹ๅ‰็š„ๆดปๅŠจ่€ฆๅˆ่กŒไธบไธŽๅๅ‘่งฃ่ฏป๏ผŒไธŽ master ไธ็ฌฆใ€‚็œŸๆญฃ้œ€่ฆๅ†™่ฟ›่ฟ็งปๆ‰‹ๅ†Œ็š„ๅทฎๅผ‚ๆ˜ฏ๏ผšMinIO ็š„ `idle_speed` ๅ็งฐๆš—็คบ"็ฉบ้—ฒๆ—ถๆ‰ๆ…ข"ไฝ†ๅฎž้™…ๆ˜ฏ้™ๆ€ๆ€ป้—ธ๏ผ›RustFS ็š„ `idle_mode=true` ๅœจๆ€ป้—ธไน‹ไธŠ่ฟ˜ๅ ๅŠ ไบ† MinIO ๆฒกๆœ‰็š„ๅ‰ๅฐ่ฏปไฟๆŠคไธ‹้™ใ€‚ +- **`false` ๆ˜ฏๅคง้”ค**๏ผšRustFS `idle_mode=false` ไผš่ฟžๅ‰ๅฐ่ฏป้€€้ฟไธ€่ตทๅ…ณ้—ญ๏ผŒscanner ไธŽๅ‰ๅฐ่ฏปๅฎŒๅ…จๆŠข็›˜๏ผ›ไป…ๅœจ benchmark ๆˆ–ๅฏ็‹ฌๅ  IO ็š„็ช—ๅฃไฝฟ็”จใ€‚ + +**ๅ†ณ็ญ–๏ผˆHS-14๏ผ‰๏ผšไฟๆŒ็Žฐ็Šถใ€‚** RustFS ่ฏญไน‰ๆ›ด็›ด่ง‚๏ผˆ`idle_mode` = ่Š‚ๆตๆ€ป้—ธ๏ผŒ`true` ๅณ่‡ช้€‚ๅบ”้™้€Ÿ๏ผ‰๏ผŒไธ”ๆฏ” MinIO ๅคšไธ€ๅฑ‚ๅ‰ๅฐ่ฏปไฟๆŠค๏ผ›ไธๆ–ฐๅขž `RUSTFS_SCANNER_IDLE_SPEED` ๅ…ผๅฎนๅˆซๅ๏ผˆๆ— ็คพๅŒบๅผบ่ฏ‰ๆฑ‚ไธๅš๏ผŒ้ฟๅ…ๅŒๅ…ฅๅฃๆผ‚็งป๏ผ‰ใ€‚ๆœฌ่Š‚ๅณๅฏน็…ง่กจไธŽ่ฟ็งป่ญฆๅ‘Š็š„ๆญฃๅผ่ฝ็‚นใ€‚ + +## 2. bitrot_cycle ้ป˜่ฎคๅทฎๅผ‚ + +| ้กน | RustFS | MinIO | +|---|---|---| +| ้”ฎ | `heal.bitrot_cycle` / `RUSTFS_SCANNER_BITROT_CYCLE_SECS`๏ผˆscanner.bitrot_cycle ไธบๅ…ผๅฎนๆ—ง้”ฎ๏ผ‰ | `heal:bitrotscan` / `MINIO_HEAL_BITROTSCAN` | +| ้ป˜่ฎค | 30 ๅคฉ๏ผˆcrates/config/src/constants/heal.rs ็š„ `DEFAULT_HEAL_BITROT_CYCLE_SECS`๏ผ‰๏ผšๆŒ‰ๅข™้’Ÿๅ‘จๆœŸๆŠŠๆ‰ซๆๅˆ‡ๆทฑๆ‰ซ๏ผˆdeep bitrot๏ผ‰ | `off`๏ผˆinternal/config/heal/heal.go ้ป˜่ฎค `EnableOff`๏ผ‰๏ผšไธๅšๅ‘จๆœŸๆ€งๆทฑๆ‰ซ๏ผŒไป…ๆ™ฎ้€šๆ‰ซๆ + ็ฎก็†็ซฏๆ‰‹ๅŠจๆทฑๆ‰ซ | +| ๅฏน้ฝๆ–นๅผ | ่ฟ็งป MinIO ่กŒไธบ๏ผš`heal.bitrot_cycle=off` ๆˆ– `RUSTFS_SCANNER_BITROT_CYCLE_SECS=disabled` | ๅๅ‘๏ผš`heal:bitrotscan=<็ง’>` | + +RustFS ็š„ 30 ๅคฉ้ป˜่ฎคๆ˜ฏๅˆปๆ„็š„่€็”จๆ€ง้ป˜่ฎค๏ผˆๅ‘จๆœŸๆ€งๅ…จ้‡ bitrot ๆ ก้ชŒ๏ผ‰๏ผŒไปฃไปทๆ˜ฏๆฏ 30 ๅคฉไธ€่ฝฎๆทฑๆ‰ซ IO๏ผ›ๅ•ๆœบๅœบๆ™ฏๅฆๆœ‰ๆธ…ๆด็ฉบ้—ฒ้€€้ฟๅฐ้กถ็บฆ 42 ๅˆ†้’Ÿ็š„ๅข™้’ŸไฟๆŠค๏ผˆ่ง scanner-runtime-controls.md๏ผ‰ใ€‚่ฟ™ๆ˜ฏ่กŒไธบๅทฎๅผ‚่€Œ้ž็ผบ้™ท๏ผŒๆ–‡ๆกฃๅŒ–ๅณๅฏใ€‚ + +## 3. alert_excess_folders ้ป˜่ฎคๅทฎๅผ‚ + +RustFS ้ป˜่ฎค 65538๏ผˆๅฎน็บณ Proxmox Backup Server ๆฏ็›ฎๅฝ• 65536 chunk ็š„ๅธƒๅฑ€๏ผ‰๏ผŒMinIO ้ป˜่ฎค 50000ใ€‚ๅทฎๅผ‚ๅŽŸๅ› ใ€ๅฆไธคไธช excess ้˜ˆๅ€ผ๏ผˆversions=100 ็›ธๅŒใ€version_size TiB vs TB๏ผ‰ใ€ไบ‹ไปถๅๆ˜ ๅฐ„ไธŽๅ†ทๅด่ฏญไน‰ๅทฒๅฎŒๆ•ด่ฎฐๅฝ•ๅœจ [Scanner Excess Alerts](scanner-excess-alerts_zh.md)๏ผŒๆญคๅค„ไธ้‡ๅคใ€‚ + +## 4. HS-18๏ผšstale multipart ไธŽ tmp/.trash ๆธ…็†ไธ‰ๆฎตๆ ธๅฏน + +MinIO ๆŠŠ"ๆธ…็†ๅทฒๅˆ ้™คๆ•ฐๆฎ"ๆ‹†ๆˆไธ‰ๆฎต๏ผšstale upload ๅ…ˆ rename ่ฟ› `.minio.sys/tmp/.trash/` ้š”็ฆป๏ผˆrename ๅฟซใ€ๅŽŸๅญ๏ผ‰๏ผ›trash ็”ฑ็‹ฌ็ซ‹ไพ‹็จ‹ๆŽ’็ฉบ๏ผ›tmp ไธ‹้ž trash ็š„ๆ—ง็›ฎๅฝ•ๅ•็‹ฌๅ›žๆ”ถใ€‚้€ๆฎตๆ ธๅฏน RustFS๏ผš + +| ๆฎต | MinIO | RustFS | ๅˆคๅฎš | +|---|---|---|---| +| stale multipart โ†’ ้š”็ฆป | `cleanupStaleUploadsOnDisk`๏ผˆcmd/erasure-multipart.go๏ผ‰้€็›˜ๅˆ—ๅ‡บ multipart ็›ฎๅฝ•๏ผŒๆŒ‰ uploadID ็›ฎๅฝ•ๅ้‡Œ็š„ UnixNano ๅˆค้พ„๏ผŒ่ถ…่ฟ‡ `stale_uploads_expiry`๏ผˆ้ป˜่ฎค 24h๏ผ‰ๅณ `renameAll` ่ฟ› `.minio.sys/tmp/.trash/`๏ผŒ็ฉบ sha ็›ฎๅฝ•ใ€tmp ๆ—ง็›ฎๅฝ•ๅŒๆณ• | `cleanup_stale_multipart_uploads_in_set`๏ผˆcrates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs๏ผ‰ๅ‘็Žฐๅ€™้€‰ๅŽๅ– ns ๅ†™้” + ้‡ๆŸฅ๏ผˆ`lock_stale_multipart_cleanup`๏ผ‰๏ผŒ`delete_all_with_quorum` ๆ‰‡ๅ‡บ้€็›˜้€’ๅฝ’ๅˆ ้™ค๏ผŒ่€Œ LocalDisk ็š„้€’ๅฝ’ๅˆ ้™คๅ†…้ƒจๅฐฑๆ˜ฏ `move_to_trash`๏ผˆcrates/ecstore/src/disk/local.rs๏ผ‰ๆŠŠ็›ฎๅฝ• rename ่ฟ› `.rustfs.sys/tmp/.trash/` | ่กŒไธบ็ญ‰ไปท๏ผˆ้ƒฝๆ˜ฏๅ…ˆ้š”็ฆปๅŽๆธ…็†๏ผ‰๏ผ›RustFS ้ขๅค–ๆœ‰ๅ†™้” + quorum ้‡ๆŸฅ + ้”ไธขๅคฑ fence๏ผˆcrates/ecstore/src/set_disk/ops/multipart.rs ็š„ `StaleMultipartCleanupGuard`๏ผ‰๏ผŒ้˜ฒๅนถๅ‘ CompleteMultipartUpload ็ซžไบ‰๏ผŒๅฎ‰ๅ…จๆ€งๅผบไบŽ MinIO ็š„ๆ— ้” rename | +| trash ๆŽ’็ฉบ | ๆฏ `delete_cleanup_interval`๏ผˆ้ป˜่ฎค 5m๏ผŒinternal/config/api/api.go๏ผ‰้€็›˜ๅˆ  `.trash` ๅ†…ๆก็›ฎ๏ผŒ้€ๆกไปฅ `deleteCleanupSleeper`๏ผˆfactor 5 / 25ms๏ผŒcmd/globals.go๏ผ‰่Š‚ๆต | ๆฏ็›˜็‹ฌ็ซ‹ `cleanup_deleted_objects_loop`๏ผŒ`DELETED_OBJECTS_CLEANUP_INTERVAL` = 5m๏ผˆcrates/ecstore/src/disk/local.rs๏ผ‰๏ผŒๅ…ˆๆŽ’็ฉบ `.trash` ๅ†ๅ›žๆ”ถ tmp ๆ—ง็›ฎๅฝ•๏ผ›ๆŽ’็ฉบไธบ้กบๅบ `remove_dir_all`/`remove_file`๏ผŒ**ๆ— ้€ๆก sleep ่Š‚ๆต** | ๅŸบๆœฌ็ญ‰ไปท๏ผ›ๅ”ฏไธ€ๅทฎๅผ‚ๆ˜ฏ RustFS ๆŽ’็ฉบไธ่Š‚ๆต๏ผŒtrash ็งฏๅŽ‹ๅคงๆ—ถๅ•่ฝฎ IO ๆ›ด็ชๅ‘๏ผˆ5m ๅ‘จๆœŸๅคฉ็„ถ้™้ข‘๏ผ‰๏ผŒๆ–‡ๆกฃๅŒ–๏ผŒๅฆ‚ๅฎžๆต‹ๅ‡บ็Žฐๆธ…็†้ฃŽๆšดๅ†่กฅ่Š‚ๆต | +| tmp ้ž trash ๆ—ง็›ฎๅฝ• | ๅนถๅœจ `cleanupStaleUploadsOnDisk` ๅ†…๏ผš้ž `.trash` ็š„ tmp ็›ฎๅฝ•่ถ…่ฟ‡ `stale_uploads_expiry`๏ผˆ24h๏ผ‰rename ่ฟ› trash๏ผˆ้š 6h ไปปๅŠก๏ผ‰ | `cleanup_stale_tmp_objects`๏ผˆcrates/ecstore/src/disk/local.rs๏ผ‰้š 5m ๅพช็Žฏๆ‰ง่กŒ๏ผš้ž `.trash` ็›ฎๅฝ•่ถ…่ฟ‡ `STALE_TMP_OBJECT_EXPIRY` = 24h ๅณ rename ่ฟ› trash๏ผ›ๅฆๆœ‰ๅฏๅŠจๆ—ถ tmp โ†’ tmp-old ๆ•ดไฝ“ๆขๅ + ๅŽๅฐๅˆ ้™ค็š„ๅดฉๆบƒๅฎ‰ๅ…จ่ทฏๅพ„ | ่กŒไธบ็ญ‰ไปท๏ผˆ้˜ˆๅ€ผๅŒไธบ 24h๏ผ‰๏ผ›RustFS ๆฃ€ๆŸฅ้ข‘็Ž‡ 5m vs MinIO 6h๏ผŒๅ›žๆ”ถๆ›ดๅŠๆ—ถ | + +ๅ‘จๆœŸไธŽ็Žฏๅขƒๅ˜้‡้ป˜่ฎคๅ€ผๅฏน็…ง๏ผˆไธค่พนไธ€่‡ด๏ผ‰๏ผš + +| ้กน | RustFS | MinIO | +|---|---|---| +| stale upload ่ฟ‡ๆœŸ้˜ˆๅ€ผ | `RUSTFS_API_STALE_UPLOADS_EXPIRY`๏ผŒ้ป˜่ฎค 24h | `MINIO_API_STALE_UPLOADS_EXPIRY`๏ผŒ้ป˜่ฎค 24h | +| stale multipart ๆธ…็†ๅ‘จๆœŸ | `RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL`๏ผŒ้ป˜่ฎค 6h | `MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL`๏ผŒ้ป˜่ฎค 6h | +| trash ๆŽ’็ฉบๅ‘จๆœŸ | 5m๏ผˆๅธธ้‡๏ผŒๆš‚ๆ— ๅผ€ๅ…ณ๏ผ‰ | `MINIO_API_DELETE_CLEANUP_INTERVAL`๏ผŒ้ป˜่ฎค 5m | + +ๅ…ณไบŽ rustfs/src/delete_tail_activity.rs๏ผšๅฎƒ**ไธ่ฆ†็›–ไธ‰ๆฎตไธญ็š„ไปปไฝ•ไธ€ๆฎต**ใ€‚่ฏฅๆจกๅ—ๆ˜ฏ delete ๅฐพ้ƒจๆดปๅŠจ็š„่ฟ›็จ‹ๅ†…ๆŒ‡ๆ ‡่ฎกๆ•ฐ๏ผˆinflight gauge + ่€—ๆ—ถ histogram๏ผ‰๏ผŒไพ› allocator ๅ›žๆ”ถๅŽ‹ๅŠ›ๅˆคๆ–ญ๏ผˆrustfs/src/allocator_reclaim.rs๏ผ‰ไฝฟ็”จ๏ผ›็”Ÿไบงไปฃ็ ็›ฎๅ‰ๅชๅœจๅฏน่ฑกๅค็”จ่ทฏๅพ„ไฝฟ็”จ `Replication`/`Notify` ไธคไธช stage ่ฎกๆ•ฐ๏ผŒ`Tail`/`Cleanup` ๆžšไธพๅ€ผๆš‚ๆ— ่ฐƒ็”จ็‚นใ€‚ + +ๅดฉๆบƒๆฎ‹็•™็ช—ๅฃ็ป“่ฎบ๏ผš + +- trash ๅ†…้ƒจๆฎ‹็•™๏ผˆๆŽ’็ฉบไธญ้€”ๅดฉๆบƒ๏ผ‰๏ผš`.trash/` ๆ˜ฏ่‡ชๅŒ…ๅซ็›ฎๅฝ•๏ผŒไธ‹ไธ€่ฝฎ 5m tick ้‡ๆ‰ซ `.trash` ่‡ช็„ถๆ”ถๆ•›๏ผŒไธŽ MinIO ็›ธๅŒใ€‚ +- ่ทจ็›˜ๆ‰‡ๅ‡บไธญ้€”ๅดฉๆบƒ๏ผˆ้ƒจๅˆ†็›˜ๅทฒ rename ่ฟ› trashใ€ๅ…ถไฝ™ๆœชๅŠจ๏ผ‰๏ผš่‹ฅๅ‰ฉไฝ™็›˜ๆ•ฐไปๆปก่ถณๅ†™ quorum๏ผŒไธ‹ไธ€่ฝฎ 6h ไปปๅŠก้‡ๆ–ฐๅ‘็Žฐๅ€™้€‰ๅนถ้‡ๅˆ ๏ผŒ่‡ช็„ถๆ”ถๆ•›๏ผ›่‹ฅๅทฒๆธ…็†็›˜ๆ•ฐ่ถ…่ฟ‡ parity๏ผˆๅ‰ฉไฝ™ไฝŽไบŽๅ†™ quorum๏ผ‰๏ผŒ`check_multipart_upload_path_exists` ๅ›  `FileNotFound` ไธๅœจ `OBJECT_OP_IGNORED_ERRS`๏ผˆcrates/ecstore/src/disk/error_reduce.rs๏ผ‰่€Œๅˆค quorum ๅคฑ่ดฅ๏ผŒๅ€™้€‰่ขซ่ทณ่ฟ‡๏ผŒๆฎ‹็•™ uploadID ็›ฎๅฝ•ไธไผš่ขซ่ฏฅไปปๅŠกๆ”ถๆ•›๏ผˆไธๅฏ่งไบŽ S3 API๏ผŒไป…ๅ ็›˜็ฉบ้—ด๏ผ‰ใ€‚่ฏฅ็ช—ๅฃๆž็ช„๏ผˆ้€็›˜ rename ไธบๆฏซ็ง’็บง๏ผŒ้œ€ๆฐๅœจๆ‰‡ๅ‡บไธญ้€”ไธ”ๅทฒ่ฟ‡ parity ็›˜ๆ—ถ่ฟ›็จ‹ๆญปไบก๏ผ‰ใ€‚MinIO ๅŒๅœบๆ™ฏไผšๆ”ถๆ•›๏ผˆ้€็›˜็‹ฌ็ซ‹ๅค„็†ใ€ๆ—  quorum ้—ธ้—จ๏ผ‰ใ€‚**ๅˆ†็บง๏ผšๆœ‰ๅดฉๆบƒๆฎ‹็•™็ช—ๅฃ๏ผˆๆž็ช„๏ผ‰โ†’ ็™ป่ฎฐๅŽ็ปญไฟฎๅค**๏ผ›ไฟฎๅค้œ€ไธบๆธ…็†ๅฎˆๅซๆไพ›ๆŠŠ"ๅทฒไธๅญ˜ๅœจ"่ฎกไธบ่พพๆˆ็ปˆๆ€็š„ไธ“็”จ quorum ๅ˜ไฝ“๏ผˆไธ่ƒฝๆ”นๅ…ฑไบซ็š„ `check_multipart_upload_path_exists` ่ฏญไน‰๏ผŒๅฎƒๅŒๆ—ถๆœๅŠก CompleteMultipartUpload๏ผ‰๏ผŒ่ถ…ๅ‡บๆœฌๆ‰น"ๅ‡ ่กŒๅฐไฟฎ"่พน็•Œ๏ผŒไธๅœจๆœฌ PR ๆ‰ฉ scopeใ€‚ + +## 5. HS-16๏ผšๅ•ๆœบ๏ผˆErasureSD๏ผ‰้ป˜่ฎคๆ‰ซๆๅ‘จๆœŸๅ†ณ็ญ– + +ๅฏๅŠจๆœŸๆ›พๆœ‰้ข„็•™้’ฉๅญ `single_disk_default_cycle_secs`๏ผŒๅฏๆŒ‰็ปดๆŠค็‰นๅพ๏ผˆlifecycle/replication/ๅทกๆฃ€ๅคฑ่ดฅ๏ผ‰ไธบๅ•ๆœบ่ฆ†็›–้ป˜่ฎคๅ‘จๆœŸ๏ผŒไฝ†ไปŽๆœชๆŽฅ็บฟใ€ๆ’่ฟ”ๅ›ž `None`๏ผŒๅทฒๅˆ ้™ค๏ผˆๆœฌๆ‰น PR๏ผ‰ใ€‚ๅ†ณ็ญ–๏ผš**ๅ•ๆœบ้ป˜่ฎคๅ‘จๆœŸไฟๆŒ้€Ÿๅบฆๆกฃๆดพ็”Ÿ๏ผˆ`default` ๆกฃ = 60s๏ผ‰๏ผŒไธๅš็‰นๆฎŠ่ฆ†็›–**ใ€‚็†็”ฑ๏ผšๅ…ถไธ€๏ผŒๆ— ไปปไฝ•ๅฎžๆต‹ไพๆฎ่กจๆ˜Žๅ•ๆœบๅ†ทๅฏๅŠจ ILM ๅปถ่ฟŸ้œ€่ฆๆ›ด็Ÿญๅ‘จๆœŸ๏ผŒๅ‡ญ็ฉบ็ผฉ็Ÿญๅชไผšๆ”พๅคง็ฉบ้—ฒๆ‰ซๆ้ข‘ๆฌก๏ผ›ๅ…ถไบŒ๏ผŒๅ•ๆœบๅทฒๆœ‰ๆธ…ๆด็ฉบ้—ฒ้€€้ฟ๏ผˆ่ฟž็ปญๅนฒๅ‡€ๅ‘จๆœŸ้—ด้š”็ฟปๅ€๏ผŒ้ป˜่ฎค bitrot ็ช—ๅฃไธ‹ๅฐ้กถ็บฆ 42 ๅˆ†้’Ÿ๏ผŒ่ง scanner-runtime-controls.md๏ผ‰๏ผŒ็ฉบ้—ฒๆ—ถ็š„ๅ‘จๆœŸๅŽ‹ๅŠ›ๅทฒ่ขซๆถˆๅŒ–๏ผ›ๅ…ถไธ‰๏ผŒ่‹ฅ็กฎๆœ‰่ฏ‰ๆฑ‚๏ผŒ็”จๆˆทๅฏ็”จ `RUSTFS_SCANNER_CYCLE` / `scanner.cycle` ๆ˜พๅผ้…็ฝฎ๏ผŒๆ— ้œ€ๅ†…็ฝฎ็‰นๆฎŠ่ทฏๅพ„ใ€‚้œ€่ฆๆ›ดๆฟ€่ฟ›็Ÿญๅ‘จๆœŸ็š„ๅœบๆ™ฏๅบ”ๅ…ˆๆ‹ฟๅฎžๆต‹ๆ•ฐๆฎๅ†่ฎฎใ€‚ + +## 6. ๅ†ณ็ญ–ๆ‘˜่ฆ + +- HS-14๏ผšไฟๆŒ `RUSTFS_SCANNER_IDLE_MODE` ็Žฐ่ฏญไน‰๏ผˆtrue=่Š‚ๆตๆ€ป้—ธ+ๅ‰ๅฐ่ฏปไธ‹้™๏ผŒfalse=ๅ…จ้€Ÿ๏ผ‰๏ผŒๆ–‡ๆกฃๅŒ–ๅฏน็…ง่กจไธŽ่ฟ็งป่ญฆๅ‘Š๏ผŒไธๅšๅ…ผๅฎนๅˆซๅใ€‚ +- HS-16๏ผšๅˆ ้™คๆ’ `None` ็š„ๅ•ๆœบ้ป˜่ฎคๅ‘จๆœŸ้’ฉๅญ๏ผŒๅ•ๆœบๅ‘จๆœŸไฟๆŒ้€Ÿๅบฆๆกฃๆดพ็”Ÿ + ๆธ…ๆด็ฉบ้—ฒ้€€้ฟใ€‚ +- HS-18๏ผšไธ‰ๆฎตๆธ…็†่กŒไธบ็ญ‰ไปท๏ผˆtrash ๆŽ’็ฉบๆ— ้€ๆก่Š‚ๆตใ€tmp ๅ›žๆ”ถ้ข‘็Ž‡ 5m vs 6h ไธคๅค„ๅฐๅทฎๅผ‚ๆ–‡ๆกฃๅŒ–๏ผ‰๏ผ›่ทจ็›˜ๆ‰‡ๅ‡บ็š„ๆž็ช„ๅดฉๆบƒๆฎ‹็•™็ช—ๅฃ็™ป่ฎฐๅŽ็ปญ๏ผ›ๅ‘จๆœŸ้ป˜่ฎคๅ€ผ 24h/6h/5m ไธŽ MinIO ๅฏน้ฝใ€‚ diff --git a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md new file mode 100644 index 000000000..931d020e8 --- /dev/null +++ b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md @@ -0,0 +1,568 @@ +# RustFS heal & scanner vs MinIO โ€” comprehensive parity analysis (v2, 2026-08-16) + +> English | [ไธญๆ–‡็‰ˆ](rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md) + +- Date: 2026-08-16 (based on that day's `main` code; audit HEAD โ‰ˆ `a118d7e4f`) +- Scope: `crates/heal` (src 19,560 lines + tests 2,274 lines), `crates/scanner` (src ~26,000 lines + tests), `crates/data-usage`, the heal/heal_walk/bitrot_self_verify and config parts of `crates/ecstore`, `crates/common/src/heal_channel.rs`, `crates/madmin` (heal/scanner wire types), `rustfs/src` (startup wiring, admin handlers, cluster RPC) +- Parity baseline: minio/minio master (HEAD `7aac2a2c5b`; the repo has entered maintenance mode with master frozen, i.e. its final state) +- Method: four parallel audit tracks (heal crate / scanner crate / ecstore integration layer / MinIO source study), with key conclusions verified by hand one by one (points marked "verified first-hand" below were checked against the source directly) +- This document supersedes `docs/rustfs-heal-scanner-vs-minio-parity-assessment.md` (2026-06-15, v1). Since v1 there have been more than 80 heal/scanner commits (the full automatic drive-replacement healing chain, the resume state machine, making usage convergence authoritative, cluster-level heal coordination, ILM restore semantics, etc.), so v1's feature inventory and gap judgments are comprehensively outdated; v1 conclusions such as "bloom filter missing" were verified this round to be **misjudgments** (see ยง5.4). + +--- + +## 0. Conclusion summary + +1. **Overall verdict: the core functional chains of heal and scanner are complete.** Object-level heal (quorum arbitration + ETag fallback + bitrot Deep verification + dangling handling), erasure set deep scans (per-set disk-walk union enumeration), per-version resumable scans (schema'd persistence layer + CAS atomic publish + crash-window backfill), automatic drive-replacement healing (readiness validation + identity fencing + durable intent + completion proof), the scanner cycle loop (leader lock + persisted leader-epoch fence), data usage statistics (bucket-level/cluster-level, primary + backup + observed snapshots, epoch/cycle anti-rollback), the full ILM action set (expiry/transition/noncurrent/free-version/delete-marker cleanup), and the admin Start/Query/Cancel protocol (clientToken semantics aligned with madmin) โ€” all of these are implemented and carry regression tests. There are **no empty implementations / early-return stubs** inside the two crates; every exceptional path has logs + metrics + error semantics. +2. **The main gaps concentrate on "entry points and the observability surface", not on the repair algorithms themselves**: the MRF/ECDecode/Metadata task executors are implemented but have no production trigger entry (`HealEvent` is entirely unwired); `CheckAbandonedParts` is `NotImplemented` at all three ecstore layers; the heal/scanner trace channels are missing; scanner excess S3 events are missing; madmin client methods are missing (only wire types exist); heal byte-level progress/ETA is not implemented. +3. **Important corrections to the v1 understanding**: the bloom filter has been **removed** from current MinIO master (`.bloomcycle.bin` stores only a cycle count), so RustFS's current state matches MinIO; the MinIO scanner is likewise a **cluster-level leader singleton**, and RustFS's leader.lock model is the same shape as MinIO's; RustFS's ETag majority-fallback arbitration is already implemented (`crates/ecstore/src/set_disk/ops/heal.rs:525-567,679`, verified first-hand) โ€” the arbitration gap v1 worried about does not exist. +4. **RustFS exceeds MinIO in several places**: the remote_scanner RPC protocol (remote peers scan locally instead of the leader reading remote drives across the network), the persisted leader-epoch CAS fence, cycle budgets and per-set/per-disk concurrency gates, the pending-heal ledger, the durable replacement intent + completion proof state machine, foreground pressure gating (mainline throttle), and the cluster heal control coordinator + envelope replay protection. +5. Gap severity tally: 8 P1 items (behavioral/operational alignment gaps), 9 P2 items (completeness), 3 P3 items (cleanup/low risk), and 7 items of "not pursuing parity by design". Full list in ยง6. + +--- + +## 1. Architecture overview + +### 1.1 RustFS's three-layer architecture + +RustFS splits the heal/scanner functionality that MinIO keeps inside the `cmd/` monolith into three layers plus two standalone crates: + +| Layer | Location | Responsibilities | +|---|---|---| +| Primitives layer | `crates/ecstore/src/set_disk/ops/heal.rs` (~3,240 lines), `ops/heal_walk.rs`, `ops/bitrot_self_verify.rs`; upper wrappers `store/heal.rs`, `store/heal_walk.rs`, `core/sets.rs` | Object/bucket/format/replacement-drive format repair, disk-walk union enumeration, write-path bitrot self-verification; the `rustfs_storage_api::HealOperations` contract is implemented by `SetDisks`/`Sets`/`ECStore` (`crates/storage-api/src/object.rs:503-519`) | +| heal runtime | `crates/heal` | Process-level HealManager (priority queue/scheduler/auto disk scanner/resumable resume), HealChannelProcessor (consumes the global heal channel), drive-replacement recovery state machine | +| scanner runtime | `crates/scanner` | Data usage scanning, ILM evaluation and enqueueing, heal candidate production, replication usage statistics, remote scanner RPC | +| Shared protocol | `crates/common/src/heal_channel.rs` (~776 lines) | Start/Query/Cancel command channel, `HealOpts`/`HealScanMode`/`HealRequestSource`/`HealAdmission*` shared types, `HealResultItem` (madmin) | +| Shared data | `crates/data-usage` | `DataUsageEntry/Info`, histograms, `hash_path`; produced by the scanner, consumed by ecstore/admin | + +Startup chain (wiring verified first-hand): + +1. `rustfs/src/startup_services.rs:93` โ†’ `init_background_service_runtime(store)`. +2. `rustfs/src/startup_background.rs:41-81`: create the global heal service cancel token; read `RUSTFS_SCANNER_ENABLED` (alias `RUSTFS_ENABLE_SCANNER`, default true) and `RUSTFS_HEAL_ENABLED` (alias `RUSTFS_ENABLE_HEAL`, default true); **the heal manager is initialized whenever either heal or scanner is enabled** (heal candidates produced by the scanner need a consumer; with both off, the heal channel is not initialized and `send_heal_request` reports "Heal channel not initialized"). +3. `crates/heal/src/lib.rs:142-216`: atomic initialization inside an owned task (a caller cancel cannot leave a half-initialized manager behind, `lib.rs:123-131`; `GLOBAL_HEAL_RUNTIME_INIT` mutex single-flight) โ†’ `HealManager::start()` โ†’ `rustfs_common::heal_channel::init_heal_channels()` โ†’ spawn `HealChannelProcessor::start_with_receipts`. +4. `crates/heal/src/heal/manager.rs:1301-1356` `HealManager::start`: `start_scheduler()` (`manager.rs:2394-2461`, interval default 10s + `Notify` event-driven wakeup) โ†’ `process_unclean_shutdown()` (`manager.rs:1362-1695`) โ†’ when `enable_auto_heal` (default true), `start_auto_disk_scanner()` (`manager.rs:2464-2999`). +5. After the server is ready, `rustfs/src/startup_lifecycle.rs:150-152`: when `enable_scanner`, `init_data_scanner(token, store)` (`crates/scanner/src/scanner.rs:1293-1372`). +6. Graceful shutdown: `rustfs/src/startup_shutdown.rs:308` `shutdown_ahm_services()` (cancel token); `:414` `clear_unclean_shutdown_markers()`. + +### 1.2 MinIO's corresponding structure (final master state) + +| MinIO file | Responsibilities | +|---|---| +| `cmd/admin-heal-ops.go` | Manual admin heal sequence (healSequence, clientToken/forceStart/forceStop) | +| `cmd/global-heal.go` | Resident background heal queue (newBgHealSequence, token fixed `0000-โ€ฆ`, never ends) + `healErasureSet` (full-object heal per set) | +| `cmd/background-heal-ops.go` | healRoutine worker pool (`_MINIO_HEAL_WORKERS`, default GOMAXPROCS/2) consuming healTask | +| `cmd/mrf.go` | MRF (Most Recent Fail) queue (capacity 100,000), persisted at process exit to `.minio.sys/buckets/.heal/mrf/list.bin` with startup replay | +| `cmd/background-newdisks-heal-ops.go` | Automatic resync for new/replaced drives (monitorLocalDisksAndHeal 10s polling + healFreshDisk + healingTracker) | +| `cmd/erasure-healing.go` / `erasure-healing-common.go` | Object-level heal core (~800 lines), listAndHeal | +| `cmd/data-scanner.go` | Scanner loop (globalLeaderLock cluster singleton) + folderScanner + applyActions | +| `cmd/erasure.go` (nsScanner) / `erasure-server-pool.go` | NSScanner three-layer structure | +| `cmd/bucket-lifecycle.go` | ILM executor (expiry/transition worker pools) | +| `cmd/xl-storage.go` | DiskInfo.Healing, CheckParts/VerifyFile, CleanAbandonedData, RenameData healing branch | +| `cmd/prepare-storage.go` | waitForFormatErasure new-drive startup handshake | + +### 1.3 Architecture-level differences (design trade-offs, not defects) + +1. **heal queue model**: MinIO funnels every heal (scanner sampling/MRF/admin/new-disk resync) into a single channel + a fixed worker pool (new-disk resync additionally has a per-drive worker pool); RustFS is a multi-policy scheduler built from a priority heap + dedup-merge + capacity-tiered dropping + per-set bulkhead + foreground pressure gating (`manager.rs:3003-3420`). RustFS is more expressive, at the cost of an observability question around "duplicate requests being merged" (already pointed out in v1; the current `HealAdmissionReceipt` canonical task_id + alias mechanism answers it, `manager.rs:1759-1846`). +2. **scanner remote-drive access**: the MinIO leader transparently reads and writes remote-node drives through the disk abstraction layer; the RustFS leader pushes scan execution down to the remote peer to run locally via the remote_scanner RPC (`crates/scanner/src/remote_scanner.rs`), with only results and progress heartbeats sent back. Both are cluster single-leader. RustFS's approach saves the leaderโ†”remote metadata read amplification, at the cost of maintaining a separate RPC protocol (HMAC per-frame authentication, session replay cache, fence re-validation, `remote_scanner.rs:52-61,405-496,1024-1065`). +3. **heal state persistence**: MinIO uses a single file `.healing.bin` (msgp healingTracker, reset whenever the diskID mismatches); RustFS uses a schema'd multi-file layout (resume/checkpoint/intent/seal/proof, each CAS-published, `resume.rs:38-61`), with the crash window explicitly backfilled (`erasure_healer.rs:389-402`, `resume.rs:1027-1057`). +4. **write-path self-protection**: MinIO relies on background heal to converge after writes; RustFS, after the commit rename in PutObject/CompleteMultipartUpload, actively checks `convergence.needs_heal()` and immediately enqueues an object heal (`set_disk/ops/object.rs:2291-2306`, `ops/multipart.rs:2574-2589`), and additionally has read repair (`io_primitives.rs:1040-1160`). + +--- + +## 2. Heal implemented-feature panorama + +### 2.1 Task types (`HealType`, `crates/heal/src/heal/task.rs:85-111`) + +| Type | Semantics | Executor | Production trigger | +|---|---|---|---| +| `Cluster` | all buckets healed in turn (structure + optional recursive objects), in-batch retry โ‰ค3 | `heal_cluster` task.rs:1420-1490 | channel: empty bucket means Cluster (channel.rs:576-577) | +| `Object{bucket,object,version_id}` | single object/version; when absent, rebuild per `recreate_missing` or error out | `heal_object` task.rs:855-1146 | admin, scanner, read-repair, write-path convergence, add_partial | +| `Bucket{bucket}` | bucket metadata/structure; `recursive` additionally walks all object versions | `heal_bucket` task.rs:1284-1418 + `heal_bucket_objects` task.rs:1508-1698 | admin (POST /v3/heal/{bucket}), scanner `build_bucket_heal_request` | +| `Prefix{bucket,prefix}` | recursive by prefix | `heal_prefix` task.rs:1492-1506 | channel: `recursive && prefix` non-empty (channel.rs:578-585) | +| `ErasureSet{buckets,set_disk_id}` | format repair + healing marker + per-bucket preprocessing + resumable per-version deep scan | `heal_erasure_set` task.rs:2158-2642 | admin (pool/set params), auto disk scanner, unclean shutdown, renew_disk, durable replacement recovery | +| `Metadata{bucket,object}` | metadata only (Deep, does not rebuild data) | `heal_metadata` task.rs:1700-1859 | **no production trigger** (ยง6 HS-01) | +| `MRF{meta_path}` | failure-path-driven Deep repair (recursive+update_parity) | `heal_mrf` task.rs:1861-1992 | **no production trigger** (only `HealEvent` can generate it, unwired) | +| `ECDecode{bucket,object,version_id}` | EC decode rebuild (Deep+recreate+update_parity), Urgent priority | `heal_ec_decode` task.rs:1994-2156 | **no production trigger** (only `HealEvent` can generate it, unwired) | + +Priorities `Low/Normal/High/Urgent` (task.rs:168-179); state machine `Pending/Running/Retrying/Completed/Failed/Cancelled/Timeout` (task.rs:225-241). + +### 2.2 Trigger-path panorama (beyond admin) + +| Channel | source | Priority | Evidence | +|---|---|---|---| +| Scanner periodic sampling (1/1024, `RUSTFS_HEAL_OBJECT_SELECT_PROB`) | Scanner | Low | `scanner_folder.rs:2117-2136`, `:1150`; `remove_corrupted=HEAL_DELETE_DANGLING(true)`, `recreate_missing=false` (`common/heal_channel.rs:24`, `scanner_folder.rs:510-511`) | +| Scanner metadata corruption (get_size failure classified HealMetadata) | Scanner | High | `scanner_folder.rs:2147-2208`, `:1244-1260` | +| Scanner abandoned children (present in cache, absent on disk, list_path_raw quorum verification) | Scanner | High (bucket-level + object-level) | `scanner_folder.rs:2528-2792` | +| Scanner pending-heal ledger retry (persisted after rejection by a full heal channel, โ‰ค128 per bucket per round, 10k cap) | Scanner | original priority | `scanner_folder.rs:1721-1763`, `:99-100` | +| auto disk scanner (unformatted drive confirmed via replacement_readiness / `runtime_state=="returning"` drive / durable-intent re-entry) | AutoHeal | Low | `manager.rs:2464-2999` | +| unclean shutdown recovery (startup reads the `unclean-shutdown` marker โ†’ ErasureSet heal for all local sets) | AutoHeal | Low | `manager.rs:1362-1695` | +| write-path convergence (after PutObject/CompleteMultipartUpload, `convergence.needs_heal()`) | Internal | Normal | `set_disk/ops/object.rs:2291-2306`, `ops/multipart.rs:2574-2589` | +| partial-object heal (add_partial) | Internal | Normal | `set_disk/ops/object.rs:5808-5825` | +| stale data-directory cleanup leftover enqueue | Internal | Normal | `set_disk/core/io_primitives.rs:3880-3907` | +| read repair (metadata_read_error / missing_shards / decode_error, TTL dedup cache) | ReadRepair | Low | `set_disk/read.rs:407,995,1079` โ†’ `submit_read_repair_heal` (`io_primitives.rs:1105-1160`), `recreate_missing=true` | +| drive reconnect hits UnformattedDisk โ†’ send_heal_disk | AutoHeal | Normal | `set_disk/ops/locking.rs:339-347` | +| Admin API (incl. cluster coordinator routing) | Admin | High | `rustfs/src/admin/handlers/heal.rs:174-212`, `:771-930` | +| cluster RPC heal (peer invocation) | โ€” | โ€” | `rustfs/src/storage/rpc/node_service/heal.rs`, `ecstore/src/cluster/rpc/peer_s3_client.rs:296,1209` | + +Note: MinIO's MRF channel (read-path immediate delivery on missing/corrupt parts + queue persistence + shutdown replay, `cmd/mrf.go`, `erasure-object.go:395-410,800-812`) is **partially replaced** in RustFS by read-repair + write-path convergence; the three executors `HealType::MRF`/`ECDecode`/`Metadata` have no production entry (see ยง6 HS-01 for details). + +### 2.3 Object-level heal semantics (ecstore `set_disk/ops/heal.rs`) + +Flow (`heal_object_with_explicit_version_regen` from :426): + +1. Take the object write lock (unless `no_lock`); an `object` ending with `/` goes through object-directory heal (`heal_object_dir_locked` :1587-1717: dangling determination + `remove` deletion + missing-volume rebuild). +2. `read_all_fileinfo` reads xl.meta from all disks; all-not-found is treated as already deleted and returns. +3. **quorum arbitration + ETag fallback** (verified first-hand): `list_online_disks` treats the mod-time quorum as authoritative; when quorum fails it falls back to ETag majority arbitration (`:525-567` `filter_by_etag`/`quorum_etag`); `pick_valid_fileinfo` picks the canonical metadata; the cannotHeal determination for "number of bad-meta disks > parity" is waived when the ETag agrees across all disks (`:679`). Matches MinIO's dual arbitration in `filterDisksByETag`. +4. `disks_with_all_parts` (:562-572) validates parts per `scan_mode`: **Normal only stats (CheckParts semantics), Deep does full bitrot verification (VerifyFile semantics)**; when a Normal scan detects `FileCorrupt` it automatically escalates to Deep and retries once (`:2022-2031`, same shape as MinIO erasure-healing.go:1101-1106); a no-parity object (EC:0) with a bitrot failure is judged unrecoverable (`:700-726`). +5. `should_heal_object_on_disk` (:606-650) classifies each disk as missing/corrupt/offline/outdated โ†’ rebuild: per-part bitrot reader/writer (using per-part checksum + algorithm), write into a temporary volume then rename to commit (`HEAL_RENAME_INCOMPLETE` retry semantics :24); dangling-deletion safety check `dangling_delete_safety` (:1488); **orphan data-directory reclamation `reclaim_orphan_data_dirs_best_effort` (:1428)** โ€” this part covers the main scenarios of MinIO's `CleanAbandonedData` (but there is no standalone `CheckAbandonedParts` API, see ยง6 HS-02). +6. Versioned objects: enumerate "every version" (`storage.rs:1494-1530`); the delete-marker path is decided by `latest_meta.deleted` (`storage.rs:262-277` comment); regression tests `tests/heal_b5_versioned_regression_test.rs:282,334`. +7. Explicit-version rebuild `try_regenerate_explicit_version_meta` (:1318); cleanup of local leftovers of transitioned objects. +8. The write path additionally has shard-level bitrot self-verification `verify_written_bitrot_shards` (`ops/bitrot_self_verify.rs:45-129`, HighwayHash256S, verifying freshly written shards right before the final rename, serving the EC:0 no-parity case) โ€” **note this is not background bitrot patrol**; background patrol is carried by scanner bitrot_cycle-driven Deep heal. + +heal-crate-side wrapper (`task.rs:855-1146`): existence check (transient errors become `TransientSkip` to avoid false failures :551-569); scanner synthetic-directory normalization (:1148-1180); `recreate_missing` rebuild (:1183-1282); data-usage-cache object-lock timeout exemption (:571-653); not-found โ†’ treated_as_deleted success (:1012-1029); results `HealResultItem` keep at most 1024 entries + truncated flag (:50,845-852). + +Recursive walk (`heal_bucket_objects` task.rs:1508-1698): paginated enumeration of all versions including delete markers, transient-error exponential-backoff retry โ‰ค3 (2^n + jitter :620-627), failure-sample log truncation โ‰ค5 entries, aggregated `BatchHealFailure`. + +### 2.4 erasure set heal and resumable scans + +`heal_erasure_set` (task.rs:2158-2642) runs in four phases (4-step progress tracking): + +1. **Replacement intent and recovery-drive selection** (AutoHeal only + non-empty heal_endpoints): reuse the drive holding the durable intent / exclude the target endpoints and pick surviving drives; already-completed generations get an idempotent CleanupPending wrap-up. +2. **Format repair**: `heal_replacement_format(dry_run, pool, set, targets)` (`storage.rs:1372-1384`, trait default fail-closed); per-target-drive results must all be ok (`erasure_healer.rs:97-102`) + identity-fence re-check (task.rs:2410-2420). +3. **healing marker**: write an owner CAS marker `{set_disk_id}:{task_id}` to the target drive (`mod.rs:80-229`, CAS + rollback + unique concurrent owner), which makes `DiskInfo.healing` true (assignment chain verified first-hand `set_disk/mod.rs:4988`). +4. **Per-bucket preprocessing + resumable deep scan**: `ErasureSetHealer::heal_erasure_set` (`erasure_healer.rs:242-278`). + +`ErasureSetHealer` scan details (benchmarked against MinIO `healErasureSet`; the `heal_walk.rs:15-23` module comment explicitly cites MinIO `global-heal.go`'s listPathRaw + objQuorum=1 + mergeXLV2Versions): + +- **Enumerator choice (backlog#920)**: Deep or AutoHeal โ†’ per-set **disk-walk union enumeration** `list_versions_for_heal_page_disk_walk` ("exists on any drive" means sub-quorum reconstructible; `storage.rs:1559-1644`, page bounds 1,000 objects/10,000 versions, `dw1:` cursor); ordinary requests go through read-quorum `list_object_versions`. +- **Resume cursor**: the authoritative cursor is an opaque continuation token (`v1:` = marker JSON, `dw1:` = disk-walk key; the two namespaces are mutually exclusive against misreads, `storage.rs:81-260`); after each completed page, persist the cursor first, then clear the dedup set (`erasure_healer.rs:922-927`). +- **In-page concurrency**: FuturesUnordered + Semaphore, default `RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY=8`, Deep/AutoHeal forces 1 (`erasure_healer.rs:105-142`). +- **per-version dedup**: `compose_key` length-prefix injection encoding (`resume.rs:281-288`). +- **Error classification**: truly absent (FileNotFound etc.) โ†’ Absent (counted as success); infrastructure-transient (quorum/DiskNotFound/SlowDown etc.) โ†’ Transient (counted as skipped); everything else Failed (`erasure_healer.rs:148-182`; the comment cites backlog#856/#799 B7: offline drives must not be recorded healed/absent). +- **Loop protection**: abort when an empty page is truncated or the page-tail version identity does not advance (:933-949). +- **Completion determination**: if any of failed/skipped/failed_buckets is >0, do not mark complete; `schedule_retry()` resets both the resume and checkpoint layers (:561-626; backlog#855/B6/#1033: a skip round must not be marked complete). +- **Replacement-drive commit proof**: physical read-back on the target endpoints `replacement_targets_have_version` (`ops/heal.rs:340-412`); unconfirmed โ†’ transient skip. + +### 2.5 Automatic drive-replacement healing (replacement recovery) + +- **Identification** (`replacement_readiness.rs:25-73`): `replacement_mount_lease_root()` exists, canonicalize succeeds, is a mount point, the physical device id is non-empty, disjoint from the root device, and shares no physical device with sibling drives (Linux uses /proc/self/mountinfo mount-id+dev+ino). The non-root mount check has a regression test (`manager.rs:3549`). +- **State machine** (`resume.rs:63-73`): `Intent โ†’ Rebuilding โ†’ (write proof) Verified โ†’ CleanupPending โ†’ cleanup`; `Abandoned` is a terminal state; state transitions write the persistence layer first, then mutate (`save_state_strict`). +- **Persistence** (`resume.rs:38-61`, schema ResumeState=5/Checkpoint=5/proof=1): `{task_id}_ahm_resume_state.json`, `_ahm_checkpoint.json`, and intent/seal/completion_proof under the `buckets/ahm-replacement/` namespace; torn write + no seal is recognizable and rebuilt atomically (:1316-1338); CAS publish, refuses to overwrite a concurrently valid proof (:1512-1585). +- **Recovery**: both unclean shutdown and the periodic scan recover unfinished/pending-cleanup replacement generations from surviving drives (`manager.rs:1435-1640,2663-2815`); multi-generation conflict / validation failure โ†’ freeze that set (`replacement_recovery_blocked_sets`, `manager.rs:69-87,2782-2815`). +- **External snapshot**: `current_replacement_recovery_snapshot` (`lib.rs:262-333`) merges local surviving-drive records; conflict โ†’ Unknown / non-definitive; admin `GET /v4/heal/replacement-recovery`. + +### 2.6 Scheduler (manager.rs) + +- Priority heap + FIFO within the same priority (:148-191,330-347); dedup key per type (:469-506); enqueue three-state dedup activeโ†’queuedโ†’retrying (:1759-1785); duplicates default to Merged and return the canonical task_id (`HealAdmissionReceipt`, :1821-1846) + client token alias (:1219-1246). +- Capacity: when the queue is full, best-effort sources (Scanner/AutoHeal/ReadRepair) or low-priority items get Dropped(QueueFull); Admin/Internal may evict queued lower-priority items (`push_displacing_lower_priority` :353-396); 80%/95% tiered pressure handling (:885-909). +- Concurrency: global `max_concurrent_heals` (default 4) + per-set bulkhead `max_concurrent_per_set` (default 1) (:3040-3073,3434-3447). +- Foreground pressure gating, mainline throttle: delay best-effort tasks when foreground read/write permit utilization is โ‰ฅ80% (:919-1009,2999-3020). +- Timeout: task-level aggregate timeout (default 300s), remaining budget preserved across retries (task.rs:444-451, PR #6101). +- Recoverable retry: `is_recoverable_heal()` (error.rs:83-136) โ‰ค3 attempts, 2^n backoff capped at 30s; retries hold ownership inside a standalone backoff task (:3235-3382). +- Completion states are retained for 10 minutes for querying (:42). + +### 2.7 Admin API and cluster coordination + +- Routes (`rustfs/src/admin/handlers/heal.rs:174-212`): `POST /rustfs/admin/v3/heal/`, `/heal/{bucket}`, `/heal/{bucket}/{prefix}` (the same POST distinguishes start/query/cancel by the query `clientToken/forceStart/forceStop`, aligned with mc admin heal semantics); `POST /v3/background-heal/status`; `GET /v4/heal/replacement-recovery`. Permission `HealAdminAction` (route_policy.rs:334-341). +- Cluster coordination (heal.rs:771-930 + `node_service.rs:514-606`): `heal_topology_fingerprint` + deterministic-by-topology coordinator-node selection + coordinator epoch; envelope validation + SHA256 digest replay protection; when the coordinator is not local, go through peer gRPC `heal_control`; `probe_heal_control` capability probe (rolling-upgrade scenario). +- Request: the body is `HealOpts` (`recursive/dryRun/remove/recreate/scanMode(0/1/2)/updateParity/nolock/pool/set`, serde camelCase, fields aligned with madmin.HealOpts); a root heal start requires `recursive=true` or a `pool+set` pair; body cap 1MB. +- Response: `HealStartSuccess{clientToken, clientAddress, startTime}`; `HealTaskStatus{summary, detail, startTime, settings, items, truncated, progress}` (summary โˆˆ running/finished/stopped/notFound); `BackgroundHealStatus` (bitrot start time/cycle/current mode + `disabled/uninitialized/idle/active/degraded` states โ€” an unreachable peer is explicitly degraded rather than impersonating idle, issue #5850) + `healOperations` as a priorityร—source matrix + cluster progress. +- `HealResultItem`/`HealDriveInfo`/`HealItemType`/DriveState enums are JSON-compatible with madmin (`crates/madmin/src/heal_commands.rs:19-65`). +- A status payload over 8MiB is truncated by halving (channel.rs:37,73-104); path-token validation (wrong token rejected; an empty path matches Cluster only). + +### 2.8 heal metrics and logs + +Metrics: `rustfs_heal_admission_total{source,result,reason,context}`, `rustfs_heal_task_start_total`, `rustfs_heal_task_running{type,set}`, `rustfs_heal_queue_delay_seconds`, `rustfs_heal_scheduler_skip_total`, `rustfs_heal_mainline_throttle_total`, `rustfs_heal_page_concurrency_current{set}`, `rustfs_heal_candidate_enqueue/merge/drop/priority_reject_total`, `rustfs_heal_read_repair_dedup_total{reason}`, etc. All logs are structured event style (PR #5720); per-object logs are demoted to prevent storms (`demote_to_debug_when!`, #5716/#5719/#5727). + +--- + +## 3. Scanner implemented-feature panorama + +### 3.1 Loop, leader, immediate triggering + +- **Cluster single leader**: distributed ns write lock `leader.lock` (`scanner.rs:3156-3207`, timeout default 5s) + **persisted leader-epoch CAS fence**: the leader writes (cycle, leader_epoch) encoded as `RSCYC001` into `.bloomcycle.bin` using an ETag precondition (`scanner.rs:118,1850-1861,2177-2334`); usage snapshots additionally carry an epoch fence (:2087-2153). Lock lost โ†’ cancel the current cycle, converging within 30s (:108-111,2623-2642). +- One round executes immediately after the lock is acquired; cycle = `RUSTFS_SCANNER_CYCLE` > config cycle > start_delay > deployment default > speed tier (ยฑ10% jitter, floor 1s). +- **clean-idle exponential backoff**: consecutive fully-clean idle intervals double (capped at 24h; bitrot-cycle compression cap; disabled when a bucket has active lifecycle/replication rules, :383-456,1382-1512). +- **superseded/deferred backoff**: exponential backoff from 5s capped at 30min (:105-106,3432-3438); maintenance probing failures get an independent backoff (:459-505). +- **Immediate wakeup**: โ‘  dirty-usage fast path โ€” write-path put/delete/multipart/bucket operations call `record_dirty_usage_bucket` (`scanner_io.rs:222-235`; call sites include `rustfs/src/app/object_usecase.rs:6221`), bump the generation and Notify-wake the leader; dirty buckets are queued first (`scanner_io.rs:462-488`); โ‘ก maintenance-config changes (lifecycle/replication settings call `record_scanner_maintenance_change`); โ‘ข runtime-config hot updates generation+Notify; โ‘ฃ cluster activity snapshot changes. +- **Cluster coordination**: `probe_scanner_activity` gathers this node's and peers' `ScannerNodeActivity` (instance_id/namespace_generation/maintenance_generation/protocol_version/topology_digest/data_movement_active/dirty usage); the topology digest covers pools/sets/drives URLs; a mismatched protocol version refuses to share the cache lock (`scanner.rs:970-1068`); **cycles are deferred during data movement (rebalance/decommission)** (`scanner_io.rs:2226-2374`); at cycle end, per-peer RPC confirms the dirty-usage ack (`scanner.rs:2925-2952`). + +### 3.2 Traversal model + +- The main traversal is a **full directory walk** (tokio::fs::read_dir recursion, `scanner_folder.rs:1915-2234`), not via metacache; metacache/`list_path_raw` is used only for the abandoned-children cross-drive verification (:2528-2792). +- Three-level concurrency: leader โ†’ per-set (semaphore default 4) โ†’ per-disk bucket scans (default 4) โ†’ single-drive recursion; a cache lock per bucket per set `.scanner-cycle.lock.pool-N.set-M` (losing the lock cancels that bucket's scan; lock contention re-queues); single-scan admission per drive (local drives also go through the semaphore, `scanner_io.rs:3246-3274`). +- Bucket ordering: after shuffle, re-ordered as dirty โ†’ uncached โ†’ cached (`scanner_io.rs:2947-2949,462-488`); entries within a directory sorted by name + resume-hint rotation (`scanner_folder.rs:333-359`). +- **Resumable scanning**: `DataUsageScanCheckpoint{version,resume_after,reason}` persisted in the cache info (`data_usage_define.rs:68,293-307`); written on budget exhaustion/cancel; resumption has Used/Stale/NoHint metrics; the resume unit is a directory (no cross-cycle object-level pagination). +- Erasure semantics: finding `xl.meta` marks an object boundary with no descent; at most 64 UUID data-dir candidate entries probed; data without metadata โ†’ record failed + high-priority heal; symlink directories ignored / cycles skipped. +- Cooperative yielding: `yield_now` every N objects (default 128). + +### 3.3 Large-bucket skip strategy (benchmarked against MinIO compaction) + +1. Cache-currency reuse: if the bucket and scan plan are unchanged (name/source/snapshot_complete/plan digest/next_cycle/leader_epoch/cache_key_format all match), the whole bucket is skipped (`scanner_io.rs:1062-1109`). +2. compacted-directory 16-cycle rotation window: rescan only when `hash mod (next_cycle, 16)` hits, otherwise copy from the old cache (`scanner_folder.rs:74,2429-2442`). +3. compaction thresholds: children <500 or pure-object leaves compress into a single entry; subfolders โ‰ฅ2500 (root 10000) pre-compressed; children โ‰ฅ10000 reduced (:75-78,2314-2340,2846-2887). +4. failed-object TTL skip: 86400s / at most 10,000 entries (:88-91,1354-1381). + +Compared with MinIO master: MinIO's skip strategy is likewise hash-mod-16 cycles + a compaction threshold tree (500/10000/2500), and the **bloom filter has been removed from master**. RustFS's constants and structure share the same origin as MinIO's current state (MinIO does not adopt cross-drive dirty-generation prioritization; RustFS additionally has two more skip layers โ€” plan digest and cache-currency validation). + +### 3.4 data usage statistics + +- Dimensions: per-directory entry (size/objects/versions/delete_markers/size histogram/version histogram/replication stats/failed_objects/per-tier stats/children/compacted, `data-usage/src/data_usage.rs:661-679`); per-object SizeSummary (incl. per-ARN replication-target stats and tier stats; tier classification: fully transitioned counts toward its tier, otherwise by storage class; free versions not counted); bucket-level `BucketUsageInfo`; cluster-level `DataUsageInfo` (incl. scanner_cycle/scanner_epoch fence + usage_snapshot_complete). +- Storage: per bucket per set `{bucket}/.usage-cache.bin` (primary + `.bkp` backup + CAS retry); the authoritative cluster snapshot `buckets/data-usage/data-usage.json` (`.bkp` synced every 10 cycles, legacy path compatible); stale snapshots rejected on write (triple epoch/cycle/last_update determination); observation snapshots superseded by a race are stored separately as `data-usage-observed.json`. +- Consumption: `replace_bucket_usage_memory_from_info` refreshes bucket-usage memory + two-level cache invalidation (`scanner.rs:4142-4152`) โ†’ bucket stats/quota/admin account_info/system; the write path overlays memory in real time; at startup, reading the snapshot detects a cold cache and skips startup delay. +- Incomplete multipart uploads are not counted (consistent with MinIO, which also does not scan the multipart bucket). + +### 3.5 ILM integration + +- Per object `ScannerItem::apply_actions` (`scanner_folder.rs:747-1032`): `Evaluator::new(lifecycle).with_lock_retention(...).with_replication_config(...).eval()` batch evaluation. +- Implemented actions (the full IlmAction set, `common/src/metrics.rs:34-45`): expiry deletes (Delete/DeleteRestored/DeleteRestoredVersion), all-versions deletes (DeleteAllVersions/DelMarkerDeleteAllVersions, stop further versions after handling), transition (Transition/TransitionVersion, tier list read at runtime), noncurrent batches (DeleteVersionAction โ†’ `enqueue_by_newer_noncurrent`), free-version cleanup (`enqueue_free_version`), object-lock retention constraints. **A one-to-one mapping onto MinIO's 9 ILM actions.** +- Execution model: the scanner is the "discover and enqueue" role (the expiry/transition queues live in ecstore `bucket_lifecycle_ops.rs`); actions are consumed by worker pools โ€” the same shape as MinIO's globalExpiryState/globalTransitionState. +- AbortIncompleteMultipartUpload is not executed inside scanner/ILM (MinIO likewise: `internal/bucket/lifecycle/rule.go` has a FIXME, and it is actually carried by the `erasureSets.cleanupStaleUploads` global routine); in RustFS it is an independent ecstore background task `init_background_stale_multipart_upload_cleanup` (`bucket_lifecycle_ops.rs:3289-3320`) + on-demand at bucket deletion. +- Integration-test coverage: transition+restore, free-version, noncurrent, delete-marker, 0-day, background-scan expiry (`scanner/tests/lifecycle_integration_test.rs:1071-2095`). + +### 3.6 heal candidate production (scanner side) + +- Sampling: `hash mod_alt(next_cycle/prob_div, 1024/prob_div)`; when rescanning via the compacted branch, prob_div=16 gives an equivalent ร—16 probability (the same compensation as MinIO, `scanner_folder.rs:125-127,2117-2122`). +- deep/normal: cycle-level `get_cycle_scan_mode` (bitrot_cycle default 30d, `scanner.rs:1626-1657`) โ†’ object-level with `HealScanMode::Deep`; fresh objects (modified within 60s) are demoted to Normal (:146-155); state persisted in `.background-heal.json` (`BackgroundHealInfo{bitrot_start_time,bitrot_start_cycle,current_scan_mode}`, same path and structure as MinIO). +- The scanner only enqueues, never executes inline (inline heal was removed; the compat flag only warns, `scanner_folder.rs:411-427`); `HealScanMode::Deep` is just a marker โ€” the bitrot-verification read happens at the heal consumer (the ecstore Deep path). +- Metadata corruption โ†’ high-priority heal (`classify_get_size_failure` โ†’ HealMetadata); abandoned children โ†’ list_path_raw quorum verification + bucket-level/object-level high-priority heal; healing drives get sticky skipping (`should_heal` :1628-1648). +- pending-heal ledger: candidates rejected by a full heal channel are persisted into the cache info and retried next round. +- Replication heal: `queue_replication_heal` โ†’ the replication queue (going through the replication channel, not the heal channel); per-ARN replication usage statistics. + +### 3.7 remote_scanner RPC protocol (RustFS-specific) + +Requests โ‰ค16KB msgpack (version/request_id/server_epoch/session_id/session_sequence/bucket/next_cycle/leader_epoch/scan_plan_digest/skip_healing/scan_mode/budget); frames โ‰ค2MB, HMAC-SHA256 per-frame authentication (domain `rustfs-ns-scanner-frame-v3`); progress heartbeats 1s (250ms in budget mode); phase announcements Scanningโ†’Persisting; RPC lifetime cap 24h, disconnect grace 2min; anti-replay session+sequence cache (capacity 65536); the server validates leader-fence and persisted-cycle consistency + fence re-validation every 5s; results Complete/Partial/NamespaceNotFound/CycleAhead; remote drives without v4-protocol support fall back to the leader scanning locally (`remote_scanner.rs` whole file; `scanner_io.rs:2750-2812`). + +### 3.8 Rate limiting / budgets / hot updates / observability + +- DynamicSleeper proportional backoff (speed tiers fastest/fast/default/slow/slowest, same five-tier parameters as MinIO); idle_mode master switch; an extra backoff capped at 250ms per request (10ms base) driven by foreground S3 read traffic. +- Cycle budget ScannerCycleBudget: max_duration/max_objects/max_directories (default 0 = unlimited); partial cycles still advance the cycle count. +- runtime_config with three-layer sources (env > config > default) and per-field source markers (Env/Config/ScannerCompatConfig/Default); admin `PUT /v3/config` hot update โ†’ generation+Notify takes effect immediately; `GET /v3/scanner/status` returns enabled/freshness(fresh/stale/unknown)/metrics/cycle_schedule/runtime_config; `GET /v3/ilm/expiry/status` returns expiry queue/workers/missed/blocked. +- Metrics: leader lock; cycle complete/partial/deferred/superseded; versions scanned; per-source (Usage/Lifecycle/BucketReplication/SiteReplication/Heal/Bitrot/Alerts) checked/executed/queued/missed; checkpoint set/used/stale; current path (per-disk+bucket in real time); cache save series; concurrency series; alerts (excess versions/version size/folders). + +--- + +## 4. Item-by-item parity versus MinIO + +### 4.1 heal trigger-channel comparison + +| MinIO channel | RustFS counterpart | Status | +|---|---|---| +| A. Manual admin heal (healSequence, clientToken/forceStart/forceStop) | heal channel Start/Query/Cancel + cluster coordinator + envelope replay protection | โœ… equivalent and enhanced (cluster routing); sequence-semantics differences in ยง6 HS-06 | +| B. Resident background heal queue (newBgHealSequence + healRoutine worker pool) | HealManager resident scheduler + priority queue + bulkhead | โœ… equivalent and enhanced | +| C. Automatic new/replaced-drive resync (monitorLocalDisksAndHeal 10s + healFreshDisk + healingTracker + waitForFormatErasure handshake) | auto disk scanner (10s) + replacement_readiness + durable intent/proof state machine + heal_replacement_format | โœ… equivalent and enhanced (identity fence + completion proof; MinIO's tracker is stronger on external visibility, see ยง6 HS-07) | +| D. MRF (100k queue + persisted list.bin + shutdown replay + read-path corrupt delivery) | read-repair (Low + TTL dedup) + write-path convergence heal carry it partially; the `HealType::MRF` executor has no production entry | โš ๏ธ partially equivalent (ยง6 HS-01) | +| E. Scanner sampled heal (1/1024 + compacted ร—16 compensation) + abandoned children | the same sampling + ร—16 compensation + abandoned children + pending-heal ledger | โœ… equivalent and enhanced (the ledger) | +| F. Read-path inline trigger โ†’ MRF (GetObject part missing/corrupt, metadata rebuild missingBlocks>0) | read repair (three entries: missing_shards/decode_error/metadata_read_error) | โœ… equivalent (enqueued into the heal queue rather than the MRF queue) | + +### 4.2 Object-level heal semantics comparison + +| Feature | MinIO | RustFS | Status | +|---|---|---|---| +| mod-time quorum arbitration | listOnlineDisks | same | โœ… | +| ETag majority fallback (clock drift) | filterDisksByETag | `filter_by_etag`/`quorum_etag` (heal.rs:525-567) | โœ… verified first-hand | +| cannotHeal ETag waiver | waived on all-consistent ETag retry | heal.rs:679 | โœ… | +| Normal=CheckParts (stat) / Deep=VerifyFile (bitrot) | yes | `disks_with_all_parts` by scan_mode (ops/heal.rs:562-572,978-1024) | โœ… | +| Normal detecting corrupt auto-escalates to one Deep retry | erasure-healing.go:1101-1106 | ops/heal.rs:2022-2031 | โœ… | +| dangling determination (not-found > parity) + deletion auditing | isObjectDangling/deleteIfDangling | `dangling_delete_safety` (:1488) + scanner HEAL_DELETE_DANGLING | โœ… (audit-tags details differ) | +| Orphan data-dir/inline cleanup (CleanAbandonedData) | CheckAbandonedParts (invoked explicitly on scanner sampling + admin Remove) | in-heal-path `reclaim_orphan_data_dirs_best_effort` (:1428); standalone API NotImplemented at all three layers | โš ๏ธ partially equivalent (ยง6 HS-02) | +| Versioned/delete-marker heal | HealObject versionID; nullVersionID special case | per-version enumeration + delete-marker latest heal (B5 regression) | โœ… | +| Object-level healing metadata marker (x-minio-healing, RenameData skips version cleanup) | yes | no object-level marker; relies on drive-level healing.bin + NSLock + rename semantics | โš ๏ธ evaluation item (ยง6 HS-12) | +| Distribution/Index consistency, three lines of defense | yes (manual modification rejected) | target-drive format results all-ok check + identity fence | โœ… (different granularity) | +| no-parity (EC:0) objects | bitrot treated as unrecoverable | judged unrecoverable (:700-726) + write self-verification | โœ… enhanced (write-path self-verification) | +| three-layer distribution inconsistency refuses heal | yes | heal_walk normalization + page-bound defense | โœ… (different implementation approach) | +| multipart orphan reconciliation | carried by CheckAbandonedParts | explicitly NotImplemented (carried by lifecycle cleanup) | โš ๏ธ ยง6 HS-02 | +| suspended/decommissioned pool handling | skipped via IsSuspended | deferral semantics (store/heal.rs:192-207, PR #5876) | โœ… | +| heal mutually exclusive with concurrent deletes | NSLock + healing marker | NSLock + write lock | โœ… | + +### 4.3 new-drive resync comparison + +| MinIO | RustFS | Status | +|---|---|---| +| waitForFormatErasure handshake waiting indefinitely on four classes of recoverable errors | startup drive resolution + renew_disk reconnect path | โœ… (different model: RustFS does not block at startup waiting for format) | +| HealFormat NSLock + errNoHealRequired + refFormat-mismatch rejection | `heal_format`/`heal_replacement_format` fail-closed + target-slot restriction (PR #1787 semantics) | โœ… enhanced | +| per (pool,set) distributed lock preventing concurrent resync | set-level queue dedup + bulkhead (manager.rs:2854-2889) | โœ… | +| brand-new-cluster detection (drives-to-heal == total drives does not trigger) | replacement_readiness (independent mount point / physical-device validation, non-root) | โœ… enhanced | +| healingTracker (.healing.bin: Bytes/Items counters, QueuedBuckets/HealedBuckets, Resume snapshot, RetryAttempts โ‰ค4, HealID linkage, diskID-change reset) | resume/checkpoint schema'd persistence + durable intent/proof (per-task files, CAS) | โœ… equivalent and enhanced (crash-window backfill); but **external snapshot visibility** is weaker than MinIO's (ยง6 HS-07) | +| skip versions written after heal start (ModTime > Started) | no such filter | โš ๏ธ ยง6 HS-13 | +| skip ILM-expired versions (filterLifecycle) | no such filter | โš ๏ธ ยง6 HS-13 | +| worker count max(GOMAXPROCS,NR)/4 floor 4, heal:drive_workers override | in-page concurrency 8 (Deep/AutoHeal forced to 1) + per-set bulkhead | โœ… (different parameter model) | +| waitForLowHTTPReq yield per entry | mainline throttle (foreground-utilization gating) | โœ… enhanced | +| heal scope includes the two pseudo-buckets `.minio.sys/config` and `.minio.sys/buckets`; newest bucket first | ErasureSet task pre-processes per bucket (meta-bucket semantics carried by heal_bucket) | โœ… (no "newest first" ordering) | +| whole-failure retry โ‰ค4 (resetHealing + errRetryHealing) | schedule_retry resets both layers + recoverable retry โ‰ค3 | โœ… | + +### 4.4 scanner comparison + +| MinIO | RustFS | Status | +|---|---|---| +| cluster single leader (globalLeaderLock) | leader.lock + persisted leader-epoch CAS fence | โœ… enhanced (epoch fence against split-brain; MinIO has no persisted epoch) | +| `.bloomcycle.bin` stores only the cycle (bloom removed) | same path stores cycle+leader_epoch (RSCYC001) | โœ… aligned (v1 misjudgment corrected) | +| folderScanner hash-mod-16 + compaction (500/10000/2500) | same constants + plan digest + cache-currency validation + dirty-first | โœ… enhanced | +| โ‰คGOMAXPROCS parallel scans per drive; healing drives excluded | per-set/per-disk semaphores + sticky skip of healing drives | โœ… | +| scannerSleeper (factor 2/max 1s, speed tiers hot-swapped) | DynamicSleeper same + idle_mode + foreground-read backoff | โœ… enhanced | +| idle semantics: `scanner:idle_speed=on` (throttle only in idle windows, full speed when busy) | `RUSTFS_SCANNER_IDLE_MODE=true` (master switch for rate limiting) | โš ๏ธ opposite semantic direction, ยง6 HS-14 | +| applyActions order (healโ†’ILMโ†’replicationโ†’alerts) | apply_actions same order (heal candidatesโ†’ILMโ†’replication healโ†’alerts) | โœ… | +| ILM 9 actions + batch evaluation + DeletePrefixObject optimization | same 9 actions + batch evaluation + expiry queue | โœ… (whether DeleteAllVersions has the single-call optimization was not checked line by line) | +| abandoned children (listPathRaw minDisks=N/2 detects under-written drives) | list_path_raw + quorum verification + high-priority heal | โœ… | +| incomplete multipart independent routine (6h interval/24h expiry, rename into .trash) | ecstore independent background task (configurable interval/expiry) | โœ… (trash two-stage cleanup detail differences, ยง6 HS-18) | +| usage dimensions (size/objects/versions/DM/histograms/replication/tier/bucket level) | full coverage + cluster snapshot with triple anti-rollback | โœ… enhanced | +| prefix-level usage (loadPrefixUsageFromBackend, consumed by console) | the cache holds the directory tree but flattens only to bucket level | โŒ ยง6 HS-08 | +| excess events s3:ObjectManyVersions/LargeVersions/PrefixManyFolders + auditing | metrics alert_excess_* only (defaults 100/1TiB/65538 vs MinIO 100/1TB/50000) | โš ๏ธ ยง6 HS-04/HS-17 | +| scanner metrics v3 (bucket_scans/directories/objects/versions/last_activity) | full rustfs_scanner_* suite + freshness | โœ… (different naming scheme) | +| TraceScanner / realtime metrics (mc admin scanner status/trace) | no trace channel; /v3/scanner/status has its own structure | โš ๏ธ ยง6 HS-03 | + +### 4.5 admin/CLI/API surface comparison + +| MinIO | RustFS | Status | +|---|---|---| +| `POST /minio/admin/v3/heal/...` start/status/cancel | `POST /rustfs/admin/v3/heal/...` same three states | โœ… (different path prefix is expected) | +| `HealStartSuccess`/`HealTaskStatus`/`HealResultItem`/DriveState | same-named fields JSON-compatible | โœ… | +| `POST /v3/background-heal/status` (BgHealState aggregate) | same path + degraded semantics + operations matrix | โœ… enhanced (no MRF per-endpoint sub-state, because there is no MRF) | +| `GET /v3/healthinfo` per-drive `HealInfo *HealingDisk` | no equivalent healthinfo heal field (replacement-recovery v4 covers part of it) | โš ๏ธ ยง6 HS-07 | +| madmin client HealStart/HealStatus/BackgroundHealStatus/ScannerStatus methods | wire types only, no client methods | โŒ ยง6 HS-05 | +| mc admin heal --pool/--set, --scan-mode, --force-start/stop | HealOpts full field support (pool/set/scanMode/forceStart/forceStop) | โœ… (server-side ready; missing the mc-side entry, HS-05) | +| ErrHealAlreadyRunning / ErrHealOverlappingPaths typed errors | dedup-merge + eviction semantics; no typed overlap rejection | โš ๏ธ ยง6 HS-06 | +| result backpressure (maxUnconsumedItems=1000, 10s keep-alive streaming, 24h unconsumed abort) | snapshot-style query (1024 entries + 8MiB truncation + 10min retention) | โš ๏ธ ยง6 HS-06 | +| `mc support inspect`/healing-bin offline dump | none (inspect.rs exists but the healing dump is unconfirmed) | โš ๏ธ P3 | + +### 4.6 observability surface comparison + +| Dimension | MinIO | RustFS | Status | +|---|---|---|---| +| heal metrics | minio_heal_objects_total/heal_total/errors_total/time_last_activity + v3 drive_health 2=healing | full rustfs_heal_* suite (admission/queue delay/running/throttle/page concurrency) | โœ… (RustFS lacks an equivalent of the single drive_health=healing gauge; DiskInfo.healing is already assigned) | +| scanner metrics | v3 6 + realtime 18 items | full rustfs_scanner_* suite + per-source dimensions | โœ… | +| ILM metrics | v3 5 (expiry/transition pending/active/missed + versions_scanned) | ilm expiry status API + scanner per-source | โœ… (different metrics and API shape) | +| trace | TraceHealing/TraceScanner channels | none | โŒ ยง6 HS-03 | +| auditing | HealObject events, dangling-deletion audit, scanner:manyversions etc. | structured logs (event style) + metrics; no audit-log events | โš ๏ธ ยง6 HS-04 | +| progress | healingTracker Bytes/Items/QueuedBuckets/current object + usage-cache total baseline | HealProgress{scanned/healed/failed/bytes/current_object/percentage}; bytes_processed annotated as 0, estimated_completion_time always None | โš ๏ธ ยง6 HS-07 | + +### 4.7 configuration surface comparison (defaults) + +| MinIO | RustFS | Notes | +|---|---|---| +| `heal:bitrotscan` (default off; on=every cycle; Nm=Nร—30ร—24h) | `heal.bitrot_cycle` / `RUSTFS_SCANNER_BITROT_CYCLE_SECS` (default 30d=2592000s; 0/on=Deep every cycle, off=disabled) | โœ… same semantics (RustFS default 30d, MinIO default off โ€” **different defaults**, RustFS more aggressive) | +| `heal:max_io=100`/`max_sleep=250ms` (waitForLowIO) | mainline throttle thresholds 80%/80%, max_sleep 250ms | โœ… same shape (different threshold model) | +| `heal:drive_workers` (default -1 auto) | in-page concurrency 8 + per-set 1 | โœ… same shape | +| `_MINIO_HEAL_WORKERS` (GOMAXPROCS/2) | `RUSTFS_HEAL_MAX_CONCURRENT_HEALS=4` + `_MAX_CONCURRENT_PER_SET=1` | โœ… | +| `_MINIO_AUTO_DRIVE_HEALING` (on) | `RUSTFS_HEAL_AUTO_HEAL_ENABLE=true` | โœ… | +| `_MINIO_SCANNER` (on) | `RUSTFS_SCANNER_ENABLED=true` | โœ… | +| `scanner:speed` five tiers (default=2x/1s/1m) | same five tiers, same names, same parameters | โœ… | +| `scanner:idle_speed` (on) | `RUSTFS_SCANNER_IDLE_MODE` (true) | โš ๏ธ semantic direction (HS-14) | +| `scanner:alert_excess_versions=100` | 100 | โœ… | +| `scanner:alert_excess_folders=50000` | 65538 (compatible with the PBS layout) | โš ๏ธ HS-17 | +| `ilm:expiration_workers=100`/`transition_workers=100` | ecstore expiry/transition worker pools (keys under the ilm subsystem) | โœ… (defaults not checked item by item) | +| `api:stale_upload_cleanup_interval=6h`/`expiry=24h` | ecstore background task, configurable via env | โœ… (defaults not checked item by item) | +| โ€” (none) | `RUSTFS_HEAL_QUEUE_SIZE=10000`, `_TASK_TIMEOUT_SECS=300`, `_INTERVAL_SECS=10`, `_LOW_PRIORITY_MERGE/DROP`, `_PAGE_*`, `_SET_BULKHEAD`, `_MAINLINE_*`, `RUSTFS_SCANNER_CYCLE_MAX_*` budgets, `_MAX_CONCURRENT_SET/DISK_SCANS=4`, `_YIELD_EVERY_N_OBJECTS=128`, etc. | RustFS-specific (finer-grained) | + +### 4.8 Where RustFS exceeds MinIO + +1. remote_scanner RPC (scan execution pushed down to the remote peer locally, with HMAC authentication/replay cache/fence re-validation/disconnect grace). +2. Persisted leader-epoch CAS fence + usage-snapshot epoch/cycle anti-rollback (MinIO has only the lock, no persisted epoch). +3. Cycle budgets (max_duration/objects/directories) + partial-cycle advancement semantics. +4. per-set/per-disk scan concurrency gates + a cache lock per bucket per set. +5. pending-heal ledger (heal candidates are not lost when the heal channel is full). +6. Drive-replacement durable intent + completion proof state machine + identity fence (MinIO's healingTracker has no proof). +7. mainline throttle foreground pressure gating (driven by permit utilization). +8. Cluster heal control coordinator + envelope replay protection + explicit degraded fallback. +9. Write-path shard bitrot self-verification (the EC:0 case). +10. dirty-usage fast-path wakeup (immediate write-path notification + dirty buckets first). +11. heal runtime observability matrix (priorityร—source operations snapshot). +12. workload admission integration (the heal scheduler reads the foreground pressure snapshot). + +--- + +## 5. Gap and improvement list + +Severity definitions: P1 = behavioral/operational alignment gap (affects production operations or toolchain compatibility); P2 = completeness (the feature exists but is missing a corner); P3 = cleanup/low risk. Each item includes current-state evidence, MinIO behavior, impact, recommendation, and acceptance. + +### P1 (8 items) + +**HS-01 The MRF/ECDecode/Metadata heal task types have no production trigger; HealEvent unwired** +- Current state: the `HealType::MRF/ECDecode/Metadata` executors are complete (task.rs:1700-2156) but have no production trigger anywhere in the repo; `HealEvent`/`HealEventHandler` (event.rs:50-367) has zero references outside the crate (verified first-hand by grep); channel conversion produces only Cluster/Object/Bucket/Prefix/ErasureSet (channel.rs:566-601). +- MinIO: mrf.go has a standalone MRF queue (capacity 100k, drop-and-count when full), msgp persistence to `.heal/mrf/list.bin` at process exit + startup replay, 1s delay for enqueues <1s (waiting for network recovery), healSleeper rate limiting; on the read path, GetObject part missing/corrupt, metadata rebuild missingBlocks>0, partial Put success, DeleteObject, multipart, and the peer client add up to 7+ delivery points. +- Impact: RustFS's read-repair + write-path convergence covers the main scenarios, but lacks: โ‘  an event-driven Urgent ECDecode rebuild entry (on ecstore decode failure there is currently only Low read-repair); โ‘ก a metadata-only heal entry (the scanner's HealMetadata classification exists but goes through ordinary object heal); โ‘ข MRF queue persistence (unconsumed repair intents are lost on restart โ€” partially mitigated by the scanner's pending-heal ledger). +- Recommendation: a pick-one-of-three decision โ€” (a) wire HealEvent (emit events at ecstore decode-failure/metadata-corruption points) + implement a persistent retry ledger; (b) delete the MRF/ECDecode/Metadata dead code and keep only a documentation note; (c) keep the executors and demote HealEvent to an internal API. (a) is recommended, but first quantify whether read-repair already meets the response-time requirements for decode-failure scenarios. +- Acceptance: an e2e decode-failure โ†’ Urgent heal-request chain; replay of pending repair intents after restart; HealEvent ring-buffer metrics. + +**HS-02 CheckAbandonedParts NotImplemented at all three layers (missing standalone abandoned-data reconciliation entry)** +- Current state: `set_disk/ops/heal.rs:2052-2056`, `core/sets.rs:1144-1148`, `store/heal.rs:258-266` explicitly return `Err(NotImplemented)` at all three layers (verified first-hand); the comment reads "intentionally retained above the set layer until there is a concrete caller". +- MinIO: `CheckAbandonedParts` โ†’ per-drive `CleanAbandonedData`: read xl.meta โ†’ list UUID data-dirs + inline entries โ†’ diff against getDataDirs โ†’ delete surplus data-dirs/inline entries and rewrite xl.meta; invoked explicitly on scanner-sampled heals and admin heal Remove. +- Impact: RustFS's in-heal-path `reclaim_orphan_data_dirs_best_effort` (:1428) covers "reclaim orphan directories while healing", but โ‘  there is no standalone trigger point (MinIO can also clean abandoned data before an object reaches the heal threshold); โ‘ก orphan inline-data entry cleanup is unconfirmed; โ‘ข multipart orphan reconciliation is explicitly out of scope (a design decision, carried by lifecycle). +- Recommendation: evaluate promoting `reclaim_orphan_data_dirs_best_effort` to a fixed step of heal_object (if it is not already) + implement a real HealOperations::check_abandoned_parts (calling the same reclamation logic), or explicitly document "carried by lifecycle" and close the API surface. +- Acceptance: construct data-dir/inline orphans โ†’ cleaned after scanner sampling/admin heal; the three-layer API returns success or an explicitly documented NotSupported. + +**HS-03 heal/scanner trace channels missing** +- Current state: zero hits for TraceHealing/TraceScanner (verified first-hand by grepping the whole repo). +- MinIO: `madmin.TraceHealing` (mc admin trace --healing, FuncName=heal.Bucket/heal.Object/heal.CheckAbandonedParts, with dry/remove/mode/version-id/disks/bytes), `TraceScanner` (mc admin scanner trace, supports --filter-size/--response-duration). +- Impact: no way to observe in real time the latency and parameters of individual heal/scanner actions; troubleshooting can rely only on aggregated metrics and logs. +- Recommendation: instrument heal-channel execution and scanner folder/item handling, and hook them into the existing admin trace subscription surface (reuse the rustfs trace infrastructure if it exists; otherwise extend it per madmin TraceType). +- Acceptance: an mc-equivalent tool can subscribe to the heal/scanner trace stream. + +**HS-04 Scanner excess S3 events and auditing missing** +- Current state: only `rustfs_scanner_excess_*_total` metrics (versions 100 / version size 1TiB / folders 65538). +- MinIO: emits `s3:ObjectManyVersions` (>100 versions), `s3:ObjectLargeVersions` (cumulative >1TB), `s3:PrefixManyFolders` (>50000 subdirectories) events (UserAgent: Scanner) + scanner:manyversions/largeversions/manyprefixes auditing. +- Impact: users relying on event subscriptions for capacity governance (console/external auditing) receive no alerts. +- Recommendation: hook the scanner_folder alert points into notify event publishing (reusing the lifecycle event-channel semantics). +- Acceptance: after configuring bucket notifications, an over-threshold object triggers an event. + +**HS-05 madmin client methods missing** +- Current state: `crates/madmin/src/heal_commands.rs` has only wire types (HealDriveInfo/Infos/HealResultItem); no HealStart/HealStatus/BackgroundHealStatus/ScannerStatus client methods. +- MinIO: madmin-go provides the full client; mc admin heal/scanner/status/trace are all built on it. +- Impact: admin tools like mc cannot directly drive the RustFS heal/scanner admin surface; automated operations must hand-write HTTP. +- Recommendation: add the client following the madmin-go interface shape (the server side is ready; this is pure client work). +- Acceptance: complete the startโ†’queryโ†’cancel flow with the madmin client. + +**HS-06 admin heal sequence semantics differ from MinIO** +- Current state: duplicate/overlapping requests are dedup-merged (returning the canonical task_id) or evicted; no ErrHealAlreadyRunning/ErrHealOverlappingPaths typed errors (verified first-hand: manager.rs:1309's already_running is an idempotent-startup guard, not an admin semantic); results are snapshot-style queries (1024 entries/8MiB truncation/10min retention), not MinIO's streaming increments (clientToken pulls increments + maxUnconsumedItems=1000 backpressure + 10s keep-alive + 24h unconsumed abort). +- Impact: mc admin heal's interaction model (long connection pulling increments) behaves against RustFS as multiple snapshot polls; automation scripts cannot easily distinguish "merged" from "newly started". +- Recommendation: โ‘  incremental semantics: channel query supports item increments since the last clientToken (or a cursor); โ‘ก overlapping requests return a typed error code (or an explicit merged_into field in the receipt โ€” the existing alias mechanism already provides the base); โ‘ข verify forceStart's stop-old-then-start-new semantics. +- Acceptance: an madmin-compatible client polling in the MinIO style can retrieve the full item set. + +**HS-07 healing progress and drive-level healing state insufficiently visible externally** +- Current state: byte-recovery progress `progress.bytes_processed = 0 // set to 0 for now` (erasure_healer.rs:967); `HealProgress::estimated_completion_time` is always None and `HealStatistics::add_healed_objects` is never written (progress.rs:38,135-139 zero calls); healthinfo has no per-drive HealInfo equivalent (MinIO HealingDisk: BytesDone/Failed/Skipped, ObjectsTotal baseline, QueuedBuckets/HealedBuckets, Resume snapshot, current object); v3 metrics lack an equivalent of the single drive_health=2 (healing) gauge. +- Impact: during a drive rebuild (potentially hours to days) operations cannot answer "where are we / how much is left / when will it finish". +- Recommendation: โ‘  accumulate bytes in erasure set heal (heal_object already yields the object size); โ‘ก read the object-total baseline from usage-cache (the same approach as MinIO); โ‘ข expose a per-drive healing snapshot in admin healthinfo/background status (DiskInfo.healing already exists; add the aggregated exposure); โ‘ฃ derive the ETA from baseline + rate. +- Acceptance: during a drive rebuild, admin shows byte progress and ETA; an mc info-equivalent output shows the Healing flag. + +**HS-08 prefix-level usage not exposed** +- Current state: the DataUsageCache holds the directory-tree entries (organized by hash_path), but `dui()` flattens only to the bucket name (data_usage_define.rs:858-915). +- MinIO: `loadPrefixUsageFromBackend` (30s cache) aggregates prefix usage from each set's `.usage-cache.bin`, consumed by console bucket-prefix statistics. +- Impact: console/front ends cannot show prefix-level usage; there is no API to locate "which prefix is using the space" in a large bucket. +- Recommendation: implement a prefix-flattening query API (the data is already in the cache; this is pure aggregation and exposure work). +- Acceptance: a ListBuckets/PrefixUsage API returns statistics matching the prefix filter. + +### P2 (9 items) + +**HS-09 get_disk_status always returns Ok (the only TODO)**: `crates/heal/src/heal/storage.rs:930-943` (verified first-hand). Currently no production caller (low risk). Recommendation: delete the method or wire it to the real ecstore disk status (the DiskStatus enum is already defined). + +**HS-10 About 1/3 of HealStorageAPI methods are dead code**: get_object_meta/get_object_data/put_object_data/delete_object/verify_object_integrity/ec_decode_rebuild/get_disk_status/format_disk/heal_bucket_metadata/get_object_size/get_object_checksum/list_objects_for_heal (the non-paginated version, with its own memory_heavy warning) all have 0 callers. Recommendation: clean up or wire them together with the HS-01 decision (dead interfaces mislead future maintainers into thinking a call path exists). + +**HS-11 bitrot self-test missing**: MinIO at startup runs bitrotSelfTest over known vectors for the four algorithms and exits Fatal on failure (guarding against silent data corruption). RustFS has no equivalent (verified first-hand by grep). Recommendation: at startup, run known-vector self-tests for HighwayHash256S and the other algorithms in use (low cost, high value). + +**HS-12 object-level healing metadata marker evaluation**: during heal, MinIO tags objects with `x-minio-healing:true`, and RenameData uses it to skip version cleanup/legacy purge (missing it lets heal and concurrent deletes destroy each other). RustFS has no object-level marker (verified first-hand by grep; object.rs has no healing branch) and relies on NSLock + rename semantics. Recommendation: audit whether the RustFS rename-commit path has a "heal commit racing concurrent delete/version cleanup" window; if not, document the difference, and if so, add a marker-equivalent mechanism. + +**HS-13 erasure set heal lacks "skip newly written / ILM-expired versions" filters**: MinIO resync skips versions with ModTime>tracker.Started (so heal does not chase the tail of new writes) and ILM-expired versions (so work is not wasted). RustFS's erasure_healer does not implement such filters (per-version dedup exists; time/ILM filters do not). Impact: a long tail on rebuild completion (the completion decision for a continuously written bucket is pushed out by new versions) and wasted heal work. Recommendation: add a started_at time filter at the disk-walk enumeration point + an evaluator pre-check. + +**HS-14 scanner idle semantics point the opposite way from MinIO**: MinIO `scanner:idle_speed=on` (default) means "throttle only when the cluster is idle, full speed when busy"; RustFS `RUSTFS_SCANNER_IDLE_MODE=true` (default) is a master switch for rate limiting (false = never sleep at all). The default behaviors may end up similar (both throttle), but the parameter semantics are not interchangeable; migration docs must state this explicitly; if mc config compatibility is the goal, a rename/re-semantization is needed. Recommendation: document the difference first, then evaluate aligning the semantics. + +**HS-15 alert_excess_folders default differs**: RustFS 65538 (compatible with the PBS/Proxmox layout, scanner_folder.rs:79) vs MinIO 50000. The behavioral difference is that the trigger threshold differs out of the box. Recommendation: document it (keeping 65538 has local rationale). + +**HS-16 single-node default-cycle hook not enabled**: `single_disk_default_cycle_secs(_features) -> None` is always empty (scanner.rs:1428-1430); single-node deployments get no dedicated default-cycle override. Recommendation: after deciding the single-node default-cycle policy, enable or delete the hook. + +**HS-17 DeleteAllVersions batch-optimization check**: MinIO uses the single DeletePrefix+DeletePrefixObject call instead of per-version fan-out. Whether RustFS's expiry-queue path has the same optimization was not verified line by line (integration tests cover behavioral correctness). Recommendation: check the `apply_expiry_rule` all-versions delete path; if there is no prefix single-call optimization, evaluate adding it. + +### P3 (3 items) + +**HS-18 trash/temp-directory two-stage cleanup detail check**: MinIO cleans `.minio.sys/tmp/.trash` (delete_cleanup_interval default 5m + deleteCleanupSleeper) and stale uploads are renamed into trash in two stages. RustFS has delete_tail_activity.rs and the stale multipart task; whether the two-stage semantics are fully aligned was not verified line by line. Recommendation: align or document. + +**HS-19 root-heal direct path is dead code**: `should_handle_root_heal_directly` is always false (admin/handlers/heal.rs:1200-1202, locked by a test); the store.heal_format direct branch is unreachable. Recommendation: delete the dead branch or restore the direct path as a fallback for cluster-coordination failure. + +**HS-20 compat flags and dead metrics cleanup**: `RUSTFS_SCANNER_INLINE_HEAL_ENABLE` (enabling only warns) + the dead `rustfs_scanner_inline_heal_total` metric + the scanner-domain code in `rustfs_common::metrics` awaiting layering migration (backlog #1843 already filed). Recommendation: clean up along with the layering migration. + +### Not pursuing parity by design (7 items, recorded to prevent later misreading as gaps) + +1. **bloom filter**: removed from MinIO master; RustFS reuses `.bloomcycle.bin` as the cycle/epoch fence, consistent with MinIO's current state. +2. **scanner cluster single leader**: both sides agree; RustFS additionally has the epoch fence. +3. **heal emits no S3 bucket notification**: both sides agree (heal results go through admin status). +4. **incomplete multipart not executed inside scanner/ILM**: both sides agree (independent background routine). +5. **inline heal removal**: a deliberate RustFS choice (the scanner only enqueues); MinIO's applyHealing inline path is not a parity target. +6. **heal-sequence resident keep-alive (10s blank write-back)**: RustFS's snapshot-query model differs; handling incremental semantics per HS-06 is enough โ€” do not copy the streaming keep-alive. +7. **`.trash`/`tmp-old` path-name compatibility**: RustFS's layout constants are independent; no literal alignment with MinIO paths. + +--- + +## 6. Configuration defaults master table (RustFS) + +heal (env prefix `RUSTFS_HEAL_`, `crates/config/src/constants/heal.rs`, consumed at `manager.rs:724-800`): + +| Setting | Default | Hot update | +|---|---|---| +| AUTO_HEAL_ENABLE | true | no | +| QUEUE_SIZE | 10000 | no | +| INTERVAL_SECS | 10 | no (fixed at startup) | +| TASK_TIMEOUT_SECS | 300 | no | +| MAX_CONCURRENT_HEALS | 4 | no | +| MAX_CONCURRENT_PER_SET | 1 (โ‰คmin(global, value)) | no | +| LOW_PRIORITY_MERGE_ENABLE | true | no | +| LOW_PRIORITY_DROP_WHEN_FULL | true | no | +| PAGE_OBJECT_CONCURRENCY | 8 (Deep/AutoHeal forced to 1) | no | +| EVENT_DRIVEN_SCHEDULER_ENABLE | true | no | +| SET_BULKHEAD_ENABLE | true | no | +| PAGE_PARALLEL_ENABLE | true | no | +| MAINLINE_THROTTLE_ENABLE | true | no | +| MAINLINE_READ/WRITE_UTILIZATION_HIGH_PERCENT | 80/80 | no | +| MAINLINE_MAX_SLEEP_MS | 250 | no | +| (master switch) RUSTFS_HEAL_ENABLED | true | no | +| admin subsystem heal.bitrot_cycle | 30d | yes (via scanner runtime config) | + +scanner (admin subsystem `scanner`, `crates/config/src/constants/scanner.rs` + `ecstore/src/config/scanner.rs` + `runtime_config.rs:527-673`): + +| Key | env | Default | +|---|---|---| +| speed | RUSTFS_SCANNER_SPEED | default (2x/1s/60s) | +| delay / max_wait / cycle / start_delay | RUSTFS_SCANNER_* | derived/empty | +| cycle_max_duration/objects/directories | โ€ฆ_MAX_* | 0 (unlimited) | +| bitrot_cycle | โ€ฆ_BITROT_CYCLE_SECS | 2592000 (30d; 0/on=every cycle, off=disabled) | +| idle_mode | โ€ฆ_IDLE_MODE | true | +| cache_save_timeout | โ€ฆ_CACHE_SAVE_TIMEOUT_SECS | 30s | +| max_concurrent_set_scans / disk_scans | โ€ฆ_MAX_CONCURRENT_* | 4/4 | +| yield_every_n_objects | โ€ฆ_YIELD_EVERY_N_OBJECTS | 128 | +| alert_excess_versions / version_size / folders | โ€ฆ_ALERT_* | 100 / 1TiB / 65538 | + +scanner-internal env: `RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=16`, `RUSTFS_HEAL_OBJECT_SELECT_PROB=1024`, `RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS=60`, `RUSTFS_DATA_USAGE_FAILED_OBJECT_TTL_SECS=86400`/`_MAX=10000`, `RUSTFS_LOCK_ACQUIRE_TIMEOUT=5s`, `RUSTFS_SCANNER_ENABLED=true`, `RUSTFS_SCANNER_INLINE_HEAL_ENABLE=false` (compat warning). + +All 17 scanner keys support the env > config dual channel + admin PUT hot update (generation+Notify takes effect immediately); heal runtime parameters are currently env-only (no admin hot-update entry; the `Arc>` structure is already reserved). + +--- + +## 7. Related backlog / history index + +- Automatic drive-replacement healing series (closed loop): backlog #1786 (redundant false-green algorithm), #1787 (target-slot restriction), #1789 (binding resume and the healing marker to the replacement instance), #1791 (black-box/white-box acceptance matrix). +- #801 DiskInfo.healing never assigned (fixed and closed; the assignment chain now lives at `set_disk/mod.rs:4988`). +- #1651 Scanner metrics node/source/bucket-drive dimensions (OPEN; related to ยง3.8/ยง4.6 of this analysis). +- #1843 crates/common 83% scanner/heal domain code layering migration (OPEN; includes HS-20). +- Historical defects cited in code comments (now guarded with regression tests): #856/#799 B7 (offline drive falsely recorded healed), #855/B6/#1033 (a skip round must not be marked complete), #920 (sub-quorum union enumeration), #856 B5 (per-version resume), #5173 (bitrot trailing bytes), #5029 (stale-version merge at regression nodes). +- v1 parity document: `docs/rustfs-heal-scanner-vs-minio-parity-assessment.md` (superseded by this document); the landing playbook `docs/rustfs-heal-scanner-vs-minio-improvement-playbook.md` (some entries have since been overtaken by implementation). +- Drive-replacement deep analyses: `docs/new-disk-replacement-and-healing-deep-analysis-zh.md`, `docs/node-disk-identity-and-healing-analysis-zh.md`. + +## 8. Audit method and limitations + +- Four parallel audit tracks (heal crate file by file, scanner crate file by file, ecstore integration-layer wiring, MinIO master source study) + the main session verifying each key "missing" conclusion first-hand (the get_disk_status TODO, HealEvent's zero external references, .bloomcycle.bin having no bloom implementation, check_abandoned_parts NotImplemented at all three layers, the ETag fallback being implemented, zero trace-channel hits, the already_running semantics). +- Points not verified line by line (marked "unconfirmed / not checked line by line" in the text): the DeleteAllVersions prefix single-call optimization (HS-17), trash two-stage cleanup details (HS-18), ilm worker default comparisons, stale multipart default comparisons, mc CLI flag spellings (MinIO side). Of these, HS-17 and HS-18 completed line-by-line verification on 2026-08-19; conclusions in ยง9.2/ยง9.3. +- MinIO-side references follow its master `7aac2a2c5b`; RustFS-side line numbers follow the 2026-08-16 workspace โ€” for later evolution, search by symbol name instead. + +## 9. Landing results (updated 2026-08-19) + +All 14 sub-issues derived from this audit (backlog #1865~#1878) are closed. This section is the final disposition record for the gap list HS-01~HS-20, and also the incremental baseline for the next parity re-audit. + +### 9.1 Landed (all PRs merged to main) + +- HS-01 MRF wiring + persistent repair ledger (#1865, PR #6189): decision (a) chosen. common MRF channel (bounded 8192, try_send never blocks) + heal mrf_queue (100k entries / 8MiB dual-capacity ring) + `buckets/.heal/mrf/journal.bin` CRC-persisted replay (torn tail truncated, deleted after replay) + three delivery points (read decode_errorโ†’Urgent ECDecode, scanner metadata corruptionโ†’High Metadata, add_partialโ†’Normal) + `RUSTFS_HEAL_MRF_ENABLE` one-switch rollback. +- HS-02 abandoned parts/data-dir reconciliation (#1866, PR #6179): wired up the abandoned-check entry, retaining dry-run / reclaim counters. +- HS-03 heal/scanner trace channels (#1867, PR #6179): in-process trace bus + `/v3/trace` admin streaming subscription + heal task / abandoned-parts / scanner folder / ILM / heal-candidate trace producers. +- HS-04 scanner excess S3 events (#1868, PR #6176): the three events `s3:Scanner:ManyVersions/LargeVersions/BigPrefix` + 24h edge cooldown; the HS-15 threshold delta documented (`docs/operations/scanner-excess-alerts.md`). +- HS-05 madmin client phase 1 (#1869, PR #6166): SigV4 admin client heal/scanner methods; incremental-consumption methods await a follow-up (the protocol was already folded in by HS-06). +- HS-06 admin heal incremental semantics and typed overlap (#1870, PR #6206): `sinceSeq/nextSeq/minSeq` incremental cursor (wire additive; absent = full snapshot) + `RUSTFS_HEAL_OVERLAP_POLICY` (default merge unchanged; under minio_error, typed AlreadyRunning/OverlappingPaths rejections) + forceStart stops the old sequence before starting the new one. +- HS-07 healing progress visibility (#1871, PR #6179): data-usage total baseline + baseline/current/healed counters. +- HS-08 prefix usage (#1872, PR #6171): `GET /v3/usage/{bucket}`. +- HS-11 bitrot startup self-test (#1873, PR #6165). +- HS-13 heal skip filters (#1875, PR #6179): filter-hit versions are no longer counted as failures. +- HS-16 single-node cycle hook (#1878, PR #6250): removed the always-None hook; the decision record is in `docs/operations/heal-scanner-parity-notes-zh.md`. +- HS-09/10/19/20 dead-code cleanup batch (#1877, PR #6256): net โˆ’911 lines, zero behavior change; the `get_disk_status` TODO (the repo's only product TODO) cleared to zero; `ec_decode_rebuild`/`get_object_meta`, kept due to the HS-01 linkage, are retained with Reserved annotations (MRF currently executes via `heal_object`). + +### 9.2 Confirmed "already implemented / not a gap" after verification (audit-period misjudgment corrections, four in total) + +- bloom filter (corrected in ยง0): removed from MinIO master; both sides now agree. +- ETag fallback arbitration (corrected in ยง0): RustFS already has the implementation (`set_disk/ops/heal.rs`). +- HS-17 (#1876, closed after line-by-line verification on 2026-08-19): the DeleteAllVersions prefix single-call optimization is fully implemented in RustFS โ€” `apply_expiry_on_non_transitioned_objects` sets `delete_prefix + delete_prefix_object` for the two `delete_all()` actions and then performs a single `delete_object` call (`bucket_lifecycle_ops.rs:5047-5056`); the SetDisks branch takes one write lock + one all-version quorum read + inline per-version object-lock checks (`set_disk/ops/object.rs:5566-5612`), aligned line by line with MinIO `expire.go`'s `applyExpiryOnNonTransitionedObjects`. The item ยง8 listed as "not verified line by line" now has a conclusion: the current state is already the optimized path; nothing to implement. +- HS-14 (#1878, checked alongside PR #6250): MinIO's "idle = throttle only when idle" was the behavior before 2024-01 minio/minio#18734 (`scannerIdleMode` is now a static config; `idle_speed=on` by default means always throttling per the speed tier โ€” the "idle" naming is a historical leftover); RustFS's `RUSTFS_SCANNER_IDLE_MODE` points the same way as MinIO's current semantics, and additionally has a foreground-read backoff floor that MinIO lacks. The real migration traps (the variable must carry the `RUSTFS_` prefix, the `on/off` vs `true/false` vocabulary, `false` also turning off foreground protection) are documented in `docs/operations/heal-scanner-parity-notes-zh.md`. + +### 9.3 Audit-style conclusions (no code change needed) + +- HS-12 (#1874, PR #6183): the class of race MinIO defends against with `x-minio-healing` does not exist โ€” every commit surface for the same (bucket, object) is mutually exclusive under the same object-level ns write lock, and the heal lock guard covers the whole rename commit; delivered 2 concurrency-invariant regression tests + the intersection matrix in `docs/operations/heal-concurrency-safety-notes-zh.md`. +- HS-18 (#1878, line-by-line verification on 2026-08-19): trash/tmp three-stage cleanup fully aligned โ€” stale multipart isolation-cleanup is equivalent and safer (`delete_all_with_quorum` recursively deletes per drive, i.e. the `move_to_trash` rename into `.rustfs.sys/tmp/.trash`, plus lock + fence); trash draining is essentially equivalent (no per-entry sleeper throttling; the 5m cycle naturally rate-limits); tmp non-trash 24h reclamation is equivalent (RustFS's 5m is more timely than MinIO's 6h); the three cycle defaults 24h/6h/5m all align. The item ยง8 listed as "not verified line by line" now has a conclusion. + +### 9.4 Handed over to follow-ups (summarized in the backlog#1862 comment thread) + +HS-01 bitrot GETโ†’MRF full-chain e2e, kill -9 journal replay e2e, queue-full RSS stress test (โ‰ค budget+10%); HS-05/06 madmin incremental-consumption methods + single-source wire + embedded e2e + multi-round polling soak; HS-08 multi-drive scanner cycle e2e; HS-04 excess audit entries; HS-18 the stale-multipart crash-residue window below quorum (crashing mid-fan-out with already-cleaned drives > parity means FileNotFound is not in the ignore set, so convergence is unnatural; the fix needs a dedicated quorum variant). + +Recommendation for the next re-audit: trigger it after the next big heal/scanner feature lands, using this section as the incremental baseline. diff --git a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md new file mode 100644 index 000000000..a75971d1a --- /dev/null +++ b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md @@ -0,0 +1,568 @@ +# RustFS heal / scanner ๅ…จ้‡ๅŠŸ่ƒฝๅˆ†ๆžไธŽ MinIO ๅฏนๆ ‡๏ผˆv2๏ผ‰ + +> English version: [rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md](rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md) + +- ๆ—ฅๆœŸ๏ผš2026-08-16๏ผˆๅŸบไบŽ main ๅˆ†ๆ”ฏๅฝ“ๆ—ฅไปฃ็ ๏ผŒๅฎก่ฎกๆ—ถ HEAD โ‰ˆ `a118d7e4f`๏ผ‰ +- ่Œƒๅ›ด๏ผš`crates/heal`๏ผˆsrc 19,560 ่กŒ + tests 2,274 ่กŒ๏ผ‰ใ€`crates/scanner`๏ผˆsrc ็บฆ 26,000 ่กŒ + tests๏ผ‰ใ€`crates/data-usage`ใ€`crates/ecstore` ไธญ heal/heal_walk/bitrot_self_verify ไธŽ configใ€`crates/common/src/heal_channel.rs`ใ€`crates/madmin`๏ผˆheal/scanner wire ็ฑปๅž‹๏ผ‰ใ€`rustfs/src`๏ผˆstartup wiringใ€admin handlersใ€้›†็พค RPC๏ผ‰ +- ๅฏนๆ ‡ๅŸบ็บฟ๏ผšminio/minio master๏ผˆHEAD `7aac2a2c5b`๏ผŒไป“ๅบ“ๅทฒ่ฟ›ๅ…ฅ็ปดๆŠคๆจกๅผ๏ผŒmaster ๅ†ป็ป“๏ผŒๅณๆœ€็ปˆๆ€๏ผ‰ +- ๆ–นๆณ•๏ผšๅ››่ทฏๅนถ่กŒๅฎก่ฎก๏ผˆheal crate / scanner crate / ecstore ้›†ๆˆๅฑ‚ / MinIO ๆบ็ ็ ”็ฉถ๏ผ‰๏ผŒๅ…ณ้”ฎ็ป“่ฎบ้€ๆกไบบๅทฅๆŠฝ้ชŒ๏ผˆๆ–‡ๅ†…ๆ ‡ๆณจ"ๅทฒไบฒ้ชŒ"ๅค„ไธบไธ€ๆ‰‹้ชŒ่ฏ๏ผ‰ +- ๆœฌๆ–‡ๆกฃๅ–ไปฃ `docs/rustfs-heal-scanner-vs-minio-parity-assessment.md`๏ผˆ2026-06-15 v1๏ผ‰ใ€‚v1 ไน‹ๅŽ heal/scanner ็›ธๅ…ณๆไบค่ถ…่ฟ‡ 80 ไธช๏ผˆๆข็›˜่‡ชๅŠจไฟฎๅคๅ…จ้“พ่ทฏใ€resume ็Šถๆ€ๆœบใ€usage ๆ”ถๆ•›ๆƒๅจๅŒ–ใ€้›†็พค็บง heal ๅ่ฐƒใ€ILM restore ่ฏญไน‰็ญ‰๏ผ‰๏ผŒv1 ็š„ๅŠŸ่ƒฝๆธ…ๅ•ไธŽๅทฎ่ทๅˆคๆ–ญๅทฒๅ…จ้ข่ฟ‡ๆ—ถ๏ผ›v1 ไธญ"bloom filter ็ผบๅคฑ"็ญ‰็ป“่ฎบ็ปๆœฌๆฌกๆ ธๅฎžไธบ**่ฏฏๅˆค**๏ผˆ่ฏฆ่ง ยง5.4๏ผ‰ใ€‚ + +--- + +## 0. ็ป“่ฎบๆ‘˜่ฆ + +1. **ๆ€ปไฝ“ๅˆคๆ–ญ๏ผšheal ไธŽ scanner ็š„ๆ ธๅฟƒๅŠŸ่ƒฝ้“พ่ทฏๅทฒ็ปๅฎŒๆ•ด**ใ€‚ๅฏน่ฑก็บง heal๏ผˆquorum ไปฒ่ฃ + ETag ๅ…œๅบ• + bitrot Deep ๆ ก้ชŒ + dangling ๅค„็†๏ผ‰ใ€erasure set ๆทฑๆ‰ซ๏ผˆper-set disk-walk ๅนถ้›†ๆžšไธพ๏ผ‰ใ€ๆŒ‰็‰ˆๆœฌๆ–ญ็‚น็ปญๆ‰ซ๏ผˆschema ๅŒ–ๆŒไน…ๅฑ‚ + CAS ๅŽŸๅญๅ‘ๅธƒ + ๅดฉๆบƒ็ช—ๅฃ่กฅ้ฝ๏ผ‰ใ€ๆข็›˜่‡ชๅŠจไฟฎๅค๏ผˆreadiness ๆ ก้ชŒ + ่บซไปฝๅ›ดๆ  + durable intent + completion proof๏ผ‰ใ€scanner ๅ‘จๆœŸๅพช็Žฏ๏ผˆleader lock + ๆŒไน…ๅŒ– leader-epoch ๅ›ดๆ ๏ผ‰ใ€data usage ็ปŸ่ฎก๏ผˆๆกถ็บง/้›†็พค็บงใ€ไธป+ๅค‡+่ง‚ๆต‹ๅฟซ็…งใ€epoch/cycle ้˜ฒๅ›ž้€€๏ผ‰ใ€ILM ๅ…จๅŠจไฝœ๏ผˆexpiry/transition/noncurrent/free-version/delete-marker ๆธ…็†๏ผ‰ใ€admin Start/Query/Cancel ๅ่ฎฎ๏ผˆclientToken ่ฏญไน‰ๅฏน้ฝ madmin๏ผ‰โ€”โ€”ไปฅไธŠๅ‡ๆœ‰ๅฎž็Žฐไธ”ๅธฆๅ›žๅฝ’ๆต‹่ฏ•ใ€‚ไธคไธช crate ๅ†…**ๆฒกๆœ‰็ฉบๅฎž็Žฐ/ๆ—ฉ้€€ๆกฉ**๏ผŒๅผ‚ๅธธ่ทฏๅพ„ๅ…จ้ƒจๆœ‰ๆ—ฅๅฟ— + ๆŒ‡ๆ ‡ + ้”™่ฏฏ่ฏญไน‰ใ€‚ +2. **ไธป่ฆ็ผบๅฃ้›†ไธญๅœจ"ๅ…ฅๅฃไธŽ่ง‚ๆต‹้ข"๏ผŒ่€Œไธๆ˜ฏไฟฎๅค็ฎ—ๆณ•ๆœฌ่บซ**๏ผšMRF/ECDecode/Metadata ไธ‰็ฑปไปปๅŠกๆ‰ง่กŒไฝ“ๅทฒๅฎž็Žฐไฝ†ๆ— ็”Ÿไบง่งฆๅ‘ๅ…ฅๅฃ๏ผˆ`HealEvent` ๅฎŒๅ…จๆœชๆŽฅ็บฟ๏ผ‰๏ผ›`CheckAbandonedParts` ๅœจ ecstore ไธ‰ๅฑ‚ๅ…จ้ƒจ `NotImplemented`๏ผ›heal/scanner trace ้€š้“็ผบๅคฑ๏ผ›scanner ่ถ…้™ S3 ไบ‹ไปถ็ผบๅคฑ๏ผ›madmin ๅฎขๆˆท็ซฏๆ–นๆณ•็ผบๅคฑ๏ผˆๅชๆœ‰ wire ็ฑปๅž‹๏ผ‰๏ผ›heal ๅญ—่Š‚็บง่ฟ›ๅบฆ/ETA ๆœชๅฎž็Žฐใ€‚ +3. **ไธŽ v1 ่ฎค็Ÿฅ็š„้‡่ฆไฟฎๆญฃ**๏ผšbloom filter ๅœจ MinIO ๅฝ“ๅ‰ master **ๅทฒๅˆ ้™ค**๏ผˆ`.bloomcycle.bin` ๅชๅญ˜ cycle ่ฎกๆ•ฐ๏ผ‰๏ผŒRustFS ็Žฐ็ŠถไธŽ MinIO ไธ€่‡ด๏ผ›MinIO scanner ๅŒๆ ทๆ˜ฏ**้›†็พค็บง leader ๅ•ไพ‹**๏ผŒRustFS ็š„ leader.lock ๆจกๅž‹ไธŽ MinIO ๅŒๅž‹๏ผ›RustFS ็š„ ETag ๅคšๆ•ฐๆดพๅ…œๅบ•ไปฒ่ฃๅทฒๅฎž็Žฐ๏ผˆ`crates/ecstore/src/set_disk/ops/heal.rs:525-567,679`๏ผŒๅทฒไบฒ้ชŒ๏ผ‰๏ผŒv1 ๆ‹…ๅฟƒ็š„ไปฒ่ฃ็ผบๅฃไธๅญ˜ๅœจใ€‚ +4. **RustFS ๅœจๅคšๅค„่ถ…ๅ‡บ MinIO**๏ผšremote_scanner RPC ๅ่ฎฎ๏ผˆ่ฟœ็ซฏ peer ๆœฌๅœฐๆ‰ซๆ่€Œ้ž leader ่ทจ็ฝ‘่ฏป่ฟœ็›˜๏ผ‰ใ€ๆŒไน…ๅŒ– leader-epoch CAS ๅ›ดๆ ใ€ๅ‘จๆœŸ้ข„็ฎ—ไธŽ per-set/per-disk ๅนถๅ‘้—ธใ€pending-heal ่ดฆๆœฌใ€durable replacement intent + completion proof ็Šถๆ€ๆœบใ€ๅ‰ๅฐๅŽ‹ๅŠ›้—จๆŽง๏ผˆmainline throttle๏ผ‰ใ€้›†็พค heal control coordinator + envelope ้‡ๆ”พ้˜ฒๆŠคใ€‚ +5. ๅทฎ่ทๅˆ†็บง็ปŸ่ฎก๏ผšP1๏ผˆ่กŒไธบ/่ฟ็ปดๅฏน้ฝ็ผบๅฃ๏ผ‰8 ้กน๏ผŒP2๏ผˆๅฎŒๅ–„ๆ€ง๏ผ‰9 ้กน๏ผŒP3๏ผˆๆธ…็†/ไฝŽ้ฃŽ้™ฉ๏ผ‰3 ้กน๏ผŒ"ๆŒ‰่ฎพ่ฎกไธ่ฟฝๅนณ"7 ้กนใ€‚ๅฎŒๆ•ดๆธ…ๅ•่ง ยง6ใ€‚ + +--- + +## 1. ๆžถๆž„ๆ€ป่งˆ + +### 1.1 RustFS ไธ‰ๅฑ‚ๆžถๆž„ + +RustFS ๆŠŠ MinIO ๅœจ `cmd/` ๅ†…ๅ•ไฝ“็š„ heal/scanner ๆ‹†ๆˆไธ‰ๅฑ‚ + ไธคไธช็‹ฌ็ซ‹ crate๏ผš + +| ๅฑ‚ | ไฝ็ฝฎ | ่Œ่ดฃ | +|---|---|---| +| ๅŽŸ่ฏญๅฑ‚ | `crates/ecstore/src/set_disk/ops/heal.rs`๏ผˆ~3,240 ่กŒ๏ผ‰ใ€`ops/heal_walk.rs`ใ€`ops/bitrot_self_verify.rs`๏ผ›ไธŠๅฑ‚ๅฐ่ฃ… `store/heal.rs`ใ€`store/heal_walk.rs`ใ€`core/sets.rs` | ๅฏน่ฑก/ๆกถ/format/ๆ›ฟๆข็›˜ๆ ผๅผไฟฎๅคใ€disk-walk ๅนถ้›†ๆžšไธพใ€ๅ†™ๅ…ฅ่ทฏๅพ„ bitrot ่‡ชๆ ก้ชŒ๏ผ›็”ฑ `SetDisks`/`Sets`/`ECStore` ๅฎž็Žฐ `rustfs_storage_api::HealOperations` ๅฅ‘็บฆ๏ผˆ`crates/storage-api/src/object.rs:503-519`๏ผ‰ | +| heal ่ฟ่กŒๆ—ถ | `crates/heal` | ่ฟ›็จ‹็บง HealManager๏ผˆไผ˜ๅ…ˆ็บง้˜Ÿๅˆ—/่ฐƒๅบฆๅ™จ/auto disk scanner/ๆ–ญ็‚น็ปญไผ  resume๏ผ‰ใ€HealChannelProcessor๏ผˆๆถˆ่ดนๅ…จๅฑ€ heal channel๏ผ‰ใ€ๆข็›˜ๆ›ฟๆขๆขๅค็Šถๆ€ๆœบ | +| scanner ่ฟ่กŒๆ—ถ | `crates/scanner` | ๆ•ฐๆฎไฝฟ็”จๆ‰ซๆใ€ILM ่ฏ„ไผฐไธŽๅ…ฅ้˜Ÿใ€heal ๅ€™้€‰็”Ÿไบงใ€ๅคๅˆถ็”จ้‡็ปŸ่ฎกใ€remote scanner RPC | +| ๅ…ฑไบซๅ่ฎฎ | `crates/common/src/heal_channel.rs`๏ผˆ~776 ่กŒ๏ผ‰ | Start/Query/Cancel ๅ‘ฝไปค้€š้“ใ€`HealOpts`/`HealScanMode`/`HealRequestSource`/`HealAdmission*` ๅ…ฑไบซ็ฑปๅž‹ใ€`HealResultItem`๏ผˆmadmin๏ผ‰ | +| ๅ…ฑไบซๆ•ฐๆฎ | `crates/data-usage` | `DataUsageEntry/Info`ใ€็›ดๆ–นๅ›พใ€`hash_path`๏ผ›scanner ไบง็”Ÿใ€ecstore/admin ๆถˆ่ดน | + +ๅฏๅŠจ้“พ่ทฏ๏ผˆๅทฒไบฒ้ชŒ wiring๏ผ‰๏ผš + +1. `rustfs/src/startup_services.rs:93` โ†’ `init_background_service_runtime(store)`ใ€‚ +2. `rustfs/src/startup_background.rs:41-81`๏ผšๅˆ›ๅปบๅ…จๅฑ€ heal ๆœๅŠกๅ–ๆถˆไปค็‰Œ๏ผ›่ฏป `RUSTFS_SCANNER_ENABLED`๏ผˆๅˆซๅ `RUSTFS_ENABLE_SCANNER`๏ผŒ้ป˜่ฎค true๏ผ‰ไธŽ `RUSTFS_HEAL_ENABLED`๏ผˆๅˆซๅ `RUSTFS_ENABLE_HEAL`๏ผŒ้ป˜่ฎค true๏ผ‰๏ผ›**ๅช่ฆ heal ๆˆ– scanner ไปปไธ€ๅผ€ๅฏๅฐฑๅˆๅง‹ๅŒ– heal manager**๏ผˆscanner ไบง็”Ÿ็š„ heal ๅ€™้€‰้œ€่ฆๆถˆ่ดน็ซฏ๏ผ›ไธค่€…้ƒฝๅ…ณๆ—ถ heal channel ไธๅˆๅง‹ๅŒ–๏ผŒ`send_heal_request` ๆŠฅ "Heal channel not initialized"๏ผ‰ใ€‚ +3. `crates/heal/src/lib.rs:142-216`๏ผšowned task ๅ†…ๅŽŸๅญๅˆๅง‹ๅŒ–๏ผˆcaller ๅ–ๆถˆไธไผš้—็•™ๅŠๅˆๅง‹ๅŒ– manager๏ผŒ`lib.rs:123-131`๏ผ›`GLOBAL_HEAL_RUNTIME_INIT` ไบ’ๆ–ฅๅ•้ฃž๏ผ‰โ†’ `HealManager::start()` โ†’ `rustfs_common::heal_channel::init_heal_channels()` โ†’ spawn `HealChannelProcessor::start_with_receipts`ใ€‚ +4. `crates/heal/src/heal/manager.rs:1301-1356` `HealManager::start`๏ผš`start_scheduler()`๏ผˆ`manager.rs:2394-2461`๏ผŒinterval ้ป˜่ฎค 10s + `Notify` ไบ‹ไปถ้ฉฑๅŠจๅ”ค้†’๏ผ‰โ†’ `process_unclean_shutdown()`๏ผˆ`manager.rs:1362-1695`๏ผ‰โ†’ `enable_auto_heal`๏ผˆ้ป˜่ฎค true๏ผ‰ๆ—ถ `start_auto_disk_scanner()`๏ผˆ`manager.rs:2464-2999`๏ผ‰ใ€‚ +5. server ready ๅŽ `rustfs/src/startup_lifecycle.rs:150-152`๏ผš`enable_scanner` ๆ—ถ `init_data_scanner(token, store)`๏ผˆ`crates/scanner/src/scanner.rs:1293-1372`๏ผ‰ใ€‚ +6. ไผ˜้›…ๅœๆœบ๏ผš`rustfs/src/startup_shutdown.rs:308` `shutdown_ahm_services()`๏ผˆๅ–ๆถˆไปค็‰Œ๏ผ‰๏ผ›`:414` `clear_unclean_shutdown_markers()`ใ€‚ + +### 1.2 MinIO ๅฏนๅบ”็ป“ๆž„๏ผˆmaster ๆœ€็ปˆๆ€๏ผ‰ + +| MinIO ๆ–‡ไปถ | ่Œ่ดฃ | +|---|---| +| `cmd/admin-heal-ops.go` | ๆ‰‹ๅŠจ admin heal ๅบๅˆ—๏ผˆhealSequenceใ€clientToken/forceStart/forceStop๏ผ‰ | +| `cmd/global-heal.go` | ๅธธ้ฉปๅŽๅฐ heal ้˜Ÿๅˆ—๏ผˆnewBgHealSequence๏ผŒtoken ๅ›บๅฎš `0000-โ€ฆ`๏ผŒๆฐธไธ็ป“ๆŸ๏ผ‰+ `healErasureSet`๏ผˆ้€ set ๅ…จ้‡ๅฏน่ฑก heal๏ผ‰ | +| `cmd/background-heal-ops.go` | healRoutine worker ๆฑ ๏ผˆ`_MINIO_HEAL_WORKERS`๏ผŒ้ป˜่ฎค GOMAXPROCS/2๏ผ‰ๆถˆ่ดน healTask | +| `cmd/mrf.go` | MRF๏ผˆMost Recent Fail๏ผ‰้˜Ÿๅˆ—๏ผˆๅฎน้‡ 100,000๏ผ‰๏ผŒ่ฟ›็จ‹้€€ๅ‡บๆ—ถๆŒไน…ๅŒ– `.minio.sys/buckets/.heal/mrf/list.bin` ๅนถๅฏๅŠจๅ›žๆ”พ | +| `cmd/background-newdisks-heal-ops.go` | ๆ–ฐ็›˜/ๆข็›˜่‡ชๅŠจ resync๏ผˆmonitorLocalDisksAndHeal 10s ่ฝฎ่ฏข + healFreshDisk + healingTracker๏ผ‰ | +| `cmd/erasure-healing.go` / `erasure-healing-common.go` | ๅฏน่ฑก็บง heal ๆ ธๅฟƒ๏ผˆ~800 ่กŒ๏ผ‰ใ€listAndHeal | +| `cmd/data-scanner.go` | scanner ๅพช็Žฏ๏ผˆglobalLeaderLock ้›†็พคๅ•ไพ‹๏ผ‰+ folderScanner + applyActions | +| `cmd/erasure.go`๏ผˆnsScanner๏ผ‰/ `erasure-server-pool.go` | NSScanner ไธ‰ๅฑ‚็ป“ๆž„ | +| `cmd/bucket-lifecycle.go` | ILM ๆ‰ง่กŒๅ™จ๏ผˆexpiry/transition worker ๆฑ ๏ผ‰ | +| `cmd/xl-storage.go` | DiskInfo.Healingใ€CheckParts/VerifyFileใ€CleanAbandonedDataใ€RenameData healing ๅˆ†ๆ”ฏ | +| `cmd/prepare-storage.go` | waitForFormatErasure ๆ–ฐ็›˜ๅฏๅŠจๆกๆ‰‹ | + +### 1.3 ๆžถๆž„็บงๅทฎๅผ‚๏ผˆ่ฎพ่ฎกๅ–่ˆ๏ผŒ้ž็ผบ้™ท๏ผ‰ + +1. **heal ้˜Ÿๅˆ—ๆจกๅž‹**๏ผšMinIO ๆ‰€ๆœ‰ heal๏ผˆscanner ๆŠฝๆ ท/MRF/admin/ๆ–ฐ็›˜ resync๏ผ‰ๆฑ‡ๅ…ฅๅ• channel + ๅ›บๅฎš worker ๆฑ ๏ผˆๆ–ฐ็›˜ resync ๅฆๆœ‰ per-drive worker ๆฑ ๏ผ‰๏ผ›RustFS ๆ˜ฏไผ˜ๅ…ˆ็บงๅ † + ๅŽป้‡ๅˆๅนถ + ๅฎน้‡ๅˆ†็บงไธขๅผƒ + per-set bulkhead + ๅ‰ๅฐๅŽ‹ๅŠ›้—จๆŽง็š„ๅคš็ญ–็•ฅ่ฐƒๅบฆๅ™จ๏ผˆ`manager.rs:3003-3420`๏ผ‰ใ€‚RustFS ่กจ่พพๅŠ›ๆ›ดๅผบ๏ผŒไปฃไปทๆ˜ฏ"้‡ๅค่ฏทๆฑ‚่ขซๅˆๅนถ"็š„ๅฏ่ง‚ๆต‹ๆ€ง้—ฎ้ข˜๏ผˆv1 ๅทฒๆŒ‡ๅ‡บ๏ผŒ็Žฐๆœ‰ `HealAdmissionReceipt` canonical task_id + alias ๆœบๅˆถๅ›žๅบ”ไบ†ๅฎƒ๏ผŒ`manager.rs:1759-1846`๏ผ‰ใ€‚ +2. **scanner ่ฟœ็ซฏ็›˜่ฎฟ้—ฎ**๏ผšMinIO leader ้€š่ฟ‡็ฃ็›˜ๆŠฝ่ฑกๅฑ‚้€ๆ˜Ž่ฏปๅ†™่ฟœ็ซฏ่Š‚็‚น็ฃ็›˜๏ผ›RustFS leader ้€š่ฟ‡ remote_scanner RPC ๆŠŠๆ‰ซๆๆ‰ง่กŒไธ‹ๆ”พๅˆฐ่ฟœ็ซฏ peer ๆœฌๅœฐ่ฟ›่กŒ๏ผˆ`crates/scanner/src/remote_scanner.rs`๏ผ‰๏ผŒๅชๅ›žไผ ็ป“ๆžœไธŽ่ฟ›ๅบฆๅฟƒ่ทณใ€‚ไธค่€…้ƒฝๆ˜ฏ้›†็พคๅ• leaderใ€‚RustFS ๆ–นๆกˆ็œ leaderโ†”่ฟœ็ซฏ็š„ๅ…ƒๆ•ฐๆฎ่ฏปๆ”พๅคง๏ผŒไปฃไปทๆ˜ฏ้œ€่ฆ็ปดๆŠค็‹ฌ็ซ‹ RPC ๅ่ฎฎ๏ผˆHMAC ้€ๅธง่ฎค่ฏใ€ไผš่ฏ้‡ๆ”พ็ผ“ๅญ˜ใ€fence ๅค้ชŒ๏ผŒ`remote_scanner.rs:52-61,405-496,1024-1065`๏ผ‰ใ€‚ +3. **heal ็Šถๆ€ๆŒไน…ๅŒ–**๏ผšMinIO ็”จๅ•ๆ–‡ไปถ `.healing.bin`๏ผˆmsgp healingTracker๏ผŒdiskID ไธๅŒน้…ๅณ้‡็ฝฎ๏ผ‰๏ผ›RustFS ็”จ schema ๅŒ–ๅคšๆ–‡ไปถ๏ผˆresume/checkpoint/intent/seal/proof ๅ„่‡ช CAS ๅ‘ๅธƒ๏ผŒ`resume.rs:38-61`๏ผ‰๏ผŒๅดฉๆบƒ็ช—ๅฃๆ˜พๅผ่กฅ้ฝ๏ผˆ`erasure_healer.rs:389-402`ใ€`resume.rs:1027-1057`๏ผ‰ใ€‚ +4. **ๅ†™่ทฏๅพ„่‡ชไฟๆŠค**๏ผšMinIO ๅ†™ๅ…ฅๅŽ้ ๅŽๅฐ heal ๆ”ถๆ•›๏ผ›RustFS ๅœจ PutObject/CompleteMultipartUpload ๆไบค rename ๅŽไธปๅŠจๆฃ€ๆŸฅ `convergence.needs_heal()` ๅนถ็ซ‹ๅณๅ…ฅ้˜Ÿๅฏน่ฑก heal๏ผˆ`set_disk/ops/object.rs:2291-2306`ใ€`ops/multipart.rs:2574-2589`๏ผ‰๏ผŒๅฆๆœ‰่ฏปไฟฎๅค read repair๏ผˆ`io_primitives.rs:1040-1160`๏ผ‰ใ€‚ + +--- + +## 2. Heal ๅทฒๅฎž็ŽฐๅŠŸ่ƒฝๅ…จๆ™ฏ + +### 2.1 ไปปๅŠก็ฑปๅž‹๏ผˆ`HealType`๏ผŒ`crates/heal/src/heal/task.rs:85-111`๏ผ‰ + +| ็ฑปๅž‹ | ่ฏญไน‰ | ๆ‰ง่กŒไฝ“ | ็”Ÿไบง่งฆๅ‘ๆ–น | +|---|---|---|---| +| `Cluster` | ๆ‰€ๆœ‰ bucket ไพๆฌก heal๏ผˆ็ป“ๆž„ + ๅฏ้€‰้€’ๅฝ’ๅฏน่ฑก๏ผ‰๏ผŒๆ‰นๅ†…้‡่ฏ• โ‰ค3 | `heal_cluster` task.rs:1420-1490 | channel๏ผšbucket ไธบ็ฉบๅณ Cluster๏ผˆchannel.rs:576-577๏ผ‰ | +| `Object{bucket,object,version_id}` | ๅ•ๅฏน่ฑก/็‰ˆๆœฌ๏ผ›ไธๅญ˜ๅœจๆ—ถๆŒ‰ `recreate_missing` ้‡ๅปบๆˆ–ๆŠฅ้”™ | `heal_object` task.rs:855-1146 | adminใ€scannerใ€read-repairใ€ๅ†™่ทฏๅพ„ๆ”ถๆ•›ใ€add_partial | +| `Bucket{bucket}` | ๆกถๅ…ƒๆ•ฐๆฎ/็ป“ๆž„๏ผ›`recursive` ๅ†้ๅŽ†ๅ…จ้ƒจๅฏน่ฑก็‰ˆๆœฌ | `heal_bucket` task.rs:1284-1418 + `heal_bucket_objects` task.rs:1508-1698 | admin๏ผˆPOST /v3/heal/{bucket}๏ผ‰ใ€scanner `build_bucket_heal_request` | +| `Prefix{bucket,prefix}` | ๆŒ‰ๅ‰็ผ€้€’ๅฝ’ | `heal_prefix` task.rs:1492-1506 | channel๏ผš`recursive && prefix` ้ž็ฉบ๏ผˆchannel.rs:578-585๏ผ‰ | +| `ErasureSet{buckets,set_disk_id}` | format ไฟฎๅค + healing ๆ ‡่ฎฐ + ้€ๆกถ้ข„ๅค„็† + ๅฏๆขๅค้€็‰ˆๆœฌๆทฑๆ‰ซ | `heal_erasure_set` task.rs:2158-2642 | admin๏ผˆpool/set ๅ‚ๆ•ฐ๏ผ‰ใ€auto disk scannerใ€unclean shutdownใ€renew_diskใ€durable replacement ๆขๅค | +| `Metadata{bucket,object}` | ไป…ๅ…ƒๆ•ฐๆฎ๏ผˆDeepใ€ไธ้‡ๅปบๆ•ฐๆฎ๏ผ‰ | `heal_metadata` task.rs:1700-1859 | **ๆ— ็”Ÿไบง่งฆๅ‘ๆ–น**๏ผˆยง6 HS-01๏ผ‰ | +| `MRF{meta_path}` | ๅคฑ่ดฅ่ทฏๅพ„้ฉฑๅŠจ็š„ Deep ไฟฎๅค๏ผˆrecursive+update_parity๏ผ‰ | `heal_mrf` task.rs:1861-1992 | **ๆ— ็”Ÿไบง่งฆๅ‘ๆ–น**๏ผˆไป… `HealEvent` ๅฏ็”Ÿๆˆ๏ผŒๆœชๆŽฅ็บฟ๏ผ‰ | +| `ECDecode{bucket,object,version_id}` | EC ่งฃ็ ้‡ๅปบ๏ผˆDeep+recreate+update_parity๏ผ‰๏ผŒUrgent ไผ˜ๅ…ˆ็บง | `heal_ec_decode` task.rs:1994-2156 | **ๆ— ็”Ÿไบง่งฆๅ‘ๆ–น**๏ผˆไป… `HealEvent` ๅฏ็”Ÿๆˆ๏ผŒๆœชๆŽฅ็บฟ๏ผ‰ | + +ไผ˜ๅ…ˆ็บง `Low/Normal/High/Urgent`๏ผˆtask.rs:168-179๏ผ‰๏ผ›็Šถๆ€ๆœบ `Pending/Running/Retrying/Completed/Failed/Cancelled/Timeout`๏ผˆtask.rs:225-241๏ผ‰ใ€‚ + +### 2.2 ่งฆๅ‘่ทฏๅพ„ๅ…จๆ™ฏ๏ผˆadmin ไน‹ๅค–๏ผ‰ + +| ้€š้“ | source | ไผ˜ๅ…ˆ็บง | ่ฏๆฎ | +|---|---|---|---| +| Scanner ๅ‘จๆœŸๆŠฝๆ ท๏ผˆ1/1024๏ผŒ`RUSTFS_HEAL_OBJECT_SELECT_PROB`๏ผ‰ | Scanner | Low | `scanner_folder.rs:2117-2136`ใ€`:1150`๏ผ›`remove_corrupted=HEAL_DELETE_DANGLING(true)`ใ€`recreate_missing=false`๏ผˆ`common/heal_channel.rs:24`ใ€`scanner_folder.rs:510-511`๏ผ‰ | +| Scanner ๅ…ƒๆ•ฐๆฎๆŸๅ๏ผˆget_size ๅคฑ่ดฅๅˆ†็ฑป HealMetadata๏ผ‰ | Scanner | High | `scanner_folder.rs:2147-2208`ใ€`:1244-1260` | +| Scanner abandoned children๏ผˆ็ผ“ๅญ˜ๆœ‰ใ€็›˜ไธŠๆ— ๏ผŒlist_path_raw quorum ๆ ธๆŸฅ๏ผ‰ | Scanner | High๏ผˆๆกถ็บง+ๅฏน่ฑก็บง๏ผ‰ | `scanner_folder.rs:2528-2792` | +| Scanner pending-heal ่ดฆๆœฌ้‡่ฏ•๏ผˆheal ้€š้“ๆปก่ขซๆ‹’ๅŽๆŒไน…ๅŒ–๏ผŒๆฏๆกถๆฏ่ฝฎ โ‰ค128 ๆกใ€ไธŠ้™ 10k๏ผ‰ | Scanner | ๅŽŸไผ˜ๅ…ˆ็บง | `scanner_folder.rs:1721-1763`ใ€`:99-100` | +| auto disk scanner๏ผˆunformatted ็›˜็ป replacement_readiness ็กฎ่ฎค / `runtime_state=="returning"` ็›˜ / durable intent ้‡ๅ…ฅ๏ผ‰ | AutoHeal | Low | `manager.rs:2464-2999` | +| unclean shutdown ๆขๅค๏ผˆๅฏๅŠจ่ฏป `unclean-shutdown` ๆ ‡่ฎฐ โ†’ ๅ…จ้ƒจๆœฌๅœฐ set ErasureSet heal๏ผ‰ | AutoHeal | Low | `manager.rs:1362-1695` | +| ๅ†™่ทฏๅพ„ๆ”ถๆ•›๏ผˆPutObject/CompleteMultipartUpload ๅŽ `convergence.needs_heal()`๏ผ‰ | Internal | Normal | `set_disk/ops/object.rs:2291-2306`ใ€`ops/multipart.rs:2574-2589` | +| ้ƒจๅˆ†ๅฏน่ฑก heal๏ผˆadd_partial๏ผ‰ | Internal | Normal | `set_disk/ops/object.rs:5808-5825` | +| ๆ—งๆ•ฐๆฎ็›ฎๅฝ•ๆธ…็†ๆฎ‹็•™ enqueue | Internal | Normal | `set_disk/core/io_primitives.rs:3880-3907` | +| ่ฏปไฟฎๅค๏ผˆmetadata_read_error / missing_shards / decode_error๏ผŒTTL ๅŽป้‡็ผ“ๅญ˜๏ผ‰ | ReadRepair | Low | `set_disk/read.rs:407,995,1079` โ†’ `submit_read_repair_heal`๏ผˆ`io_primitives.rs:1105-1160`๏ผ‰๏ผŒ`recreate_missing=true` | +| ็›˜้‡่ฟž้‡ UnformattedDisk โ†’ send_heal_disk | AutoHeal | Normal | `set_disk/ops/locking.rs:339-347` | +| Admin API๏ผˆๅซ้›†็พค coordinator ่ทฏ็”ฑ๏ผ‰ | Admin | High | `rustfs/src/admin/handlers/heal.rs:174-212`ใ€`:771-930` | +| ้›†็พค RPC heal๏ผˆpeer ่ฐƒ็”จ๏ผ‰ | โ€” | โ€” | `rustfs/src/storage/rpc/node_service/heal.rs`ใ€`ecstore/src/cluster/rpc/peer_s3_client.rs:296,1209` | + +ๆณจๆ„๏ผšMinIO ็š„ MRF ้€š้“๏ผˆ่ฏป่ทฏๅพ„ๆฃ€ๅ‡บ part ็ผบๅคฑ/ๆŸๅๅณๆ—ถๆŠ•้€’ + ้˜Ÿๅˆ—ๆŒไน…ๅŒ– + shutdown ๅ›žๆ”พ๏ผŒ`cmd/mrf.go`ใ€`erasure-object.go:395-410,800-812`๏ผ‰ๅœจ RustFS ็”ฑ read-repair + ๅ†™่ทฏๅพ„ๆ”ถๆ•›**้ƒจๅˆ†ๆ›ฟไปฃ**๏ผ›`HealType::MRF`/`ECDecode`/`Metadata` ไธ‰ไธชๆ‰ง่กŒไฝ“ๆฒกๆœ‰็”Ÿไบงๅ…ฅๅฃ๏ผˆ่ฏฆ่ง ยง6 HS-01๏ผ‰ใ€‚ + +### 2.3 ๅฏน่ฑก็บง heal ่ฏญไน‰๏ผˆecstore `set_disk/ops/heal.rs`๏ผ‰ + +ๆต็จ‹๏ผˆ`heal_object_with_explicit_version_regen` :426 ่ตท๏ผ‰๏ผš + +1. ๅ–ๅฏน่ฑกๅ†™้”๏ผˆ้™ค้ž `no_lock`๏ผ‰๏ผ›`object` ไปฅ `/` ็ป“ๅฐพ่ตฐๅฏน่ฑก็›ฎๅฝ• heal๏ผˆ`heal_object_dir_locked` :1587-1717๏ผšdangling ๅˆคๅฎš + `remove` ๅˆ ้™ค + ็ผบ volume ้‡ๅปบ๏ผ‰ใ€‚ +2. `read_all_fileinfo` ๅ…จ็›˜่ฏป xl.meta๏ผŒๅ…จ้ƒจ not-found ่ง†ไธบๅทฒๅˆ ้™ค่ฟ”ๅ›žใ€‚ +3. **quorum ไปฒ่ฃ + ETag ๅ…œๅบ•**๏ผˆๅทฒไบฒ้ชŒ๏ผ‰๏ผš`list_online_disks` ไปฅ mod-time quorum ไธบๅ‡†๏ผ›quorum ๅคฑๆ•ˆๆ—ถๅ›ž้€€ ETag ๅคšๆ•ฐๆดพไปฒ่ฃ๏ผˆ`:525-567` `filter_by_etag`/`quorum_etag`๏ผ‰๏ผ›`pick_valid_fileinfo` ้€‰ canonical ๅ…ƒๆ•ฐๆฎ๏ผ›"meta ๅ็›˜ๆ•ฐ > parity" ็š„ cannotHeal ๅˆคๅฎšๅœจ ETag ๅ…จ็›˜ไธ€่‡ดๆ—ถ่ฑๅ…๏ผˆ`:679`๏ผ‰ใ€‚ไธŽ MinIO `filterDisksByETag` ๅŒไปฒ่ฃไธ€่‡ดใ€‚ +4. `disks_with_all_parts`๏ผˆ:562-572๏ผ‰ๆŒ‰ `scan_mode` ๆ ก้ชŒ part๏ผš**Normal ไป… stat๏ผˆCheckParts ่ฏญไน‰๏ผ‰๏ผŒDeep ๅšๅ…จ้‡ bitrot ๆ ก้ชŒ๏ผˆVerifyFile ่ฏญไน‰๏ผ‰**๏ผ›Normal ๆ‰ซๆๆฃ€ๅ‡บ `FileCorrupt` ่‡ชๅŠจๅ‡็บง Deep ้‡่ฏ•ไธ€ๆฌก๏ผˆ`:2022-2031`๏ผŒไธŽ MinIO erasure-healing.go:1101-1106 ๅŒๅž‹๏ผ‰๏ผ›ๆ—  parity ๅฏน่ฑก๏ผˆEC:0๏ผ‰bitrot ๅคฑ่ดฅๅˆคไธๅฏๆขๅค๏ผˆ`:700-726`๏ผ‰ใ€‚ +5. `should_heal_object_on_disk`๏ผˆ:606-650๏ผ‰้€็›˜ๅˆ†็ฑป missing/corrupt/offline/outdated โ†’ ้‡ๅปบ๏ผšper-part bitrot reader/writer๏ผˆ็”จ per-part checksum + ็ฎ—ๆณ•๏ผ‰ใ€ๅ†™ไธดๆ—ถๅทๅŽ rename ๆไบค๏ผˆ`HEAL_RENAME_INCOMPLETE` ้‡่ฏ•่ฏญไน‰ :24๏ผ‰๏ผ›dangling ๅˆ ้™คๅฎ‰ๅ…จๆฃ€ๆŸฅ `dangling_delete_safety`๏ผˆ:1488๏ผ‰๏ผ›**ๅญคๅ„ฟๆ•ฐๆฎ็›ฎๅฝ•ๅ›žๆ”ถ `reclaim_orphan_data_dirs_best_effort`๏ผˆ:1428๏ผ‰**โ€”โ€”่ฟ™้ƒจๅˆ†่ฆ†็›–ไบ† MinIO `CleanAbandonedData` ็š„ไธปๅœบๆ™ฏ๏ผˆไฝ†ๆ— ็‹ฌ็ซ‹ `CheckAbandonedParts` API๏ผŒ่ง ยง6 HS-02๏ผ‰ใ€‚ +6. ็‰ˆๆœฌๅŒ–ๅฏน่ฑก๏ผšๆžšไธพ"ๆฏไธช็‰ˆๆœฌ"๏ผˆ`storage.rs:1494-1530`๏ผ‰๏ผ›delete-marker ่ทฏๅพ„็”ฑ `latest_meta.deleted` ๅ†ณๅฎš๏ผˆ`storage.rs:262-277` ๆณจ้‡Š๏ผ‰๏ผ›ๅ›žๅฝ’ๆต‹่ฏ• `tests/heal_b5_versioned_regression_test.rs:282,334`ใ€‚ +7. ๆ˜พๅผ็‰ˆๆœฌ้‡ๅปบ `try_regenerate_explicit_version_meta`๏ผˆ:1318๏ผ‰๏ผ›transitioned ๅฏน่ฑกๆœฌๅœฐๆฎ‹็•™ๆธ…็†ใ€‚ +8. ๅ†™ๅ…ฅ่ทฏๅพ„ๅฆๆœ‰ shard ็บง bitrot ่‡ชๆ ก้ชŒ `verify_written_bitrot_shards`๏ผˆ`ops/bitrot_self_verify.rs:45-129`๏ผŒHighwayHash256S๏ผŒๆœ€็ปˆ rename ๅ‰ๆ ก้ชŒๅˆšๅ†™ๅ‡บ็š„ shard๏ผŒๆœๅŠก EC:0 ๆ—  parity ๅœบๆ™ฏ๏ผ‰โ€”โ€”**ๆณจๆ„่ฟ™ไธๆ˜ฏๅŽๅฐ bitrot ๅทกๆฃ€**๏ผ›ๅŽๅฐๅทกๆฃ€็”ฑ scanner bitrot_cycle ้ฉฑๅŠจ Deep heal ๆ‰ฟๆ‹…ใ€‚ + +heal crate ไพงๅŒ…่ฃ…๏ผˆ`task.rs:855-1146`๏ผ‰๏ผšๅญ˜ๅœจๆ€งๆฃ€ๆŸฅ๏ผˆ็žฌๆ—ถ้”™่ฏฏ่ฝฌ `TransientSkip` ไธ่ฏฏๅˆคๅคฑ่ดฅ :551-569๏ผ‰๏ผ›scanner ๅˆๆˆ็›ฎๅฝ•่ง„่ŒƒๅŒ–๏ผˆ:1148-1180๏ผ‰๏ผ›`recreate_missing` ้‡ๅปบ๏ผˆ:1183-1282๏ผ‰๏ผ›data-usage-cache ๅฏน่ฑก้”่ถ…ๆ—ถ่ฑๅ…๏ผˆ:571-653๏ผ‰๏ผ›not-found โ†’ treated_as_deleted ๆˆๅŠŸ๏ผˆ:1012-1029๏ผ‰๏ผ›็ป“ๆžœ `HealResultItem` ไฟ็•™่‡ณๅคš 1024 ๆก + truncated ๆ ‡ๅฟ—๏ผˆ:50,845-852๏ผ‰ใ€‚ + +้€’ๅฝ’้ๅŽ†๏ผˆ`heal_bucket_objects` task.rs:1508-1698๏ผ‰๏ผšๅˆ†้กตๆžšไธพๅ…จ้ƒจ็‰ˆๆœฌๅซ delete markerใ€็žฌๆ—ถ้”™่ฏฏๆŒ‡ๆ•ฐ้€€้ฟ้‡่ฏ• โ‰ค3๏ผˆ2^n + ๆŠ–ๅŠจ :620-627๏ผ‰ใ€ๅคฑ่ดฅๆ ทๆœฌๆ—ฅๅฟ—ๆˆชๆ–ญ โ‰ค5 ๆกใ€่šๅˆ `BatchHealFailure`ใ€‚ + +### 2.4 erasure set heal ไธŽๆ–ญ็‚น็ปญๆ‰ซ + +`heal_erasure_set`๏ผˆtask.rs:2158-2642๏ผ‰ๅ››้˜ถๆฎต๏ผˆ4 ๆญฅ่ฟ›ๅบฆ่ทŸ่ธช๏ผ‰๏ผš + +1. **ๆ›ฟๆขๆ„ๅ›พไธŽๆขๅค็›˜้€‰ๆ‹ฉ**๏ผˆไป… AutoHeal + heal_endpoints ้ž็ฉบ๏ผ‰๏ผšๅค็”จ durable intent ๆ‰€ๅœจ็›˜ / ๆŽ’้™ค็›ฎๆ ‡็ซฏ็‚น้€‰ๅนธๅญ˜็›˜๏ผ›ๅทฒๅฎŒๆˆไปฃ๏ผˆCleanupPending๏ผ‰ๅน‚็ญ‰ๆ”ถๅฐพใ€‚ +2. **ๆ ผๅผไฟฎๅค**๏ผš`heal_replacement_format(dry_run, pool, set, targets)`๏ผˆ`storage.rs:1372-1384`๏ผŒtrait ้ป˜่ฎคๅฎž็Žฐ fail-closed๏ผ‰๏ผ›้€็›ฎๆ ‡็›˜็ป“ๆžœๅฟ…้กปๅ…จ ok๏ผˆ`erasure_healer.rs:97-102`๏ผ‰+ ่บซไปฝๅ›ดๆ ๅคๆ ธ๏ผˆtask.rs:2410-2420๏ผ‰ใ€‚ +3. **healing ๆ ‡่ฎฐ**๏ผšๅฏน็›ฎๆ ‡็›˜ๅ†™ owner CAS ๆ ‡่ฎฐ `{set_disk_id}:{task_id}`๏ผˆ`mod.rs:80-229`๏ผŒCAS + ๅ›žๆปš + ๅนถๅ‘ๅ”ฏไธ€ owner๏ผ‰๏ผŒไฝฟ `DiskInfo.healing` ไธบ็œŸ๏ผˆๅทฒไบฒ้ชŒ่ต‹ๅ€ผ้“พ `set_disk/mod.rs:4988`๏ผ‰ใ€‚ +4. **้€ๆกถ้ข„ๅค„็† + ๅฏๆขๅคๆทฑๆ‰ซ**๏ผš`ErasureSetHealer::heal_erasure_set`๏ผˆ`erasure_healer.rs:242-278`๏ผ‰ใ€‚ + +`ErasureSetHealer` ๆ‰ซๆ็ป†่Š‚๏ผˆๅฏนๆ ‡ MinIO `healErasureSet`๏ผŒ`heal_walk.rs:15-23` ๆจกๅ—ๆณจ้‡Šๆ˜Ž็กฎๅผ•็”จ MinIO `global-heal.go` ็š„ listPathRaw + objQuorum=1 + mergeXLV2Versions๏ผ‰๏ผš + +- **ๆžšไธพๅ™จ้€‰ๆ‹ฉ๏ผˆbacklog#920๏ผ‰**๏ผšDeep ๆˆ– AutoHeal โ†’ per-set **disk-walk ๅนถ้›†ๆžšไธพ** `list_versions_for_heal_page_disk_walk`๏ผˆ"ไปปๆ„็›˜ไธŠๅญ˜ๅœจ"ๅณ sub-quorum ๅฏ้‡ๅปบ๏ผ›`storage.rs:1559-1644`๏ผŒ้กต็•Œ 1000 ๅฏน่ฑก/10,000 ็‰ˆๆœฌ๏ผŒ`dw1:` cursor๏ผ‰๏ผ›ๆ™ฎ้€š่ฏทๆฑ‚่ตฐ read-quorum `list_object_versions`ใ€‚ +- **็ปญๆ‰ซๆธธๆ ‡**๏ผšๆƒๅจ cursor ไธบ opaque continuation token๏ผˆ`v1:`=marker JSONใ€`dw1:`=disk-walk key๏ผŒไธคๅ‘ฝๅ็ฉบ้—ดไบ’ๆ–ฅ้˜ฒ่ฏฏ่ฏป๏ผŒ`storage.rs:81-260`๏ผ‰๏ผ›ๆฏๅฎŒๆˆไธ€้กตๅ…ˆๆŒไน…ๅŒ– cursor ๅ†ๆธ… dedup ้›†ๅˆ๏ผˆ`erasure_healer.rs:922-927`๏ผ‰ใ€‚ +- **้กตๅ†…ๅนถๅ‘**๏ผšFuturesUnordered + Semaphore๏ผŒ้ป˜่ฎค `RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY=8`๏ผŒDeep/AutoHeal ๅผบๅˆถ 1๏ผˆ`erasure_healer.rs:105-142`๏ผ‰ใ€‚ +- **per-version dedup**๏ผš`compose_key` ้•ฟๅบฆๅ‰็ผ€ๆณจๅ…ฅ็ผ–็ ๏ผˆ`resume.rs:281-288`๏ผ‰ใ€‚ +- **้”™่ฏฏๅˆ†็ฑป**๏ผš็œŸ็ผบๅธญ๏ผˆFileNotFound ็ญ‰๏ผ‰โ†’ Absent๏ผˆ่ฎกๆˆๅŠŸ๏ผ‰๏ผ›ๅŸบ็ก€่ฎพๆ–ฝ็žฌๆ—ถ๏ผˆquorum/DiskNotFound/SlowDown ็ญ‰๏ผ‰โ†’ Transient๏ผˆ่ฎก skipped๏ผ‰๏ผ›ๅ…ถไฝ™ Failed๏ผˆ`erasure_healer.rs:148-182`๏ผŒๆณจ้‡Šๅผ• backlog#856/#799 B7๏ผš็ฆป็บฟ็›˜ไธๅพ—่ฎฐ healed/absent๏ผ‰ใ€‚ +- **้˜ฒๆญปๅพช็Žฏ**๏ผš็ฉบ้กต truncated ๆˆ–้กตๅฐพ็‰ˆๆœฌ่บซไปฝไธๅ‰่ฟ›ๅณไธญๆญข๏ผˆ:933-949๏ผ‰ใ€‚ +- **ๅฎŒๆˆๅˆคๅฎš**๏ผšfailed/skipped/failed_buckets ไปปไธ€ >0 ไธๆ ‡่ฎฐๅฎŒๆˆ๏ผŒ`schedule_retry()` ๅคไฝ resume+checkpoint ไธคๅฑ‚๏ผˆ:561-626๏ผŒbacklog#855/B6/#1033๏ผšskip ่ฝฎไธๅพ—ๆ ‡่ฎฐๅฎŒๆˆ๏ผ‰ใ€‚ +- **ๆ›ฟๆข็›˜ๆไบค่ฏๆฎ**๏ผš็›ฎๆ ‡็ซฏ็‚น็‰ฉ็†ๅ›ž่ฏป `replacement_targets_have_version`๏ผˆ`ops/heal.rs:340-412`๏ผ‰๏ผŒๆœช็กฎ่ฎค โ†’ transient skipใ€‚ + +### 2.5 ๆข็›˜่‡ชๅŠจไฟฎๅค๏ผˆreplacement recovery๏ผ‰ + +- **่ฏ†ๅˆซ**๏ผˆ`replacement_readiness.rs:25-73`๏ผ‰๏ผš`replacement_mount_lease_root()` ๅญ˜ๅœจใ€canonicalize ๆˆๅŠŸใ€ๆ˜ฏๆŒ‚่ฝฝ็‚นใ€็‰ฉ็†่ฎพๅค‡ id ้ž็ฉบใ€ไธŽๆ น่ฎพๅค‡ไธ็›ธไบคใ€ไธไธŽๅ…„ๅผŸ็›˜ๅ…ฑไบซ็‰ฉ็†่ฎพๅค‡๏ผˆLinux ็”จ /proc/self/mountinfo mount-id+dev+ino๏ผ‰ใ€‚้ž root ๆŒ‚่ฝฝๆฃ€ๆŸฅๆœ‰ๅ›žๅฝ’ๆต‹่ฏ•๏ผˆ`manager.rs:3549`๏ผ‰ใ€‚ +- **็Šถๆ€ๆœบ**๏ผˆ`resume.rs:63-73`๏ผ‰๏ผš`Intent โ†’ Rebuilding โ†’๏ผˆๅ†™ proof๏ผ‰Verified โ†’ CleanupPending โ†’ ๆธ…็†`๏ผ›`Abandoned` ็ปˆๆ€๏ผ›่ทจ็Šถๆ€่ฟ็งปๅ…ˆๅ†™ๆŒไน…ๅฑ‚ๅ†ๅ˜ๆ›ด๏ผˆ`save_state_strict`๏ผ‰ใ€‚ +- **ๆŒไน…ๅŒ–**๏ผˆ`resume.rs:38-61`๏ผŒschema ResumeState=5/Checkpoint=5/proof=1๏ผ‰๏ผš`{task_id}_ahm_resume_state.json`ใ€`_ahm_checkpoint.json`ใ€`buckets/ahm-replacement/` ๅ‘ฝๅ็ฉบ้—ดไธ‹ intent/seal/completion_proof๏ผ›torn write + ๆ—  seal ๅฏ่ฏ†ๅˆซๅนถๅŽŸๅญ้‡ๅปบ๏ผˆ:1316-1338๏ผ‰๏ผ›CAS ๅ‘ๅธƒใ€ๆ‹’็ป่ฆ†็›–ๅนถๅ‘ๆœ‰ๆ•ˆ proof๏ผˆ:1512-1585๏ผ‰ใ€‚ +- **ๆขๅค**๏ผšunclean shutdown ไธŽๅ‘จๆœŸๆ‰ซๆ้ƒฝไปŽๅนธๅญ˜็›˜ๆขๅคๆœชๅฎŒๆˆ/ๅพ…ๆธ…็†ๆ›ฟๆขไปฃ๏ผˆ`manager.rs:1435-1640,2663-2815`๏ผ‰๏ผ›ๅคšไปฃๅ†ฒ็ช/ๆ ก้ชŒๅคฑ่ดฅ โ†’ ๅ†ป็ป“่ฏฅ set๏ผˆ`replacement_recovery_blocked_sets`๏ผŒ`manager.rs:69-87,2782-2815`๏ผ‰ใ€‚ +- **ๅฏนๅค–ๅฟซ็…ง**๏ผš`current_replacement_recovery_snapshot`๏ผˆ`lib.rs:262-333`๏ผ‰ๅˆๅนถๆœฌๅœฐๅนธๅญ˜็›˜่ฎฐๅฝ•๏ผŒๅ†ฒ็ช โ†’ Unknown/้ž definitive๏ผ›admin `GET /v4/heal/replacement-recovery`ใ€‚ + +### 2.6 ่ฐƒๅบฆๅ™จ๏ผˆmanager.rs๏ผ‰ + +- ไผ˜ๅ…ˆ็บงๅ † + ๅŒไผ˜ๅ…ˆ็บง FIFO๏ผˆ:148-191,330-347๏ผ‰๏ผ›dedup key ๆŒ‰็ฑปๅž‹๏ผˆ:469-506๏ผ‰๏ผ›ๅ…ฅ้˜Ÿไธ‰ๆ€ๆŸฅ้‡ activeโ†’queuedโ†’retrying๏ผˆ:1759-1785๏ผ‰๏ผ›้‡ๅค้ป˜่ฎค Merged ๅนถ่ฟ”ๅ›ž canonical task_id๏ผˆ`HealAdmissionReceipt`๏ผŒ:1821-1846๏ผ‰+ client token alias๏ผˆ:1219-1246๏ผ‰ใ€‚ +- ๅฎน้‡๏ผš้˜Ÿๅˆ—ๆปกๆ—ถ best-effort ๆฅๆบ๏ผˆScanner/AutoHeal/ReadRepair๏ผ‰ๆˆ–ไฝŽไผ˜ๅ…ˆ็บง่ขซ Dropped(QueueFull)๏ผ›Admin/Internal ๅฏ้ฉฑ้€ไฝŽไผ˜ๅ…ˆ็บงๆŽ’้˜Ÿ้กน๏ผˆ`push_displacing_lower_priority` :353-396๏ผ‰๏ผ›80%/95% ๅŽ‹ๅŠ›ๅˆ†็บง๏ผˆ:885-909๏ผ‰ใ€‚ +- ๅนถๅ‘๏ผšๅ…จๅฑ€ `max_concurrent_heals`๏ผˆ้ป˜่ฎค 4๏ผ‰+ per-set bulkhead `max_concurrent_per_set`๏ผˆ้ป˜่ฎค 1๏ผ‰๏ผˆ:3040-3073,3434-3447๏ผ‰ใ€‚ +- ๅ‰ๅฐๅŽ‹ๅŠ›้—จๆŽง mainline throttle๏ผšๅ‰ๅฐ่ฏป/ๅ†™ permit ๅˆฉ็”จ็Ž‡ โ‰ฅ80% ๆ—ถๅปถ่ฟŸ best-effort ไปปๅŠก๏ผˆ:919-1009,2999-3020๏ผ‰ใ€‚ +- ่ถ…ๆ—ถ๏ผšไปปๅŠก็บง่šๅˆ่ถ…ๆ—ถ๏ผˆ้ป˜่ฎค 300s๏ผ‰๏ผŒ่ทจ้‡่ฏ•ไฟ็•™ๅ‰ฉไฝ™้ข„็ฎ—๏ผˆtask.rs:444-451๏ผŒPR #6101๏ผ‰ใ€‚ +- ๅฏๆขๅค้‡่ฏ•๏ผš`is_recoverable_heal()`๏ผˆerror.rs:83-136๏ผ‰โ‰ค3 ๆฌกใ€2^n ้€€้ฟๅฐ้กถ 30s๏ผ›retry ๅœจ็‹ฌ็ซ‹ backoff task ไธญๆŒๆœ‰ๆ‰€ๆœ‰ๆƒ๏ผˆ:3235-3382๏ผ‰ใ€‚ +- ๅฎŒๆˆๆ€ไฟ็•™ 10 ๅˆ†้’Ÿไพ›ๆŸฅ่ฏข๏ผˆ:42๏ผ‰ใ€‚ + +### 2.7 Admin API ไธŽ้›†็พคๅ่ฐƒ + +- ่ทฏ็”ฑ๏ผˆ`rustfs/src/admin/handlers/heal.rs:174-212`๏ผ‰๏ผš`POST /rustfs/admin/v3/heal/`ใ€`/heal/{bucket}`ใ€`/heal/{bucket}/{prefix}`๏ผˆๅŒไธ€ POST ๆŒ‰ query `clientToken/forceStart/forceStop` ๅŒบๅˆ† start/query/cancel๏ผŒไธŽ mc admin heal ่ฏญไน‰ๅฏน้ฝ๏ผ‰๏ผ›`POST /v3/background-heal/status`๏ผ›`GET /v4/heal/replacement-recovery`ใ€‚ๆƒ้™ `HealAdminAction`๏ผˆroute_policy.rs:334-341๏ผ‰ใ€‚ +- ้›†็พคๅ่ฐƒ๏ผˆheal.rs:771-930 + `node_service.rs:514-606`๏ผ‰๏ผš`heal_topology_fingerprint` + ๆŒ‰ๆ‹“ๆ‰‘็กฎๅฎšๆ€ง้€‰ coordinator ่Š‚็‚น + coordinator epoch๏ผ›envelope ๆ ก้ชŒ + SHA256 digest ้‡ๆ”พ็ผ“้˜ฒ้‡ๆ”พ๏ผ›coordinator ้žๆœฌๆœบ่ตฐ peer gRPC `heal_control`๏ผ›`probe_heal_control` ่ƒฝๅŠ›ๆŽขๆต‹๏ผˆๆปšๅŠจๅ‡็บงๅœบๆ™ฏ๏ผ‰ใ€‚ +- ่ฏทๆฑ‚๏ผšbody ไธบ `HealOpts`๏ผˆ`recursive/dryRun/remove/recreate/scanMode(0/1/2)/updateParity/nolock/pool/set`๏ผŒserde camelCase๏ผŒไธŽ madmin.HealOpts ๅญ—ๆฎตๅฏน้ฝ๏ผ‰๏ผ›ๆ น heal start ้œ€ `recursive=true` ๆˆ– `pool+set` ๆˆๅฏน๏ผ›body ไธŠ้™ 1MBใ€‚ +- ๅ“ๅบ”๏ผš`HealStartSuccess{clientToken, clientAddress, startTime}`๏ผ›`HealTaskStatus{summary, detail, startTime, settings, items, truncated, progress}`๏ผˆsummary โˆˆ running/finished/stopped/notFound๏ผ‰๏ผ›`BackgroundHealStatus`๏ผˆbitrot ่ตทๅง‹ๆ—ถ้—ด/ๅ‘จๆœŸ/ๅฝ“ๅ‰ๆจกๅผ + `disabled/uninitialized/idle/active/degraded` ็Šถๆ€โ€”โ€”peer ไธๅฏ่พพๆ˜พๅผ degraded ไธๅ†’ๅ…… idle๏ผŒissue #5850 + `healOperations` ๆŒ‰ไผ˜ๅ…ˆ็บงร—ๆฅๆบ็Ÿฉ้˜ต + ้›†็พค่ฟ›ๅบฆ๏ผ‰ใ€‚ +- `HealResultItem`/`HealDriveInfo`/`HealItemType`/DriveState ๆžšไธพไธŽ madmin JSON ๅ…ผๅฎน๏ผˆ`crates/madmin/src/heal_commands.rs:19-65`๏ผ‰ใ€‚ +- ็Šถๆ€ payload ่ถ… 8MiB ๅฏนๆŠ˜ๆˆชๆ–ญ๏ผˆchannel.rs:37,73-104๏ผ‰๏ผ›path-token ๆ ก้ชŒ๏ผˆ้”™่ฏฏ token ๆ‹’็ป๏ผŒ็ฉบ path ไป…ๅŒน้… Cluster๏ผ‰ใ€‚ + +### 2.8 heal ๆŒ‡ๆ ‡ไธŽๆ—ฅๅฟ— + +ๆŒ‡ๆ ‡๏ผš`rustfs_heal_admission_total{source,result,reason,context}`ใ€`rustfs_heal_task_start_total`ใ€`rustfs_heal_task_running{type,set}`ใ€`rustfs_heal_queue_delay_seconds`ใ€`rustfs_heal_scheduler_skip_total`ใ€`rustfs_heal_mainline_throttle_total`ใ€`rustfs_heal_page_concurrency_current{set}`ใ€`rustfs_heal_candidate_enqueue/merge/drop/priority_reject_total`ใ€`rustfs_heal_read_repair_dedup_total{reason}` ็ญ‰ใ€‚ๆ—ฅๅฟ—ๅ…จ้ƒจ็ป“ๆž„ๅŒ– event style๏ผˆPR #5720๏ผ‰๏ผ›per-object ๆ—ฅๅฟ—้™็บง้˜ฒ้ฃŽๆšด๏ผˆ`demote_to_debug_when!`๏ผŒ#5716/#5719/#5727๏ผ‰ใ€‚ + +--- + +## 3. Scanner ๅทฒๅฎž็ŽฐๅŠŸ่ƒฝๅ…จๆ™ฏ + +### 3.1 ๅพช็Žฏใ€leaderใ€็ซ‹ๅณ่งฆๅ‘ + +- **้›†็พคๅ• leader**๏ผšๅˆ†ๅธƒๅผ ns ๅ†™้” `leader.lock`๏ผˆ`scanner.rs:3156-3207`๏ผŒ่ถ…ๆ—ถ้ป˜่ฎค 5s๏ผ‰+ **ๆŒไน…ๅŒ– leader-epoch CAS ๅ›ดๆ **๏ผšleader ็”จ ETag ๅ‰็ฝฎๆกไปถๅ‘ `.bloomcycle.bin` ๅ†™ `RSCYC001` ็ผ–็ ็š„ (cycle, leader_epoch)๏ผˆ`scanner.rs:118,1850-1861,2177-2334`๏ผ‰๏ผ›usage ๅฟซ็…งๅ†ๆ‰“ epoch fence๏ผˆ:2087-2153๏ผ‰ใ€‚้”ไธขๅคฑ โ†’ ๅ–ๆถˆๅฝ“ๅ‰ๅ‘จๆœŸ๏ผŒ30s ๆ”ถๆ•›๏ผˆ:108-111,2623-2642๏ผ‰ใ€‚ +- ๆŠข้”ๅŽ็ซ‹ๅณๆ‰ง่กŒไธ€่ฝฎ๏ผ›ๅ‘จๆœŸ = `RUSTFS_SCANNER_CYCLE` > config cycle > start_delay > ้ƒจ็ฝฒ้ป˜่ฎค > ้€Ÿๅบฆๆกฃไฝ๏ผˆยฑ10% ๆŠ–ๅŠจใ€ไธ‹้™ 1s๏ผ‰ใ€‚ +- **clean-idle ๆŒ‡ๆ•ฐ้€€้ฟ**๏ผš่ฟž็ปญๅฎŒๆ•ดๆ— ่„ๅ‘จๆœŸ้—ด้š” ร—2๏ผˆๅฐ้กถ 24h๏ผ›bitrot ๅ‘จๆœŸๅŽ‹็ผฉไธŠ้™๏ผ›ๆกถๆœ‰ lifecycle/replication ๆดปๅŠจ่ง„ๅˆ™็ฆ็”จ๏ผŒ:383-456,1382-1512๏ผ‰ใ€‚ +- **superseded/deferred ้€€้ฟ**๏ผš5s ่ตทๆŒ‡ๆ•ฐ้€€้ฟๅฐ้กถ 30min๏ผˆ:105-106,3432-3438๏ผ‰๏ผ›็ปดๆŠคๆŽขๆต‹ๅคฑ่ดฅ็‹ฌ็ซ‹้€€้ฟ๏ผˆ:459-505๏ผ‰ใ€‚ +- **็ซ‹ๅณๅ”ค้†’**๏ผšโ‘  dirty-usage ๅฟซ่ทฏๅพ„โ€”โ€”ๅ†™่ทฏๅพ„ put/delete/multipart/bucket ๆ“ไฝœ่ฐƒ็”จ `record_dirty_usage_bucket`๏ผˆ`scanner_io.rs:222-235`๏ผ›่ฐƒ็”จ็‚น `rustfs/src/app/object_usecase.rs:6221` ็ญ‰๏ผ‰๏ผŒ่‡ชๅขž generation ๅนถ Notify ๅ”ค้†’ leader๏ผŒ่„ๆกถไผ˜ๅ…ˆๆŽ’้˜Ÿ๏ผˆ`scanner_io.rs:462-488`๏ผ‰๏ผ›โ‘ก ็ปดๆŠค้…็ฝฎๅ˜ๆ›ด๏ผˆlifecycle/replication ่ฎพ็ฝฎๆ—ถ `record_scanner_maintenance_change`๏ผ‰๏ผ›โ‘ข ่ฟ่กŒๆ—ถ้…็ฝฎ็ƒญๆ›ด generation+Notify๏ผ›โ‘ฃ ้›†็พคๆดปๅŠจๅฟซ็…งๅ˜ๅŒ–ใ€‚ +- **้›†็พคๅ่ฐƒ**๏ผš`probe_scanner_activity` ๆฑ‡้›†ๆœฌๆœบ+peer ็š„ `ScannerNodeActivity`๏ผˆinstance_id/namespace_generation/maintenance_generation/protocol_version/topology_digest/data_movement_active/dirty usage๏ผ‰๏ผŒๆ‹“ๆ‰‘ๆ‘˜่ฆ่ฆ†็›– pools/sets/drives URL๏ผŒๅ่ฎฎ็‰ˆๆœฌไธ้ฝๆ‹’็ปๅ…ฑไบซ็ผ“ๅญ˜้”๏ผˆ`scanner.rs:970-1068`๏ผ‰๏ผ›**ๆ•ฐๆฎ่ฟ็งป๏ผˆrebalance/decommission๏ผ‰ๆœŸ้—ดๆŽจ่ฟŸๅ‘จๆœŸ**๏ผˆ`scanner_io.rs:2226-2374`๏ผ‰๏ผ›ๅ‘จๆœŸ็ป“ๆŸ้€ peer RPC ็กฎ่ฎค dirty-usage ack๏ผˆ`scanner.rs:2925-2952`๏ผ‰ใ€‚ + +### 3.2 ้ๅŽ†ๆจกๅž‹ + +- ไธป้ๅŽ†ๆ˜ฏ**ๅ…จ้‡็›ฎๅฝ• walk**๏ผˆtokio::fs::read_dir ้€’ๅฝ’๏ผŒ`scanner_folder.rs:1915-2234`๏ผ‰๏ผŒไธ่ตฐ metacache๏ผ›metacache/`list_path_raw` ไป…็”จไบŽ abandoned children ่ทจ็›˜ๆ ธๆŸฅ๏ผˆ:2528-2792๏ผ‰ใ€‚ +- ไธ‰็บงๅนถๅ‘๏ผšleader โ†’ per-set๏ผˆไฟกๅท้‡้ป˜่ฎค 4๏ผ‰โ†’ per-disk ๆกถๆ‰ซๆ๏ผˆ้ป˜่ฎค 4๏ผ‰โ†’ ๅ•็›˜้€’ๅฝ’๏ผ›ๆฏๆกถๆฏ set ็ผ“ๅญ˜้” `.scanner-cycle.lock.pool-N.set-M`๏ผˆ้”ไธขๅคฑๅ–ๆถˆ่ฏฅๆกถๆ‰ซๆ๏ผŒ้”็ซžไบ‰้‡ๆŽ’้˜Ÿ๏ผ‰๏ผ›ๆฏ็›˜ๅ•ๆ‰ซๆๅ‡†ๅ…ฅ๏ผˆๆœฌๅœฐ็›˜ไนŸ่ตฐไฟกๅท้‡๏ผŒ`scanner_io.rs:3246-3274`๏ผ‰ใ€‚ +- ๆกถ้กบๅบ๏ผšshuffle ๅŽๆŒ‰ dirty โ†’ ๆœช็ผ“ๅญ˜ โ†’ ๅทฒ็ผ“ๅญ˜้‡ๆŽ’๏ผˆ`scanner_io.rs:2947-2949,462-488`๏ผ‰๏ผ›็›ฎๅฝ•ๅ†…ๆŒ‰ๅๅญ—ๆŽ’ๅบ + resume ๆ็คบๆ—‹่ฝฌ๏ผˆ`scanner_folder.rs:333-359`๏ผ‰ใ€‚ +- **ๆ–ญ็‚น็ปญๆ‰ซ**๏ผš`DataUsageScanCheckpoint{version,resume_after,reason}` ๆŒไน…ไบŽ็ผ“ๅญ˜ info๏ผˆ`data_usage_define.rs:68,293-307`๏ผ‰๏ผ›้ข„็ฎ—่€—ๅฐฝ/ๅ–ๆถˆๅ†™ๅ…ฅ๏ผŒๆขๅคๆœ‰ Used/Stale/NoHint ๆŒ‡ๆ ‡๏ผ›็ปญๆ‰ซๅ•ไฝๆ˜ฏ็›ฎๅฝ•๏ผˆๆ— ่ทจๅ‘จๆœŸๅฏน่ฑก็บงๅˆ†้กต๏ผ‰ใ€‚ +- erasure ่ฏญไน‰๏ผšๅ‘็Žฐ `xl.meta` ๅณๅฏน่ฑก่พน็•Œไธไธ‹้’ป๏ผ›UUID data-dir ๅ€™้€‰ๆœ€ๅคšๆŽขๆต‹ 64 entry๏ผ›ๆœ‰ๆ•ฐๆฎๆ— ๅ…ƒๆ•ฐๆฎ โ†’ ่ฎฐ failed + ้ซ˜ไผ˜ heal๏ผ›symlink ็›ฎๅฝ•ๅฟฝ็•ฅ/็Žฏ่ทณ่ฟ‡ใ€‚ +- ๅไฝœ่ฎฉๅ‡บ๏ผšๆฏ N ๅฏน่ฑก๏ผˆ้ป˜่ฎค 128๏ผ‰`yield_now`ใ€‚ + +### 3.3 ๅคงๆกถ่ทณ่ฟ‡็ญ–็•ฅ๏ผˆๅฏนๆ ‡ MinIO compaction๏ผ‰ + +1. ็ผ“ๅญ˜ๅฝ“ๅ‰ๆ€งๅค็”จ๏ผšๆกถไธŽๆ‰ซๆ่ฎกๅˆ’ๆœชๅ˜๏ผˆname/source/snapshot_complete/plan digest/next_cycle/leader_epoch/cache_key_format ๅ…จๅŒน้…๏ผ‰ๆ•ดๆกถ่ทณ่ฟ‡๏ผˆ`scanner_io.rs:1062-1109`๏ผ‰ใ€‚ +2. compacted ็›ฎๅฝ• 16 ๅ‘จๆœŸ่ฝฎๆข็ช—ๅฃ๏ผš`hash mod (next_cycle, 16)` ๅ‘ฝไธญๆ‰้‡ๆ‰ซ๏ผŒๅฆๅˆ™ไปŽๆ—ง็ผ“ๅญ˜ๆ‹ท่ด๏ผˆ`scanner_folder.rs:74,2429-2442`๏ผ‰ใ€‚ +3. compaction ้˜ˆๅ€ผ๏ผšๅญ้กน <500 ๆˆ–็บฏๅฏน่ฑกๅถๅญๅŽ‹็ผฉไธบๅ• entry๏ผ›ๅญๆ–‡ไปถๅคน โ‰ฅ2500๏ผˆๆ น 10000๏ผ‰้ข„ๅŽ‹็ผฉ๏ผ›children โ‰ฅ10000 ๅฝ’็บฆ๏ผˆ:75-78,2314-2340,2846-2887๏ผ‰ใ€‚ +4. ๅคฑ่ดฅๅฏน่ฑก TTL ่ทณ่ฟ‡๏ผš86400s/ๆœ€ๅคš 10000 ๆก๏ผˆ:88-91,1354-1381๏ผ‰ใ€‚ + +ไธŽ MinIO master ๅฏนๆฏ”๏ผšMinIO ็š„่ทณ่ฟ‡็ญ–็•ฅๅŒๆ ทๆ˜ฏ hash-mod-16 ๅ‘จๆœŸ + compaction ้˜ˆๅ€ผๆ ‘๏ผˆ500/10000/2500๏ผ‰๏ผŒ**bloom filter ๅทฒไปŽ master ๅˆ ้™ค**ใ€‚RustFS ็š„ๅธธ้‡ไธŽ็ป“ๆž„ไธŽ MinIO ็Žฐ็ŠถๅŒๆบ๏ผˆMinIO ๆœช้‡‡็”จ่ทจ็›˜ dirty-generation ไผ˜ๅ…ˆ๏ผŒRustFS ้ขๅค–ๅคšไธคๅฑ‚่ทณ่ฟ‡โ€”โ€”plan digest ไธŽ็ผ“ๅญ˜ๅฝ“ๅ‰ๆ€งๆ ก้ชŒ๏ผ‰ใ€‚ + +### 3.4 data usage ็ปŸ่ฎก + +- ็ปดๅบฆ๏ผšๆฏ็›ฎๅฝ• entry๏ผˆsize/objects/versions/delete_markers/ๅคงๅฐ็›ดๆ–นๅ›พ/็‰ˆๆœฌ็›ดๆ–นๅ›พ/ๅคๅˆถ็ปŸ่ฎก/failed_objects/per-tier stats/children/compacted๏ผŒ`data-usage/src/data_usage.rs:661-679`๏ผ‰๏ผ›ๆฏๅฏน่ฑก SizeSummary๏ผˆๅซ per-ARN ๅคๅˆถ็›ฎๆ ‡็ปŸ่ฎกใ€tier ็ปŸ่ฎก๏ผŒtier ๅˆ†็ฑป๏ผštransitioned ๅฎŒๆˆ่ฎฐๅ…ฅๅ…ถ tier ๅฆๅˆ™ๆŒ‰ storage class๏ผŒfree version ไธ่ฎก๏ผ‰๏ผ›ๆกถ็บง `BucketUsageInfo`๏ผ›้›†็พค็บง `DataUsageInfo`๏ผˆๅซ scanner_cycle/scanner_epoch ๅ›ดๆ  + usage_snapshot_complete๏ผ‰ใ€‚ +- ๅญ˜ๅ‚จ๏ผšๆฏๆกถๆฏ set `{bucket}/.usage-cache.bin`๏ผˆไธป + `.bkp` ๅค‡ไปฝ + CAS ้‡่ฏ•๏ผ‰๏ผ›ๆƒๅจ้›†็พคๅฟซ็…ง `buckets/data-usage/data-usage.json`๏ผˆๆฏ 10 ๅ‘จๆœŸๅŒๆญฅ `.bkp`๏ผŒlegacy ่ทฏๅพ„ๅ…ผๅฎน๏ผ‰๏ผ›้™ˆๆ—งๅฟซ็…งๆ‹’็ปๅ†™ๅ…ฅ๏ผˆepoch/cycle/last_update ไธ‰้‡ๅˆคๅฎš๏ผ‰๏ผ›่ขซ็ซžไบ‰ superseded ็š„่ง‚ๆต‹ๅฟซ็…งๅฆๅญ˜ `data-usage-observed.json`ใ€‚ +- ๆถˆ่ดน๏ผš`replace_bucket_usage_memory_from_info` ๅˆทๆ–ฐๆกถ็”จ้‡ๅ†…ๅญ˜ + ไธคๅฑ‚็ผ“ๅญ˜ๅคฑๆ•ˆ๏ผˆ`scanner.rs:4142-4152`๏ผ‰โ†’ bucket stats/quota/admin account_info/system๏ผ›ๅ†™่ทฏๅพ„ๅ†…ๅญ˜ๅฎžๆ—ถๅ ๅŠ  overlay๏ผ›ๅฏๅŠจ่ฏปๅฟซ็…งๅˆคๆ–ญๅ†ท็ผ“ๅญ˜่ทณ่ฟ‡ๅฏๅŠจๅปถ่ฟŸใ€‚ +- ๆœชๅฎŒๆˆ multipart ไธๅ‚ไธŽ็ปŸ่ฎก๏ผˆไธŽ MinIO ไธ€่‡ด๏ผŒMinIO ไนŸไธๆ‰ซ multipart ๆกถ๏ผ‰ใ€‚ + +### 3.5 ILM ้›†ๆˆ + +- ๆฏๅฏน่ฑก `ScannerItem::apply_actions`๏ผˆ`scanner_folder.rs:747-1032`๏ผ‰๏ผš`Evaluator::new(lifecycle).with_lock_retention(...).with_replication_config(...).eval()` ๆ‰น้‡่ฏ„ไผฐใ€‚ +- ๅทฒๅฎž็ŽฐๅŠจไฝœ๏ผˆIlmAction ๅ…จ้›†๏ผŒ`common/src/metrics.rs:34-45`๏ผ‰๏ผšexpiry ๅˆ ้™ค๏ผˆDelete/DeleteRestored/DeleteRestoredVersion๏ผ‰ใ€ๅ…จ็‰ˆๆœฌๅˆ ้™ค๏ผˆDeleteAllVersions/DelMarkerDeleteAllVersions๏ผŒๅค„็†ๅŽๅœๆญขๅŽ็ปญ็‰ˆๆœฌ๏ผ‰ใ€transition๏ผˆTransition/TransitionVersion๏ผŒtier ๅˆ—่กจ่ฟ่กŒๆ—ถ่ฏปๅ–๏ผ‰ใ€noncurrent ๆ‰น้‡๏ผˆDeleteVersionAction โ†’ `enqueue_by_newer_noncurrent`๏ผ‰ใ€free-version ๆธ…็†๏ผˆ`enqueue_free_version`๏ผ‰ใ€object-lock retention ็บฆๆŸใ€‚**ไธŽ MinIO ็š„ 9 ไธช ILM ๅŠจไฝœไธ€ไธ€ๅฏนๅบ”**ใ€‚ +- ๆ‰ง่กŒๆจกๅž‹๏ผšscanner ๆ˜ฏ"ๅ‘็ŽฐไธŽๅ…ฅ้˜Ÿ"่ง’่‰ฒ๏ผˆexpiry ้˜Ÿๅˆ—/transition ้˜Ÿๅˆ—ๅœจ ecstore `bucket_lifecycle_ops.rs`๏ผ‰๏ผŒๅŠจไฝœ็”ฑ worker ๆฑ ๆถˆ่ดนโ€”โ€”ไธŽ MinIO globalExpiryState/globalTransitionState ๅŒๅž‹ใ€‚ +- AbortIncompleteMultipartUpload ไธๅœจ scanner/ILM ๅ†…ๆ‰ง่กŒ๏ผˆMinIO ๅŒๆ ทไธๅœจ๏ผš`internal/bucket/lifecycle/rule.go` ๆœ‰ FIXME๏ผŒๅฎž้™…็”ฑ `erasureSets.cleanupStaleUploads` ๅ…จๅฑ€ไพ‹็จ‹ๆ‰ฟๆ‹…๏ผ‰๏ผ›RustFS ็”ฑ ecstore ็‹ฌ็ซ‹ๅŽๅฐไปปๅŠก `init_background_stale_multipart_upload_cleanup`๏ผˆ`bucket_lifecycle_ops.rs:3289-3320`๏ผ‰+ ๆกถๅˆ ้™คๆ—ถ on-demandใ€‚ +- ้›†ๆˆๆต‹่ฏ•่ฆ†็›–๏ผštransition+restoreใ€free-versionใ€noncurrentใ€delete-markerใ€0-dayใ€ๅŽๅฐๆ‰ซๆ่ฟ‡ๆœŸ๏ผˆ`scanner/tests/lifecycle_integration_test.rs:1071-2095`๏ผ‰ใ€‚ + +### 3.6 heal ๅ€™้€‰็”Ÿไบง๏ผˆscanner ไพง๏ผ‰ + +- ๆŠฝๆ ท๏ผš`hash mod_alt(next_cycle/prob_div, 1024/prob_div)`๏ผŒ่ฟ›ๅ…ฅ compacted ๅˆ†ๆ”ฏ้‡ๆ‰ซๆ—ถ prob_div=16 ็ญ‰ๆ•ˆๆฆ‚็Ž‡ ร—16๏ผˆไธŽ MinIO ๅŒๆฌพ่กฅๅฟ๏ผŒ`scanner_folder.rs:125-127,2117-2122`๏ผ‰ใ€‚ +- deep/normal๏ผšๅ‘จๆœŸ็บง `get_cycle_scan_mode`๏ผˆbitrot_cycle ้ป˜่ฎค 30d๏ผŒ`scanner.rs:1626-1657`๏ผ‰โ†’ ๅฏน่ฑก็บงๅธฆ `HealScanMode::Deep`๏ผ›ๆ–ฐ้ฒœๅฏน่ฑก๏ผˆ60s ๅ†…ไฟฎๆ”น๏ผ‰้™็บง Normal๏ผˆ:146-155๏ผ‰๏ผ›็Šถๆ€ๆŒไน… `.background-heal.json`๏ผˆ`BackgroundHealInfo{bitrot_start_time,bitrot_start_cycle,current_scan_mode}`๏ผŒไธŽ MinIO ๅŒ่ทฏๅพ„ๅŒ็ป“ๆž„๏ผ‰ใ€‚ +- scanner ๅชๅ…ฅ้˜Ÿไธๅ†…่”ๆ‰ง่กŒ๏ผˆๅ†…่” heal ๅทฒ็งป้™ค๏ผŒๅ…ผๅฎนๆ——ๆ ‡ไป…ๅ‘Š่ญฆ๏ผŒ`scanner_folder.rs:411-427`๏ผ‰๏ผ›`HealScanMode::Deep` ๅชๆ˜ฏๆ ‡่ฎฐ๏ผŒbitrot ๆ ก้ชŒ่ฏปๅ‘็”Ÿๅœจ heal ๆถˆ่ดน็ซฏ๏ผˆecstore Deep ่ทฏๅพ„๏ผ‰ใ€‚ +- ๅ…ƒๆ•ฐๆฎๆŸๅ โ†’ ้ซ˜ไผ˜ heal๏ผˆ`classify_get_size_failure` โ†’ HealMetadata๏ผ‰๏ผ›abandoned children โ†’ list_path_raw quorum ๆ ธๆŸฅ + ๆกถ็บง/ๅฏน่ฑก็บง้ซ˜ไผ˜ heal๏ผ›healing ็›˜็ฒ˜ๆ€ง่ทณ่ฟ‡๏ผˆ`should_heal` :1628-1648๏ผ‰ใ€‚ +- pending-heal ่ดฆๆœฌ๏ผšheal ้€š้“ๆปก่ขซๆ‹’ๆŒไน…ๅŒ–ๅˆฐ็ผ“ๅญ˜ info๏ผŒไธ‹่ฝฎ้‡่ฏ•ใ€‚ +- ๅคๅˆถ heal๏ผš`queue_replication_heal` โ†’ replication ้˜Ÿๅˆ—๏ผˆ่ตฐ replication ้€š้“่€Œ้ž heal channel๏ผ‰๏ผ›per-ARN ๅคๅˆถ็”จ้‡็ปŸ่ฎกใ€‚ + +### 3.7 remote_scanner RPC ๅ่ฎฎ๏ผˆRustFS ็‰นๆœ‰๏ผ‰ + +่ฏทๆฑ‚ โ‰ค16KB msgpack๏ผˆversion/request_id/server_epoch/session_id/session_sequence/bucket/next_cycle/leader_epoch/scan_plan_digest/skip_healing/scan_mode/budget๏ผ‰๏ผ›ๅธง โ‰ค2MBใ€HMAC-SHA256 ้€ๅธง่ฎค่ฏ๏ผˆๅŸŸ `rustfs-ns-scanner-frame-v3`๏ผ‰๏ผ›่ฟ›ๅบฆๅฟƒ่ทณ 1s๏ผˆ้ข„็ฎ—ๆจกๅผ 250ms๏ผ‰๏ผ›้˜ถๆฎตๆ’ญๆŠฅ Scanningโ†’Persisting๏ผ›RPC ็”Ÿๅ‘ฝๅ‘จๆœŸไธŠ้™ 24hใ€ๆ–ญ่ฟžๅฎฝ้™ 2min๏ผ›้˜ฒ้‡ๆ”พ session+sequence ็ผ“ๅญ˜๏ผˆๅฎน้‡ 65536๏ผ‰๏ผ›ๆœๅŠก็ซฏๆ ก้ชŒ leader fence ไธŽๆŒไน…ๅŒ– cycle ไธ€่‡ด + ๆฏ 5s fence ๅค้ชŒ๏ผ›็ป“ๆžœ Complete/Partial/NamespaceNotFound/CycleAhead๏ผ›ไธๆ”ฏๆŒ v4 ๅ่ฎฎ็š„่ฟœ็ซฏ็›˜ๅ›ž้€€ leader ๆœฌๅœฐๆ‰ซๆ๏ผˆ`remote_scanner.rs` ๅ…จๆ–‡ไปถ๏ผ›`scanner_io.rs:2750-2812`๏ผ‰ใ€‚ + +### 3.8 ้™้€Ÿ/้ข„็ฎ—/็ƒญๆ›ด/่ง‚ๆต‹ + +- DynamicSleeper ๆฏ”ไพ‹้€€้ฟ๏ผˆ้€Ÿๅบฆๆกฃ fastest/fast/default/slow/slowest๏ผŒๅŒ MinIO ไบ”ๆกฃๅ‚ๆ•ฐ๏ผ‰๏ผ›idle_mode ๆ€ป้—ธ๏ผ›ๅ‰ๅฐ S3 ่ฏปๆต้‡ๆฏ่ฏทๆฑ‚ 10ms ๅฐ้กถ 250ms ้ขๅค–้€€้ฟใ€‚ +- ๅ‘จๆœŸ้ข„็ฎ— ScannerCycleBudget๏ผšmax_duration/max_objects/max_directories๏ผˆ้ป˜่ฎค 0=ไธ้™๏ผ‰๏ผŒpartial ๅ‘จๆœŸไปๆŽจ่ฟ› cycle ่ฎกๆ•ฐใ€‚ +- runtime_config ไธ‰ๅฑ‚ๆฅๆบ๏ผˆenv > config > default๏ผ‰้€ๅญ—ๆฎตๆฅๆบๆ ‡่ฎฐ๏ผˆEnv/Config/ScannerCompatConfig/Default๏ผ‰๏ผŒadmin `PUT /v3/config` ็ƒญๆ›ด โ†’ generation+Notify ๅณๆ—ถ็”Ÿๆ•ˆ๏ผ›`GET /v3/scanner/status` ่ฟ”ๅ›ž enabled/freshness(fresh/stale/unknown)/metrics/cycle_schedule/runtime_config๏ผ›`GET /v3/ilm/expiry/status` ่ฟ”ๅ›ž expiry ้˜Ÿๅˆ—/worker/missed/blockedใ€‚ +- ๆŒ‡ๆ ‡๏ผšleader lockใ€ๅ‘จๆœŸ complete/partial/deferred/supersededใ€versions scannedใ€per-source๏ผˆUsage/Lifecycle/BucketReplication/SiteReplication/Heal/Bitrot/Alerts๏ผ‰checked/executed/queued/missedใ€checkpoint set/used/staleใ€ๅฝ“ๅ‰่ทฏๅพ„๏ผˆper-disk+bucket ๅฎžๆ—ถ๏ผ‰ใ€็ผ“ๅญ˜ save ็ณปๅˆ—ใ€ๅนถๅ‘็ณปๅˆ—ใ€ๅ‘Š่ญฆ๏ผˆexcess versions/version size/folders๏ผ‰ใ€‚ + +--- + +## 4. ไธŽ MinIO ้€้กนๅฏนๆ ‡ + +### 4.1 heal ่งฆๅ‘้€š้“ๅฏน็…ง + +| MinIO ้€š้“ | RustFS ๅฏนๅบ” | ็Šถๆ€ | +|---|---|---| +| A. ๆ‰‹ๅŠจ admin heal๏ผˆhealSequence๏ผŒclientToken/forceStart/forceStop๏ผ‰ | heal channel Start/Query/Cancel + ้›†็พค coordinator + envelope ้‡ๆ”พ้˜ฒๆŠค | โœ… ็ญ‰ไปทไธ”ๅขžๅผบ๏ผˆ้›†็พค่ทฏ็”ฑ๏ผ‰๏ผ›ๅบๅˆ—่ฏญไน‰ๅทฎๅผ‚่ง ยง6 HS-06 | +| B. ๅธธ้ฉปๅŽๅฐ heal ้˜Ÿๅˆ—๏ผˆnewBgHealSequence + healRoutine worker ๆฑ ๏ผ‰ | HealManager ๅธธ้ฉป่ฐƒๅบฆๅ™จ + ไผ˜ๅ…ˆ็บง้˜Ÿๅˆ— + bulkhead | โœ… ็ญ‰ไปทไธ”ๅขžๅผบ | +| C. ๆ–ฐ็›˜/ๆข็›˜่‡ชๅŠจ resync๏ผˆmonitorLocalDisksAndHeal 10s + healFreshDisk + healingTracker + waitForFormatErasure ๆกๆ‰‹๏ผ‰ | auto disk scanner๏ผˆ10s๏ผ‰+ replacement_readiness + durable intent/proof ็Šถๆ€ๆœบ + heal_replacement_format | โœ… ็ญ‰ไปทไธ”ๅขžๅผบ๏ผˆidentity fence + completion proof๏ผ›MinIO ็š„ tracker ้ขๅ‘ๅฏนๅค–ๅฏ่งๆ€งๆ›ดๅผบ๏ผŒ่ง ยง6 HS-07๏ผ‰ | +| D. MRF๏ผˆ้˜Ÿๅˆ— 100k + ๆŒไน…ๅŒ– list.bin + shutdown ๅ›žๆ”พ + ่ฏป่ทฏๅพ„ corrupt ๆŠ•้€’๏ผ‰ | read-repair๏ผˆLow+TTL ๅŽป้‡๏ผ‰+ ๅ†™่ทฏๅพ„ convergence heal ้ƒจๅˆ†ๆ‰ฟๆ‹…๏ผ›`HealType::MRF` ๆ‰ง่กŒไฝ“ๆ— ็”Ÿไบงๅ…ฅๅฃ | โš ๏ธ ้ƒจๅˆ†็ญ‰ไปท๏ผˆยง6 HS-01๏ผ‰ | +| E. Scanner ๆŠฝๆ ท heal๏ผˆ1/1024 + compacted ร—16 ่กฅๅฟ๏ผ‰+ abandoned children | ๅŒๆฌพๆŠฝๆ ท + ร—16 ่กฅๅฟ + abandoned children + pending-heal ่ดฆๆœฌ | โœ… ็ญ‰ไปทไธ”ๅขžๅผบ๏ผˆ่ดฆๆœฌ๏ผ‰ | +| F. ่ฏป่ทฏๅพ„ๅ†…่”่งฆๅ‘ โ†’ MRF๏ผˆGetObject part ็ผบๅคฑ/ๆŸๅใ€ๅ…ƒๆ•ฐๆฎ้‡ๅปบ missingBlocks>0๏ผ‰ | read repair๏ผˆmissing_shards/decode_error/metadata_read_error ไธ‰ๅ…ฅๅฃ๏ผ‰ | โœ… ็ญ‰ไปท๏ผˆๅ…ฅ heal ้˜Ÿๅˆ—่€Œ้ž MRF ้˜Ÿๅˆ—๏ผ‰ | + +### 4.2 ๅฏน่ฑก็บง heal ่ฏญไน‰ๅฏน็…ง + +| ็‰นๆ€ง | MinIO | RustFS | ็Šถๆ€ | +|---|---|---|---| +| mod-time quorum ไปฒ่ฃ | listOnlineDisks | ๅŒ | โœ… | +| ETag ๅคšๆ•ฐๆดพๅ…œๅบ•๏ผˆๆ—ถ้’Ÿๆผ‚็งป๏ผ‰ | filterDisksByETag | `filter_by_etag`/`quorum_etag`๏ผˆheal.rs:525-567๏ผ‰ | โœ… ๅทฒไบฒ้ชŒ | +| cannotHeal ็š„ ETag ่ฑๅ… | ETag ๅ…จไธ€่‡ด่ฑๅ…้‡่ฏ• | heal.rs:679 | โœ… | +| Normal=CheckParts๏ผˆstat๏ผ‰/ Deep=VerifyFile๏ผˆbitrot๏ผ‰ | ๆ˜ฏ | `disks_with_all_parts` ๆŒ‰ scan_mode๏ผˆops/heal.rs:562-572,978-1024๏ผ‰ | โœ… | +| Normal ๆฃ€ๅ‡บ corrupt ่‡ชๅŠจๅ‡ Deep ้‡่ฏ•ไธ€ๆฌก | erasure-healing.go:1101-1106 | ops/heal.rs:2022-2031 | โœ… | +| dangling ๅˆคๅฎš๏ผˆnot-found > parity๏ผ‰+ ๅˆ ้™คๅฎก่ฎก | isObjectDangling/deleteIfDangling | `dangling_delete_safety`๏ผˆ:1488๏ผ‰+ scanner HEAL_DELETE_DANGLING | โœ…๏ผˆๅฎก่ฎก tags ็ป†่Š‚ๆœ‰ๅทฎๅผ‚๏ผ‰ | +| ๅญคๅ„ฟ data-dir/inline ๆธ…็†๏ผˆCleanAbandonedData๏ผ‰ | CheckAbandonedParts๏ผˆscanner ๆŠฝไธญ + admin Remove ๆ—ถๆ˜พๅผ่ฐƒ็”จ๏ผ‰ | heal ่ทฏๅพ„ๅ†… `reclaim_orphan_data_dirs_best_effort`๏ผˆ:1428๏ผ‰๏ผ›็‹ฌ็ซ‹ API ไธ‰ๅฑ‚ NotImplemented | โš ๏ธ ้ƒจๅˆ†็ญ‰ไปท๏ผˆยง6 HS-02๏ผ‰ | +| ็‰ˆๆœฌๅŒ–/delete-marker heal | HealObject versionID๏ผ›nullVersionID ็‰นๅˆค | ้€็‰ˆๆœฌๆžšไธพ + delete-marker latest heal๏ผˆB5 ๅ›žๅฝ’๏ผ‰ | โœ… | +| ๅฏน่ฑก็บง healing ๅ…ƒๆ•ฐๆฎๆ ‡่ฎฐ๏ผˆx-minio-healing๏ผŒRenameData ่ทณ่ฟ‡็‰ˆๆœฌๆธ…็†๏ผ‰ | ๆœ‰ | ๆ— ๅฏน่ฑก็บงๆ ‡่ฎฐ๏ผ›ไพ่ต–็›˜็บง healing.bin + NSLock + rename ่ฏญไน‰ | โš ๏ธ ่ฏ„ไผฐ้กน๏ผˆยง6 HS-12๏ผ‰ | +| Distribution/Index ไธ€่‡ดๆ€งไธ‰ๅค„้˜ฒ็บฟ | ๆœ‰๏ผˆmanual modification ๆ‹’็ป๏ผ‰ | ็›ฎๆ ‡็›˜ๆ ผๅผ็ป“ๆžœๅ…จ ok ๆ ก้ชŒ + ่บซไปฝๅ›ดๆ  | โœ…๏ผˆ็ฒ’ๅบฆไธๅŒ๏ผ‰ | +| ๆ—  parity๏ผˆEC:0๏ผ‰ๅฏน่ฑก | bitrot ไธๅฏๆขๅคๅค„็† | ๅˆคไธๅฏๆขๅค๏ผˆ:700-726๏ผ‰+ ๅ†™ๅ…ฅ่‡ชๆ ก้ชŒ | โœ… ๅขžๅผบ๏ผˆๅ†™่ทฏๅพ„่‡ชๆ ก้ชŒ๏ผ‰ | +| ไธ‰ๅฑ‚ๅˆ†ๅธƒไธไธ€่‡ดๆ‹’็ป heal | ๆœ‰ | heal_walk ๅฝ’ไธ€ๅŒ– + ้กต็•Œ้˜ฒๅพก | โœ…๏ผˆๅฎž็Žฐๆ–นๅผไธๅŒ๏ผ‰ | +| multipart ๅญคๅ„ฟๅฏน่ดฆ | CheckAbandonedParts ๆ‰ฟๆ‹… | ๆ˜พๅผ NotImplemented๏ผˆ็”ฑ lifecycle ๆธ…็†ๆ‰ฟๆ‹…๏ผ‰ | โš ๏ธ ยง6 HS-02 | +| suspended/decommissioned pool ๅค„็† | IsSuspended ่ทณ่ฟ‡ | deferral ่ฏญไน‰๏ผˆstore/heal.rs:192-207๏ผŒPR #5876๏ผ‰ | โœ… | +| heal ไธŽๅนถๅ‘ๅˆ ้™คไบ’ๆ–ฅ | NSLock + healing ๆ ‡่ฎฐ | NSLock + ๅ†™้” | โœ… | + +### 4.3 ๆ–ฐ็›˜ resync ๅฏน็…ง + +| MinIO | RustFS | ็Šถๆ€ | +|---|---|---| +| waitForFormatErasure ๅ››็ฑปๅฏๆขๅค้”™่ฏฏๆ— ้™็ญ‰ๅพ…ๆกๆ‰‹ | startup ็›˜่งฃๆž + renew_disk ้‡่ฟž่ทฏๅพ„ | โœ…๏ผˆๆจกๅž‹ไธๅŒ๏ผšRustFS ไธๅœจๅฏๅŠจๆ—ถ้˜ปๅกž็ญ‰ๅพ… format๏ผ‰ | +| HealFormat NSLock + errNoHealRequired + refFormat ไธไธ€่‡ดๆ‹’็ป | `heal_format`/`heal_replacement_format` fail-closed + ็›ฎๆ ‡ๆงฝไฝ้™ๅฎš๏ผˆPR #1787 ่ฏญไน‰๏ผ‰ | โœ… ๅขžๅผบ | +| per (pool,set) ๅˆ†ๅธƒๅผ้”้˜ฒๅนถๅ‘ resync | set ็บง้˜Ÿๅˆ—ๅŽป้‡ + bulkhead๏ผˆmanager.rs:2854-2889๏ผ‰ | โœ… | +| ๅ…จๆ–ฐ้›†็พคๆฃ€ๆต‹๏ผˆๅพ… heal ็›˜ๆ•ฐ==ๆ€ป็›˜ๆ•ฐไธ่งฆๅ‘๏ผ‰ | replacement_readiness๏ผˆ็‹ฌ็ซ‹ๆŒ‚่ฝฝ็‚น/็‰ฉ็†่ฎพๅค‡ๆ ก้ชŒ๏ผŒ้ž root๏ผ‰ | โœ… ๅขžๅผบ | +| healingTracker๏ผˆ.healing.bin๏ผšBytes/Items ่ฎกๆ•ฐใ€QueuedBuckets/HealedBucketsใ€Resume ๅฟซ็…งใ€RetryAttempts โ‰ค4ใ€HealID ่”ๅŠจใ€diskID ๅ˜ๆ›ด้‡็ฝฎ๏ผ‰ | resume/checkpoint schema ๅŒ–ๆŒไน…ๅฑ‚ + durable intent/proof๏ผˆper-task ๆ–‡ไปถ๏ผŒCAS๏ผ‰ | โœ… ็ญ‰ไปทไธ”ๅขžๅผบ๏ผˆๅดฉๆบƒ็ช—ๅฃ่กฅ้ฝ๏ผ‰๏ผ›ไฝ†**ๅฏนๅค–ๅฟซ็…งๅฏ่งๆ€ง**ๅผฑไบŽ MinIO๏ผˆยง6 HS-07๏ผ‰ | +| ่ทณ่ฟ‡ heal ๅผ€ๅง‹ๅŽๆ–ฐๅ†™ๅ…ฅ็‰ˆๆœฌ๏ผˆModTime > Started๏ผ‰ | ๆ— ๅŒๆฌพ่ฟ‡ๆปค | โš ๏ธ ยง6 HS-13 | +| ่ทณ่ฟ‡ ILM ๅทฒ่ฟ‡ๆœŸ็‰ˆๆœฌ๏ผˆfilterLifecycle๏ผ‰ | ๆ— ๅŒๆฌพ่ฟ‡ๆปค | โš ๏ธ ยง6 HS-13 | +| worker ๆ•ฐ max(GOMAXPROCS,NR)/4 ไธ‹้™ 4๏ผŒheal:drive_workers ่ฆ†็›– | ้กตๅ†…ๅนถๅ‘ 8๏ผˆDeep/AutoHeal ๅผบๅˆถ 1๏ผ‰+ per-set bulkhead | โœ…๏ผˆๅ‚ๆ•ฐๆจกๅž‹ไธๅŒ๏ผ‰ | +| ๆฏ entry waitForLowHTTPReq ่ฎฉ่ทฏ | mainline throttle๏ผˆๅ‰ๅฐๅˆฉ็”จ็Ž‡้—จๆŽง๏ผ‰ | โœ… ๅขžๅผบ | +| heal ่Œƒๅ›ดๅซ `.minio.sys/config`ใ€`.minio.sys/buckets` ไธคไธชไผชๆกถ๏ผ›ๆœ€ๆ–ฐๆกถไผ˜ๅ…ˆ | ErasureSet ไปปๅŠก้€ bucket ้ข„ๅค„็†๏ผˆๅซ meta bucket ่ฏญไน‰็”ฑ heal_bucket ๆ‰ฟๆ‹…๏ผ‰ | โœ…๏ผˆ้กบๅบๆ— "ๆœ€ๆ–ฐไผ˜ๅ…ˆ"๏ผ‰ | +| ๅคฑ่ดฅๆ•ดไฝ“้‡่ฏ• โ‰ค4 ๆฌก๏ผˆresetHealing + errRetryHealing๏ผ‰ | schedule_retry ๅคไฝๅŒๅฑ‚ + ๅฏๆขๅค้‡่ฏ• โ‰ค3 | โœ… | + +### 4.4 scanner ๅฏน็…ง + +| MinIO | RustFS | ็Šถๆ€ | +|---|---|---| +| ้›†็พคๅ• leader๏ผˆglobalLeaderLock๏ผ‰ | leader.lock + ๆŒไน…ๅŒ– leader-epoch CAS ๅ›ดๆ  | โœ… ๅขžๅผบ๏ผˆepoch ๅ›ดๆ ้˜ฒ่„‘่ฃ‚๏ผŒMinIO ๆ— ๆŒไน…ๅŒ– epoch๏ผ‰ | +| `.bloomcycle.bin` ๅชๅญ˜ cycle๏ผˆbloom ๅทฒๅˆ ้™ค๏ผ‰ | ๅŒ่ทฏๅพ„ๅญ˜ cycle+leader_epoch๏ผˆRSCYC001๏ผ‰ | โœ… ๅฏน้ฝ๏ผˆv1 ่ฏฏๅˆคๅทฒไฟฎๆญฃ๏ผ‰ | +| folderScanner hash-mod-16 + compaction๏ผˆ500/10000/2500๏ผ‰ | ๅŒๆฌพๅธธ้‡ + plan digest + ็ผ“ๅญ˜ๅฝ“ๅ‰ๆ€งๆ ก้ชŒ + dirty ไผ˜ๅ…ˆ | โœ… ๅขžๅผบ | +| ๆฏ็›˜ๆ‰ซๆๅนถ่กŒ โ‰คGOMAXPROCS๏ผ›healing ็›˜ๆŽ’้™ค | per-set/per-disk ไฟกๅท้‡ + healing ็›˜็ฒ˜ๆ€ง่ทณ่ฟ‡ | โœ… | +| scannerSleeper๏ผˆfactor 2/max 1s๏ผŒspeed ๆกฃ็ƒญๆ›ด๏ผ‰ | DynamicSleeper ๅŒๆฌพ + idle_mode + ๅ‰ๅฐ่ฏป้€€้ฟ | โœ… ๅขžๅผบ | +| idle ่ฏญไน‰๏ผš`scanner:idle_speed=on`๏ผˆ็ฉบ้—ฒๆ—ถๆฎตๆ‰่Š‚ๆต๏ผŒๅฟ™ๆ—ถๅ…จ้€Ÿ๏ผ‰ | `RUSTFS_SCANNER_IDLE_MODE=true`๏ผˆๅฏ็”จ้™้€Ÿๆ€ป้—ธ๏ผ‰ | โš ๏ธ ่ฏญไน‰ๆ–นๅ‘็›ธๅ๏ผŒยง6 HS-14 | +| applyActions ้กบๅบ๏ผˆhealโ†’ILMโ†’ๅคๅˆถโ†’ๅ‘Š่ญฆ๏ผ‰ | apply_actions ๅŒๅบ๏ผˆheal ๅ€™้€‰โ†’ILMโ†’ๅคๅˆถ healโ†’ๅ‘Š่ญฆ๏ผ‰ | โœ… | +| ILM 9 ๅŠจไฝœ + ๆ‰น้‡่ฏ„ไผฐ + DeletePrefixObject ไผ˜ๅŒ– | ๅŒ 9 ๅŠจไฝœ + ๆ‰น้‡่ฏ„ไผฐ + expiry ้˜Ÿๅˆ— | โœ…๏ผˆDeleteAllVersions ๆ˜ฏๅฆๅ•่ฐƒ็”จไผ˜ๅŒ–ๆœช้€่กŒๆ ธ๏ผ‰ | +| abandoned children๏ผˆlistPathRaw minDisks=N/2 ๅ‘็Žฐๆผๅ†™็›˜๏ผ‰ | list_path_raw + quorum ๆ ธๆŸฅ + ้ซ˜ไผ˜ heal | โœ… | +| incomplete multipart ็‹ฌ็ซ‹ไพ‹็จ‹๏ผˆ6h ้—ด้š”/24h ่ฟ‡ๆœŸ๏ผŒrename ่ฟ› .trash๏ผ‰ | ecstore ็‹ฌ็ซ‹ๅŽๅฐไปปๅŠก๏ผˆๅฏ้…้—ด้š”/่ฟ‡ๆœŸ๏ผ‰ | โœ…๏ผˆtrash ไบŒๆฎตๆธ…็†็ป†่Š‚ๅทฎๅผ‚๏ผŒยง6 HS-18๏ผ‰ | +| usage ็ปดๅบฆ๏ผˆsize/objects/versions/DM/็›ดๆ–นๅ›พ/ๅคๅˆถ/tier/bucket ็บง๏ผ‰ | ๅ…จ่ฆ†็›– + ้›†็พคๅฟซ็…งไธ‰้‡้˜ฒๅ›ž้€€ | โœ… ๅขžๅผบ | +| prefix ็บง usage๏ผˆloadPrefixUsageFromBackend๏ผŒconsole ๆถˆ่ดน๏ผ‰ | ็ผ“ๅญ˜ๅ†…ๆœ‰็›ฎๅฝ•ๆ ‘ไฝ†ไป… flatten ๆกถ็บง | โŒ ยง6 HS-08 | +| ่ถ…้™ไบ‹ไปถ s3:ObjectManyVersions/LargeVersions/PrefixManyFolders + ๅฎก่ฎก | ไป…ๆŒ‡ๆ ‡ alert_excess_*๏ผˆ้ป˜่ฎค 100/1TiB/65538 vs MinIO 100/1TB/50000๏ผ‰ | โš ๏ธ ยง6 HS-04/HS-17 | +| scanner ๆŒ‡ๆ ‡ v3๏ผˆbucket_scans/directories/objects/versions/last_activity๏ผ‰ | rustfs_scanner_* ๅ…จๅฅ— + freshness | โœ…๏ผˆๅ‘ฝๅไฝ“็ณปไธๅŒ๏ผ‰ | +| TraceScanner / realtime metrics๏ผˆmc admin scanner status/trace๏ผ‰ | ๆ—  trace ้€š้“๏ผ›/v3/scanner/status ่‡ชๆœ‰็ป“ๆž„ | โš ๏ธ ยง6 HS-03 | + +### 4.5 admin/CLI/API ้ขๅฏน็…ง + +| MinIO | RustFS | ็Šถๆ€ | +|---|---|---| +| `POST /minio/admin/v3/heal/...` start/status/cancel | `POST /rustfs/admin/v3/heal/...` ๅŒไธ‰ๆ€ | โœ…๏ผˆ่ทฏๅพ„ๅ‰็ผ€ไธๅŒๅฑž้ข„ๆœŸ๏ผ‰ | +| `HealStartSuccess`/`HealTaskStatus`/`HealResultItem`/DriveState | ๅŒๅๅญ—ๆฎต JSON ๅ…ผๅฎน | โœ… | +| `POST /v3/background-heal/status`๏ผˆBgHealState ่šๅˆ๏ผ‰ | ๅŒ่ทฏๅพ„ + degraded ่ฏญไน‰ + operations ็Ÿฉ้˜ต | โœ… ๅขžๅผบ๏ผˆMRF per-endpoint ๅญ็Šถๆ€ๆ— ๏ผŒๅ› ๆ—  MRF๏ผ‰ | +| `GET /v3/healthinfo` ๆฏ drive `HealInfo *HealingDisk` | ๆ— ๅŒๆฌพ healthinfo heal ๅญ—ๆฎต๏ผˆreplacement-recovery v4 ๆ‰ฟๆ‹…้ƒจๅˆ†๏ผ‰ | โš ๏ธ ยง6 HS-07 | +| madmin ๅฎขๆˆท็ซฏ HealStart/HealStatus/BackgroundHealStatus/ScannerStatus ๆ–นๆณ• | ไป… wire ็ฑปๅž‹๏ผŒๆ— ๅฎขๆˆท็ซฏๆ–นๆณ• | โŒ ยง6 HS-05 | +| mc admin heal --pool/--setใ€--scan-modeใ€--force-start/stop | HealOpts ๅ…จๅญ—ๆฎตๆ”ฏๆŒ๏ผˆpool/set/scanMode/forceStart/forceStop๏ผ‰ | โœ…๏ผˆๆœๅŠก็ซฏๅฐฑ็ปช๏ผ›็ผบ mc ไพงๅ…ฅๅฃ๏ผŒHS-05๏ผ‰ | +| ErrHealAlreadyRunning / ErrHealOverlappingPaths ็ฑปๅž‹ๅŒ–้”™่ฏฏ | ๅŽป้‡ๅˆๅนถ + ้ฉฑ้€่ฏญไน‰๏ผ›ๆ— ็ฑปๅž‹ๅŒ–้‡ๅ ๆ‹’็ป | โš ๏ธ ยง6 HS-06 | +| ็ป“ๆžœ backpressure๏ผˆmaxUnconsumedItems=1000ใ€10s ไฟๆดปๆตๅผใ€24h ๆœชๆถˆ่ดน abort๏ผ‰ | ๅฟซ็…งๅผๆŸฅ่ฏข๏ผˆ1024 ๆก + 8MiB ๆˆชๆ–ญ + 10min ไฟ็•™๏ผ‰ | โš ๏ธ ยง6 HS-06 | +| `mc support inspect`/healing-bin ็ฆป็บฟ dump | ๆ— ๏ผˆinspect.rs ๅญ˜ๅœจไฝ† healing dump ๆœช็กฎ่ฎค๏ผ‰ | โš ๏ธ P3 | + +### 4.6 ่ง‚ๆต‹้ขๅฏน็…ง + +| ็ปดๅบฆ | MinIO | RustFS | ็Šถๆ€ | +|---|---|---|---| +| heal ๆŒ‡ๆ ‡ | minio_heal_objects_total/heal_total/errors_total/time_last_activity + v3 drive_health 2=healing | rustfs_heal_* ๅ…จๅฅ—๏ผˆadmission/queue delay/running/throttle/page concurrency๏ผ‰ | โœ…๏ผˆRustFS ็ผบ drive_health=healing ๅ•ไธ€ gauge ็ญ‰ไปท็‰ฉ๏ผ›DiskInfo.healing ๅทฒ่ต‹ๅ€ผ๏ผ‰ | +| scanner ๆŒ‡ๆ ‡ | v3 6 ไธช + realtime 18 ้กน | rustfs_scanner_* ๅ…จๅฅ— + per-source ็ปดๅบฆ | โœ… | +| ILM ๆŒ‡ๆ ‡ | v3 5 ไธช๏ผˆexpiry/transition pending/active/missed + versions_scanned๏ผ‰ | ilm expiry status API + scanner per-source | โœ…๏ผˆๆŒ‡ๆ ‡ไธŽ API ๅฝขๆ€ไธๅŒ๏ผ‰ | +| trace | TraceHealing/TraceScanner ไธค้€š้“ | ๆ—  | โŒ ยง6 HS-03 | +| ๅฎก่ฎก | HealObject ไบ‹ไปถใ€dangling ๅˆ ้™คๅฎก่ฎกใ€scanner:manyversions ็ญ‰ | ็ป“ๆž„ๅŒ–ๆ—ฅๅฟ—๏ผˆevent style๏ผ‰+ ๆŒ‡ๆ ‡๏ผ›ๆ—  audit log ไบ‹ไปถ | โš ๏ธ ยง6 HS-04 | +| ่ฟ›ๅบฆ | healingTracker Bytes/Items/QueuedBuckets/ๅฝ“ๅ‰ๅฏน่ฑก + usage-cache ๆ€ป้‡ๅŸบ็บฟ | HealProgress{scanned/healed/failed/bytes/current_object/percentage}๏ผ›bytes_processed ๆณจ้‡Šไธบ 0ใ€estimated_completion_time ๆ’ None | โš ๏ธ ยง6 HS-07 | + +### 4.7 ้…็ฝฎ้ขๅฏน็…ง๏ผˆ้ป˜่ฎคๅ€ผ๏ผ‰ + +| MinIO | RustFS | ๅค‡ๆณจ | +|---|---|---| +| `heal:bitrotscan`๏ผˆ้ป˜่ฎค off๏ผ›on=ๆฏ่ฝฎ๏ผ›Nm=Nร—30ร—24h๏ผ‰ | `heal.bitrot_cycle` / `RUSTFS_SCANNER_BITROT_CYCLE_SECS`๏ผˆ้ป˜่ฎค 30d=2592000s๏ผ›0/on=ๆฏ่ฝฎ Deep๏ผŒoff=็ฆ็”จ๏ผ‰ | โœ… ๅŒ่ฏญไน‰๏ผˆRustFS ้ป˜่ฎค 30d๏ผŒMinIO ้ป˜่ฎค offโ€”โ€”**้ป˜่ฎคๅ€ผไธๅŒ**๏ผŒRustFS ๆ›ดๆฟ€่ฟ›๏ผ‰ | +| `heal:max_io=100`/`max_sleep=250ms`๏ผˆwaitForLowIO๏ผ‰ | mainline throttle ้˜ˆๅ€ผ 80%/80%ใ€max_sleep 250ms | โœ… ๅŒๅž‹๏ผˆ้˜ˆๅ€ผๆจกๅž‹ไธๅŒ๏ผ‰ | +| `heal:drive_workers`๏ผˆ้ป˜่ฎค -1 ่‡ชๅŠจ๏ผ‰ | ้กตๅ†…ๅนถๅ‘ 8 + per-set 1 | โœ… ๅŒๅž‹ | +| `_MINIO_HEAL_WORKERS`๏ผˆGOMAXPROCS/2๏ผ‰ | `RUSTFS_HEAL_MAX_CONCURRENT_HEALS=4` + `_MAX_CONCURRENT_PER_SET=1` | โœ… | +| `_MINIO_AUTO_DRIVE_HEALING`๏ผˆon๏ผ‰ | `RUSTFS_HEAL_AUTO_HEAL_ENABLE=true` | โœ… | +| `_MINIO_SCANNER`๏ผˆon๏ผ‰ | `RUSTFS_SCANNER_ENABLED=true` | โœ… | +| `scanner:speed` ไบ”ๆกฃ๏ผˆdefault=2x/1s/1m๏ผ‰ | ๅŒไบ”ๆกฃๅŒๅๅŒๅ‚ๆ•ฐ | โœ… | +| `scanner:idle_speed`๏ผˆon๏ผ‰ | `RUSTFS_SCANNER_IDLE_MODE`๏ผˆtrue๏ผ‰ | โš ๏ธ ่ฏญไน‰ๆ–นๅ‘๏ผˆHS-14๏ผ‰ | +| `scanner:alert_excess_versions=100` | 100 | โœ… | +| `scanner:alert_excess_folders=50000` | 65538๏ผˆๅ…ผๅฎน PBS ๅธƒๅฑ€๏ผ‰ | โš ๏ธ HS-17 | +| `ilm:expiration_workers=100`/`transition_workers=100` | ecstore expiry/transition worker ๆฑ ๏ผˆ้”ฎ่ง ilm ๅญ็ณป็ปŸ๏ผ‰ | โœ…๏ผˆ้ป˜่ฎคๅ€ผๆœช้€้กนๆ ธๅฏน๏ผ‰ | +| `api:stale_upload_cleanup_interval=6h`/`expiry=24h` | ecstore ๅŽๅฐไปปๅŠก env ๅฏ้… | โœ…๏ผˆ้ป˜่ฎคๅ€ผๆœช้€้กนๆ ธๅฏน๏ผ‰ | +| โ€”๏ผˆๆ— ๏ผ‰ | `RUSTFS_HEAL_QUEUE_SIZE=10000`ใ€`_TASK_TIMEOUT_SECS=300`ใ€`_INTERVAL_SECS=10`ใ€`_LOW_PRIORITY_MERGE/DROP`ใ€`_PAGE_*`ใ€`_SET_BULKHEAD`ใ€`_MAINLINE_*`ใ€`RUSTFS_SCANNER_CYCLE_MAX_*` ้ข„็ฎ—ใ€`_MAX_CONCURRENT_SET/DISK_SCANS=4`ใ€`_YIELD_EVERY_N_OBJECTS=128` ็ญ‰ | RustFS ็‰นๆœ‰๏ผˆๆ›ด็ป†็ฒ’ๅบฆ๏ผ‰ | + +### 4.8 RustFS ่ถ…ๅ‡บ MinIO ็š„้ƒจๅˆ† + +1. remote_scanner RPC๏ผˆๆ‰ซๆๆ‰ง่กŒไธ‹ๆ”พ่ฟœ็ซฏ peer ๆœฌๅœฐ๏ผŒๅซ HMAC ่ฎค่ฏ/้‡ๆ”พ็ผ“ๅญ˜/fence ๅค้ชŒ/ๆ–ญ่ฟžๅฎฝ้™๏ผ‰ใ€‚ +2. ๆŒไน…ๅŒ– leader-epoch CAS ๅ›ดๆ  + usage ๅฟซ็…ง epoch/cycle ้˜ฒๅ›ž้€€๏ผˆMinIO ไป…้”๏ผŒๆ— ๆŒไน… epoch๏ผ‰ใ€‚ +3. ๅ‘จๆœŸ้ข„็ฎ—๏ผˆmax_duration/objects/directories๏ผ‰+ partial ๅ‘จๆœŸๆŽจ่ฟ›่ฏญไน‰ใ€‚ +4. per-set/per-disk ๆ‰ซๆๅนถๅ‘้—ธ + ๆฏๆกถๆฏ set ็ผ“ๅญ˜้”ใ€‚ +5. pending-heal ่ดฆๆœฌ๏ผˆheal ้€š้“ๆปกไธไธขๅ€™้€‰๏ผ‰ใ€‚ +6. ๆข็›˜ durable intent + completion proof ็Šถๆ€ๆœบ + ่บซไปฝๅ›ดๆ ๏ผˆMinIO healingTracker ๆ—  proof๏ผ‰ใ€‚ +7. mainline throttle ๅ‰ๅฐๅŽ‹ๅŠ›้—จๆŽง๏ผˆpermit ๅˆฉ็”จ็Ž‡้ฉฑๅŠจ๏ผ‰ใ€‚ +8. ้›†็พค heal control coordinator + envelope ้‡ๆ”พ้˜ฒๆŠค + degraded ๆ˜พๅผ้™็บงใ€‚ +9. ๅ†™่ทฏๅพ„ shard bitrot ่‡ชๆ ก้ชŒ๏ผˆEC:0 ๅœบๆ™ฏ๏ผ‰ใ€‚ +10. dirty-usage ๅฟซ่ทฏๅพ„ๅ”ค้†’๏ผˆๅ†™่ทฏๅพ„ๅณๆ—ถ้€š็Ÿฅ + ่„ๆกถไผ˜ๅ…ˆ๏ผ‰ใ€‚ +11. heal ่ฟ่กŒๆ—ถๅฏ่ง‚ๆต‹็Ÿฉ้˜ต๏ผˆไผ˜ๅ…ˆ็บงร—ๆฅๆบ operations snapshot๏ผ‰ใ€‚ +12. workload admission ่”ๅŠจ๏ผˆheal ่ฐƒๅบฆๅ™จ่ฏปๅ‰ๅฐๅŽ‹ๅŠ›ๅฟซ็…ง๏ผ‰ใ€‚ + +--- + +## 5. ๅทฎ่ทไธŽๆ”น่ฟ›ๆธ…ๅ• + +ๅˆ†็บงๅฎšไน‰๏ผšP1=่กŒไธบ/่ฟ็ปดๅฏน้ฝ็ผบๅฃ๏ผˆๅฝฑๅ“็”Ÿไบง่ฟ็ปดๆˆ–ๅทฅๅ…ท้“พๅ…ผๅฎน๏ผ‰๏ผ›P2=ๅฎŒๅ–„ๆ€ง๏ผˆๅŠŸ่ƒฝๅœจไฝ†็ผบไธ€่ง’๏ผ‰๏ผ›P3=ๆธ…็†/ไฝŽ้ฃŽ้™ฉใ€‚ๆฏ้กนๅซ็Žฐ็Šถ่ฏๆฎใ€MinIO ่กŒไธบใ€ๅฝฑๅ“ใ€ๅปบ่ฎฎใ€้ชŒๆ”ถๆ–นๅผใ€‚ + +### P1๏ผˆ8 ้กน๏ผ‰ + +**HS-01 MRF/ECDecode/Metadata ไธ‰็ฑป heal ไปปๅŠกๆ— ็”Ÿไบง่งฆๅ‘ๅ…ฅๅฃ๏ผŒHealEvent ๆœชๆŽฅ็บฟ** +- ็Žฐ็Šถ๏ผš`HealType::MRF/ECDecode/Metadata` ๆ‰ง่กŒไฝ“ๅฎŒๆ•ด๏ผˆtask.rs:1700-2156๏ผ‰ไฝ†ๅ…จไป“ๅบ“ๆ— ็”Ÿไบง่งฆๅ‘ๆ–น๏ผ›`HealEvent`/`HealEventHandler`๏ผˆevent.rs:50-367๏ผ‰crate ๅค–้›ถๅผ•็”จ๏ผˆๅทฒไบฒ้ชŒ grep๏ผ‰๏ผ›channel ่ฝฌๆขๅชไบง็”Ÿ Cluster/Object/Bucket/Prefix/ErasureSet๏ผˆchannel.rs:566-601๏ผ‰ใ€‚ +- MinIO๏ผšmrf.go ็‹ฌ็ซ‹ MRF ้˜Ÿๅˆ—๏ผˆๅฎน้‡ 100k๏ผŒๆปกไธขๅผƒ่ฎกๆ•ฐ๏ผ‰ใ€่ฟ›็จ‹้€€ๅ‡บ msgp ๆŒไน…ๅŒ– `.heal/mrf/list.bin` + ๅฏๅŠจๅ›žๆ”พใ€ๅ…ฅ้˜Ÿ <1s ๅปถ่ฟŸ 1s๏ผˆ็ญ‰็ฝ‘็ปœๆขๅค๏ผ‰ใ€healSleeper ้™้€Ÿ๏ผ›่ฏป่ทฏๅพ„ GetObject part ็ผบๅคฑ/ๆŸๅใ€ๅ…ƒๆ•ฐๆฎ้‡ๅปบ missingBlocks>0ใ€Put ้ƒจๅˆ†ๆˆๅŠŸใ€DeleteObjectใ€multipartใ€peer client ๅ…ฑ 7+ ๆŠ•้€’็‚นใ€‚ +- ๅฝฑๅ“๏ผšRustFS ็š„ read-repair + ๅ†™่ทฏๅพ„ๆ”ถๆ•›่ฆ†็›–ไบ†ไธปๅœบๆ™ฏ๏ผŒไฝ†็ผบๅฐ‘๏ผšโ‘  ไบ‹ไปถ้ฉฑๅŠจ็š„ Urgent ECDecode ้‡ๅปบๅ…ฅๅฃ๏ผˆecstore ่งฃ็ ๅคฑ่ดฅๆ—ถ็›ฎๅ‰ไป… Low read-repair๏ผ‰๏ผ›โ‘ก Metadata-only heal ๅ…ฅๅฃ๏ผˆscanner HealMetadata ๅˆ†็ฑปๅญ˜ๅœจไฝ†่ตฐๆ™ฎ้€šๅฏน่ฑก heal๏ผ‰๏ผ›โ‘ข MRF ้˜Ÿๅˆ—ๆŒไน…ๅŒ–๏ผˆ้‡ๅฏไธขๆœชๆถˆ่ดนไฟฎๅคๆ„ๅ›พโ€”โ€”scanner pending-heal ่ดฆๆœฌ้ƒจๅˆ†็ผ“่งฃ๏ผ‰ใ€‚ +- ๅปบ่ฎฎ๏ผšไธ‰้€‰ไธ€ๅ†ณ็ญ–โ€”โ€”(a) ๆŽฅ็บฟ HealEvent๏ผˆๅœจ ecstore ่งฃ็ ๅคฑ่ดฅ/metadata ๆŸๅ็‚นๅ‘ไบ‹ไปถ๏ผ‰+ ๅฎž็ŽฐๆŒไน…ๅŒ–้‡่ฏ•่ดฆๆœฌ๏ผ›(b) ๅˆ ้™ค MRF/ECDecode/Metadata ๆญปไปฃ็ ๅชไฟ็•™ๆ–‡ๆกฃ่ฏดๆ˜Ž๏ผ›(c) ไฟ็•™ๆ‰ง่กŒไฝ“ใ€ๆŠŠ HealEvent ้™็บงไธบๅ†…้ƒจ APIใ€‚ๆŽจ่ (a) ไฝ†้œ€ๅ…ˆ้‡ๅŒ– read-repair ๆ˜ฏๅฆๅทฒ่ฆ†็›–่งฃ็ ๅคฑ่ดฅๅœบๆ™ฏ็š„ๅ“ๅบ”ๆ—ถ้—ด่ฆๆฑ‚ใ€‚ +- ้ชŒๆ”ถ๏ผš่งฃ็ ๅคฑ่ดฅ โ†’ Urgent heal ่ฏทๆฑ‚้“พ่ทฏ e2e๏ผ›้‡ๅฏๅŽ pending ไฟฎๅคๆ„ๅ›พๅ›žๆ”พ๏ผ›HealEvent ็Žฏๅฝข็ผ“ๅ†ฒๆŒ‡ๆ ‡ใ€‚ + +**HS-02 CheckAbandonedParts ไธ‰ๅฑ‚ NotImplemented๏ผˆabandoned data ็‹ฌ็ซ‹ๅฏน่ดฆๅ…ฅๅฃ็ผบๅคฑ๏ผ‰** +- ็Žฐ็Šถ๏ผš`set_disk/ops/heal.rs:2052-2056`ใ€`core/sets.rs:1144-1148`ใ€`store/heal.rs:258-266` ไธ‰ๅฑ‚ๆ˜พๅผ `Err(NotImplemented)`๏ผˆๅทฒไบฒ้ชŒ๏ผ‰๏ผŒๆณจ้‡Š"intentionally retained above the set layer until there is a concrete caller"ใ€‚ +- MinIO๏ผš`CheckAbandonedParts` โ†’ ๆฏ็›˜ `CleanAbandonedData`๏ผš่ฏป xl.meta โ†’ ๅˆ— UUID data-dir + inline entries โ†’ ไธŽ getDataDirs ๅทฎ้›† โ†’ ๅˆ ๅคšไฝ™ data-dir/inline ๅนถ้‡ๅ†™ xl.meta๏ผ›็”ฑ scanner ๆŠฝไธญ heal ไธŽ admin heal Remove ๆ—ถๆ˜พๅผ่ฐƒ็”จใ€‚ +- ๅฝฑๅ“๏ผšRustFS heal ่ทฏๅพ„ๅ†… `reclaim_orphan_data_dirs_best_effort`๏ผˆ:1428๏ผ‰่ฆ†็›–"heal ๆ—ถๅ›žๆ”ถๅญคๅ„ฟ็›ฎๅฝ•"๏ผŒไฝ† โ‘  ๆ— ็‹ฌ็ซ‹่งฆๅ‘็‚น๏ผˆMinIO ๅœจๅฏน่ฑกๆœชๅˆฐ heal ้˜ˆๅ€ผๆ—ถไนŸ่ƒฝๆธ… abandoned data๏ผ‰๏ผ›โ‘ก inline data ๅญคๅ„ฟๆก็›ฎๆธ…็†ๆœช็กฎ่ฎค๏ผ›โ‘ข multipart ๅญคๅ„ฟๅฏน่ดฆๆ˜Ž็กฎไธๅš๏ผˆ่ฎพ่ฎกๅ†ณๅฎš๏ผŒ็”ฑ lifecycle ๆ‰ฟๆ‹…๏ผ‰ใ€‚ +- ๅปบ่ฎฎ๏ผš่ฏ„ไผฐๆŠŠ `reclaim_orphan_data_dirs_best_effort` ๆๅ‡ไธบ heal_object ๅ›บๅฎšๆญฅ้ชค๏ผˆ่‹ฅๅฐš้ž๏ผ‰+ ๅฎž็Žฐ HealOperations::check_abandoned_parts ็œŸๅฎž็Žฐ๏ผˆ่ฐƒ็”จๅŒไธ€ๅ›žๆ”ถ้€ป่พ‘๏ผ‰๏ผŒๆˆ–ๆ˜Ž็กฎๆ–‡ๆกฃๅŒ–"็”ฑ lifecycle ๆ‰ฟๆ‹…"ๅนถๅ…ณ้—ญ API ้ขใ€‚ +- ้ชŒๆ”ถ๏ผšๆž„้€  data-dir/inline ๅญคๅ„ฟ โ†’ scanner ๆŠฝๆ ท/admin heal ๅŽ่ขซๆธ…็†๏ผ›ไธ‰ๅฑ‚ API ่ฟ”ๅ›žๆˆๅŠŸๆˆ–ๆ˜พๅผ NotSupported ๆ–‡ๆกฃๅŒ–ใ€‚ + +**HS-03 heal/scanner trace ้€š้“็ผบๅคฑ** +- ็Žฐ็Šถ๏ผšTraceHealing/TraceScanner ้›ถๅ‘ฝไธญ๏ผˆๅทฒไบฒ้ชŒ grep ๅ…จไป“ๅบ“๏ผ‰ใ€‚ +- MinIO๏ผš`madmin.TraceHealing`๏ผˆmc admin trace --healing๏ผŒFuncName=heal.Bucket/heal.Object/heal.CheckAbandonedParts๏ผŒๅธฆ dry/remove/mode/version-id/disks/bytes๏ผ‰ใ€`TraceScanner`๏ผˆmc admin scanner trace๏ผŒๆ”ฏๆŒ --filter-size/--response-duration๏ผ‰ใ€‚ +- ๅฝฑๅ“๏ผšๆ— ๆณ•ๅฎžๆ—ถ่ง‚ๆต‹ๅ•ไธช heal/scanner ๅŠจไฝœ็š„่€—ๆ—ถไธŽๅ‚ๆ•ฐ๏ผ›ๆŽ’้šœๅช่ƒฝ้ ๆŒ‡ๆ ‡่šๅˆไธŽๆ—ฅๅฟ—ใ€‚ +- ๅปบ่ฎฎ๏ผšๅœจ heal channel ๆ‰ง่กŒไธŽ scanner folder/item ๅค„็†ๅŸ‹็‚น๏ผŒๆŽฅๅ…ฅ็Žฐๆœ‰ admin trace ่ฎข้˜…้ข๏ผˆ่‹ฅ rustfs ๅทฒๆœ‰ trace ๅŸบๅปบๅˆ™ๅค็”จ๏ผŒๆ— ๅˆ™ๆŒ‰ madmin TraceType ๆ‰ฉๅฑ•๏ผ‰ใ€‚ +- ้ชŒๆ”ถ๏ผšmc ็ญ‰ไปทๅทฅๅ…ท่ƒฝ่ฎข้˜… heal/scanner trace ๆตใ€‚ + +**HS-04 scanner ่ถ…้™ S3 ไบ‹ไปถไธŽๅฎก่ฎก็ผบๅคฑ** +- ็Žฐ็Šถ๏ผšไป… `rustfs_scanner_excess_*_total` ๆŒ‡ๆ ‡๏ผˆversions 100/version size 1TiB/folders 65538๏ผ‰ใ€‚ +- MinIO๏ผšๅ‘ `s3:ObjectManyVersions`๏ผˆ>100 ็‰ˆๆœฌ๏ผ‰ใ€`s3:ObjectLargeVersions`๏ผˆ็ดฏ่ฎก >1TB๏ผ‰ใ€`s3:PrefixManyFolders`๏ผˆ>50000 ๅญ็›ฎๅฝ•๏ผ‰ไบ‹ไปถ๏ผˆUserAgent: Scanner๏ผ‰+ scanner:manyversions/largeversions/manyprefixes ๅฎก่ฎกใ€‚ +- ๅฝฑๅ“๏ผšไพ่ต–ไบ‹ไปถ่ฎข้˜…ๅšๅฎน้‡ๆฒป็†็š„็”จๆˆท๏ผˆconsole/ๅค–้ƒจๅฎก่ฎก๏ผ‰ๆ”ถไธๅˆฐๅ‘Š่ญฆใ€‚ +- ๅปบ่ฎฎ๏ผšscanner_folder ๅ‘Š่ญฆ็‚นๆŽฅๅ…ฅ notify ไบ‹ไปถๅ‘ๅธƒ๏ผˆๅค็”จ lifecycle ไบ‹ไปถ้€š้“่ฏญไน‰๏ผ‰ใ€‚ +- ้ชŒๆ”ถ๏ผš้…็ฝฎๆกถ้€š็ŸฅๅŽ่ถ…้™ๅฏน่ฑก่งฆๅ‘ไบ‹ไปถใ€‚ + +**HS-05 madmin ๅฎขๆˆท็ซฏๆ–นๆณ•็ผบๅคฑ** +- ็Žฐ็Šถ๏ผš`crates/madmin/src/heal_commands.rs` ๅชๆœ‰ wire ็ฑปๅž‹๏ผˆHealDriveInfo/Infos/HealResultItem๏ผ‰๏ผ›ๆ—  HealStart/HealStatus/BackgroundHealStatus/ScannerStatus ๅฎขๆˆท็ซฏๆ–นๆณ•ใ€‚ +- MinIO๏ผšmadmin-go ๆไพ›ๅฎŒๆ•ดๅฎขๆˆท็ซฏ๏ผ›mc admin heal/scanner/status/trace ้ƒฝๅปบ็ซ‹ๅœจไธŠ้ขใ€‚ +- ๅฝฑๅ“๏ผšmc ็ญ‰็ฎก็†ๅทฅๅ…ทๆ— ๆณ•็›ดๆŽฅๅฏนๆŽฅ RustFS heal/scanner ็ฎก็†้ข๏ผ›่‡ชๅŠจๅŒ–่ฟ็ปดๅช่ƒฝๆ‰‹ๅ†™ HTTPใ€‚ +- ๅปบ่ฎฎ๏ผšๆŒ‰ madmin-go ๆŽฅๅฃๅฝข็Šถ่กฅๅฎขๆˆท็ซฏ๏ผˆๆœๅŠก็ซฏๅทฒๅฐฑ็ปช๏ผŒ็บฏๅฎขๆˆท็ซฏๅทฅไฝœ๏ผ‰ใ€‚ +- ้ชŒๆ”ถ๏ผš็”จ madmin ๅฎขๆˆท็ซฏๅฎŒๆˆ startโ†’queryโ†’cancel ๅ…จๆต็จ‹ใ€‚ + +**HS-06 admin heal ๅบๅˆ—่ฏญไน‰ไธŽ MinIO ๅทฎๅผ‚** +- ็Žฐ็Šถ๏ผš้‡ๅค/้‡ๅ ่ฏทๆฑ‚่ขซๅŽป้‡ๅˆๅนถ๏ผˆ่ฟ”ๅ›ž canonical task_id๏ผ‰ๆˆ–้ฉฑ้€๏ผ›ๆ—  ErrHealAlreadyRunning/ErrHealOverlappingPaths ็ฑปๅž‹ๅŒ–้”™่ฏฏ๏ผˆๅทฒไบฒ้ชŒ๏ผšmanager.rs:1309 ็š„ already_running ๆ˜ฏๅน‚็ญ‰ๅฏๅŠจไฟๆŠค๏ผŒ้ž admin ่ฏญไน‰๏ผ‰๏ผ›็ป“ๆžœไธบๅฟซ็…งๅผๆŸฅ่ฏข๏ผˆ1024 ๆก/8MiB ๆˆชๆ–ญ/10min ไฟ็•™๏ผ‰๏ผŒ้ž MinIO ็š„ๆตๅผๅขž้‡๏ผˆclientToken ๆ‹‰ๅขž้‡ + maxUnconsumedItems=1000 backpressure + 10s ไฟๆดป + 24h ๆœชๆถˆ่ดน abort๏ผ‰ใ€‚ +- ๅฝฑๅ“๏ผšmc admin heal ็š„ไบคไบ’ๆจกๅž‹๏ผˆ้•ฟ่ฟžๆŽฅๆ‹‰ๅขž้‡๏ผ‰ๅฏน RustFS ่กจ็Žฐไธบๅคšๆฌกๅฟซ็…ง่ฝฎ่ฏข๏ผ›่‡ชๅŠจๅŒ–่„šๆœฌ้šพไปฅๅŒบๅˆ†"ๅทฒๅˆๅนถ"ไธŽ"ๆ–ฐๅฏๅŠจ"ใ€‚ +- ๅปบ่ฎฎ๏ผšโ‘  ๅขž้‡่ฏญไน‰๏ผšchannel query ๆ”ฏๆŒ่‡ชไธŠๆฌก clientToken ่ตท็š„ items ๅขž้‡๏ผˆๆˆ– cursor๏ผ‰๏ผ›โ‘ก ้‡ๅ ่ฏทๆฑ‚่ฟ”ๅ›ž็ฑปๅž‹ๅŒ–้”™่ฏฏ็ ๏ผˆๆˆ– receipt ไธญๆ˜พๅผ merged_into ๅญ—ๆฎตโ€”โ€”็Žฐๆœ‰ alias ๆœบๅˆถๅทฒๆœ‰ๅŸบ็ก€๏ผ‰๏ผ›โ‘ข forceStart ๅ…ˆๅœๆ—งๅ†ๅฏๆ–ฐ่ฏญไน‰ๆ ธๅฏนใ€‚ +- ้ชŒๆ”ถ๏ผšmadmin ๅ…ผๅฎนๅฎขๆˆท็ซฏๆŒ‰ MinIO ๆจกๅผ่ฝฎ่ฏข่ƒฝๅ–ๅพ—ๅ…จ้‡ itemsใ€‚ + +**HS-07 healing ่ฟ›ๅบฆไธŽ็›˜็บง healing ็Šถๆ€ๅฏนๅค–ๅฏ่งๆ€งไธ่ถณ** +- ็Žฐ็Šถ๏ผšbytes ๆขๅค่ฟ›ๅบฆ `progress.bytes_processed = 0 // set to 0 for now`๏ผˆerasure_healer.rs:967๏ผ‰๏ผ›`HealProgress::estimated_completion_time` ๆ’ Noneใ€`HealStatistics::add_healed_objects` ๆœชๅ†™ๅ…ฅ๏ผˆprogress.rs:38,135-139 ้›ถ่ฐƒ็”จ๏ผ‰๏ผ›healthinfo ๆ— ๆฏ็›˜ HealInfo ็ญ‰ไปท๏ผˆMinIO HealingDisk๏ผšBytesDone/Failed/Skippedใ€ObjectsTotal ๅŸบ็บฟใ€QueuedBuckets/HealedBucketsใ€Resume ๅฟซ็…งใ€ๅฝ“ๅ‰ object๏ผ‰๏ผ›v3 ๆŒ‡ๆ ‡ๆ—  drive_health=2(healing) ๅ•ไธ€ gauge ็ญ‰ไปทใ€‚ +- ๅฝฑๅ“๏ผšๆข็›˜้‡ๅปบ๏ผˆๅฏ่ƒฝๆ•ฐๅฐๆ—ถ~ๅคฉ๏ผ‰ๆœŸ้—ด่ฟ็ปดๆ— ๆณ•ๅ›ž็ญ”"่ฟ›่กŒๅˆฐๅ“ช/่ฟ˜ๅ‰ฉๅคšๅฐ‘/้ข„่ฎกไฝ•ๆ—ถๅฎŒๆˆ"ใ€‚ +- ๅปบ่ฎฎ๏ผšโ‘  erasure set heal ็ปŸ่ฎก bytes๏ผˆheal_object ่ฟ”ๅ›žๅฏน่ฑกๅคงๅฐๅทฒๅฏๅพ—๏ผ‰๏ผ›โ‘ก ไปŽ usage-cache ่ฏปๅฏน่ฑกๆ€ป้‡ๅŸบ็บฟ๏ผˆMinIO ๅŒๆฌพๅšๆณ•๏ผ‰๏ผ›โ‘ข admin healthinfo/่ƒŒๆ™ฏ็Šถๆ€ๆšด้œฒๆฏ็›˜ healing ๅฟซ็…ง๏ผˆDiskInfo.healing ๅทฒๆœ‰๏ผŒ่กฅ่šๅˆๆšด้œฒ๏ผ‰๏ผ›โ‘ฃ ETA ็”ฑๅŸบ็บฟ+้€Ÿ็Ž‡ๆŽจๅฏผใ€‚ +- ้ชŒๆ”ถ๏ผšๆข็›˜้‡ๅปบไธญ admin ๅฏ่ง bytes ่ฟ›ๅบฆไธŽ ETA๏ผ›mc info ็ญ‰ไปท่พ“ๅ‡บ Healing ๆ ‡ๅฟ—ใ€‚ + +**HS-08 prefix ็บง usage ๆœชๆšด้œฒ** +- ็Žฐ็Šถ๏ผšDataUsageCache ๅ†…็›ฎๅฝ•ๆ ‘ entry ๅญ˜ๅœจ๏ผˆhash_path ็ป„็ป‡๏ผ‰๏ผŒไฝ† `dui()` ๅช flatten ๅˆฐๆกถๅ๏ผˆdata_usage_define.rs:858-915๏ผ‰ใ€‚ +- MinIO๏ผš`loadPrefixUsageFromBackend`๏ผˆ30s cache๏ผ‰ไปŽๆฏ set `.usage-cache.bin` ่šๅˆ prefix usage๏ผŒconsole ๆกถๅ‰็ผ€็ปŸ่ฎกๆถˆ่ดนใ€‚ +- ๅฝฑๅ“๏ผšconsole/ๅ‰็ซฏๆ— ๆณ•ๅฑ•็คบๅ‰็ผ€็บง็”จ้‡๏ผ›ๅคงๆกถๅฎšไฝ"ๅ“ชไธชๅ‰็ผ€ๅ ็ฉบ้—ด"ๆ—  APIใ€‚ +- ๅปบ่ฎฎ๏ผšๅฎž็Žฐ flatten ๅ‰็ผ€ๆŸฅ่ฏข API๏ผˆๆ•ฐๆฎๅทฒๅœจ็ผ“ๅญ˜ๅ†…๏ผŒ็บฏ่šๅˆไธŽๆšด้œฒๅทฅไฝœ๏ผ‰ใ€‚ +- ้ชŒๆ”ถ๏ผšListBuckets/PrefixUsage API ่ฟ”ๅ›žไธŽๅ‰็ผ€่ฟ‡ๆปคๅŒน้…็š„็ปŸ่ฎกใ€‚ + +### P2๏ผˆ9 ้กน๏ผ‰ + +**HS-09 get_disk_status ๆ’่ฟ”ๅ›ž Ok๏ผˆๅ”ฏไธ€ TODO๏ผ‰**๏ผš`crates/heal/src/heal/storage.rs:930-943`๏ผˆๅทฒไบฒ้ชŒ๏ผ‰ใ€‚ๅฝ“ๅ‰ๆ— ็”Ÿไบง่ฐƒ็”จๆ–น๏ผˆไฝŽ้ฃŽ้™ฉ๏ผ‰ใ€‚ๅปบ่ฎฎ๏ผšๅˆ ้™ค่ฏฅๆ–นๆณ•ๆˆ–ๆŽฅ ecstore disk ็Šถๆ€็œŸๅฎž็Žฐ๏ผˆDiskStatus ๆžšไธพๅทฒๅฎšไน‰๏ผ‰ใ€‚ + +**HS-10 HealStorageAPI ็บฆ 1/3 ๆ–นๆณ•ไธบๆญปไปฃ็ **๏ผšget_object_meta/get_object_data/put_object_data/delete_object/verify_object_integrity/ec_decode_rebuild/get_disk_status/format_disk/heal_bucket_metadata/get_object_size/get_object_checksum/list_objects_for_heal๏ผˆ้žๅˆ†้กต็‰ˆ๏ผŒ่‡ชๅธฆ memory_heavy ่ญฆๅ‘Š๏ผ‰ๅ‡ 0 ่ฐƒ็”จๆ–นใ€‚ๅปบ่ฎฎ๏ผš้š HS-01 ๅ†ณ็ญ–ไธ€ๅนถๆธ…็†ๆˆ–ๆŽฅ็บฟ๏ผˆๆญปๆŽฅๅฃ่ฏฏๅฏผๅŽ็ปญ็ปดๆŠค่€…ไปฅไธบๅญ˜ๅœจ่ฐƒ็”จ่ทฏๅพ„๏ผ‰ใ€‚ + +**HS-11 bitrot ่‡ชๆฃ€็ผบๅคฑ**๏ผšMinIO ๅฏๅŠจๆ—ถ bitrotSelfTest ๅฏนๅ››็ฎ—ๆณ•ๅทฒ็Ÿฅๅ‘้‡่‡ชๆฃ€ๅคฑ่ดฅๅณ Fatal๏ผˆ้˜ฒ้™้ป˜ๆ•ฐๆฎๆŸๅ๏ผ‰ใ€‚RustFS ๆ— ็ญ‰ไปท๏ผˆๅทฒไบฒ้ชŒ grep๏ผ‰ใ€‚ๅปบ่ฎฎ๏ผšๅฏๅŠจๆ—ถๅฏน HighwayHash256S ็ญ‰ๅœจ็”จ็ฎ—ๆณ•ๅšๅทฒ็Ÿฅๅ‘้‡่‡ชๆฃ€๏ผˆไฝŽๆˆๆœฌ้ซ˜ไปทๅ€ผ๏ผ‰ใ€‚ + +**HS-12 ๅฏน่ฑก็บง healing ๅ…ƒๆ•ฐๆฎๆ ‡่ฎฐ่ฏ„ไผฐ**๏ผšMinIO heal ๆœŸ้—ดๅฏน่ฑกๆ‰“ `x-minio-healing:true`๏ผŒRenameData ๆฎๆญค่ทณ่ฟ‡็‰ˆๆœฌๆธ…็†/legacy purge๏ผˆๆผๆŽ‰ไผšๅฏผ่‡ด heal ไธŽๅนถๅ‘ๅˆ ้™คไบ’ๆฏ๏ผ‰ใ€‚RustFS ๆ— ๅฏน่ฑก็บงๆ ‡่ฎฐ๏ผˆๅทฒไบฒ้ชŒ grep object.rs ๆ—  healing ๅˆ†ๆ”ฏ๏ผ‰๏ผŒไพ่ต– NSLock + rename ่ฏญไน‰ใ€‚ๅปบ่ฎฎ๏ผšๅฎก่ฎก RustFS rename ๆไบค่ทฏๅพ„ๆ˜ฏๅฆๅญ˜ๅœจ"heal ๆไบคไธŽๅนถๅ‘ delete/version ๆธ…็†็ซžไบ‰"็ช—ๅฃ๏ผ›่‹ฅๆ— ๅˆ™ๆ–‡ๆกฃๅŒ–ๅทฎๅผ‚๏ผŒ่‹ฅๆœ‰ๅˆ™่กฅๆ ‡่ฎฐ็ญ‰ไปทๆœบๅˆถใ€‚ + +**HS-13 erasure set heal ๆ— "่ทณ่ฟ‡ๆ–ฐๅ†™ๅ…ฅ/ILM ๅทฒ่ฟ‡ๆœŸ็‰ˆๆœฌ"่ฟ‡ๆปค**๏ผšMinIO resync ่ทณ่ฟ‡ ModTime>tracker.Started ็š„็‰ˆๆœฌ๏ผˆ้ฟๅ… heal ่ฟฝๆ–ฐๅ†™ๅ…ฅๅฐพๅทด๏ผ‰ไธŽ ILM ๅทฒ่ฟ‡ๆœŸ็‰ˆๆœฌ๏ผˆ้ฟๅ…็™ฝๅš๏ผ‰ใ€‚RustFS erasure_healer ๆœชๅฎž็ŽฐๅŒๆฌพ่ฟ‡ๆปค๏ผˆๆŒ‰็‰ˆๆœฌ dedup ๆœ‰๏ผŒๆ—ถ้—ด/ILM ่ฟ‡ๆปคๆ— ๏ผ‰ใ€‚ๅฝฑๅ“๏ผš้‡ๅปบๅฐพ้ƒจ้•ฟๅฐพ๏ผˆๆŒ็ปญๅ†™ๅ…ฅ็š„ๆกถ heal ๅฎŒๆˆๅˆคๅฎš่ขซๆ–ฐ็‰ˆๆœฌๆŽจ่ฟŸ๏ผ‰ไธŽๆ— ๆ•ˆ heal ๅทฅไฝœ้‡ใ€‚ๅปบ่ฎฎ๏ผšdisk-walk ๆžšไธพๅค„ๅŠ  started_at ๆ—ถ้—ด่ฟ‡ๆปค + evaluator ้ข„ๆฃ€ใ€‚ + +**HS-14 scanner idle ่ฏญไน‰ๆ–นๅ‘ไธŽ MinIO ็›ธๅ**๏ผšMinIO `scanner:idle_speed=on`๏ผˆ้ป˜่ฎค๏ผ‰= ้›†็พค็ฉบ้—ฒๆ—ถๆ‰่Š‚ๆตใ€ๅฟ™ๆ—ถๅ…จ้€Ÿ๏ผ›RustFS `RUSTFS_SCANNER_IDLE_MODE=true`๏ผˆ้ป˜่ฎค๏ผ‰= ้™้€Ÿๆ€ป้—ธ๏ผˆfalse=ๅฎŒๅ…จไธไผ‘็œ ๏ผ‰ใ€‚ไธค่€…้ป˜่ฎค่กŒไธบๅฏ่ƒฝ็›ธ่ฟ‘๏ผˆ้ƒฝ้™้€Ÿ๏ผ‰ไฝ†ๅ‚ๆ•ฐ่ฏญไน‰ไธๅฏไบ’ๆข๏ผŒ่ฟ็งปๆ–‡ๆกฃ้œ€ๆ˜พๅผ่ฏดๆ˜Ž๏ผ›่‹ฅ่ฟฝๆฑ‚ mc config ๅ…ผๅฎน้œ€้‡ๅ‘ฝๅ/้‡่ฏญไน‰ใ€‚ๅปบ่ฎฎ๏ผšๅ…ˆๆ–‡ๆกฃๅŒ–ๅทฎๅผ‚๏ผŒ่ฏ„ไผฐๆ˜ฏๅฆๅฏน้ฝ่ฏญไน‰ใ€‚ + +**HS-15 alert_excess_folders ้ป˜่ฎคๅ€ผๅทฎๅผ‚**๏ผšRustFS 65538๏ผˆๅ…ผๅฎน PBS/Proxmox ๅธƒๅฑ€๏ผŒscanner_folder.rs:79๏ผ‰vs MinIO 50000ใ€‚่กŒไธบๅทฎๅผ‚้ป˜่ฎคๅณ่งฆๅ‘้˜ˆๅ€ผไธๅŒใ€‚ๅปบ่ฎฎ๏ผšๆ–‡ๆกฃๅŒ–๏ผˆไฟ็•™ 65538 ๆœ‰ๆœฌๅœฐ็†็”ฑ๏ผ‰ใ€‚ + +**HS-16 ๅ•ๆœบ้ป˜่ฎคๅ‘จๆœŸ้’ฉๅญๆœชๅฏ็”จ**๏ผš`single_disk_default_cycle_secs(_features) -> None` ๆ’็ฉบ๏ผˆscanner.rs:1428-1430๏ผ‰๏ผŒๅ•ๆœบ้ƒจ็ฝฒๆ— ไธ“ๅฑž้ป˜่ฎคๅ‘จๆœŸ่ฆ†็›–ใ€‚ๅปบ่ฎฎ๏ผšๅ†ณๅฎšๅ•ๆœบ้ป˜่ฎคๅ‘จๆœŸ็ญ–็•ฅๅŽๅฏ็”จๆˆ–ๅˆ ้™ค้’ฉๅญใ€‚ + +**HS-17 DeleteAllVersions ๆ‰น้‡ไผ˜ๅŒ–ๆ ธๅฏน**๏ผšMinIO ็”จ DeletePrefix+DeletePrefixObject ๅ•่ฐƒ็”จไปฃๆ›ฟ้€็‰ˆๆœฌ fan-outใ€‚RustFS expiry ้˜Ÿๅˆ—่ทฏๅพ„ๆ˜ฏๅฆๅŒๆฌพไผ˜ๅŒ–ๆœช้€่กŒๆ ธๅฎž๏ผˆ้›†ๆˆๆต‹่ฏ•่ฆ†็›–่กŒไธบๆญฃ็กฎๆ€ง๏ผ‰ใ€‚ๅปบ่ฎฎ๏ผšๆ ธๅฏน `apply_expiry_rule` ๅ…จ็‰ˆๆœฌๅˆ ้™ค่ทฏๅพ„๏ผŒ่‹ฅๆ— ๅ‰็ผ€ๅ•่ฐƒ็”จไผ˜ๅŒ–ๅˆ™่ฏ„ไผฐ่กฅ้ฝใ€‚ + +### P3๏ผˆ3 ้กน๏ผ‰ + +**HS-18 trash/ไธดๆ—ถ็›ฎๅฝ•ไบŒๆฎตๆธ…็†็ป†่Š‚ๆ ธๅฏน**๏ผšMinIO `.minio.sys/tmp/.trash` ๆธ…็†๏ผˆdelete_cleanup_interval ้ป˜่ฎค 5m + deleteCleanupSleeper๏ผ‰ไธŽ stale uploads rename-into-trash ไบŒๆฎตๅผใ€‚RustFS ๆœ‰ delete_tail_activity.rs ไธŽ stale multipart ไปปๅŠก๏ผŒไบŒๆฎต่ฏญไน‰ๆ˜ฏๅฆๅฎŒๆ•ดๅฏน้ฝๆœช้€่กŒๆ ธๅฎžใ€‚ๅปบ่ฎฎ๏ผšๅฏน็…ง่กฅ้ฝๆˆ–ๆ–‡ๆกฃๅŒ–ใ€‚ + +**HS-19 root heal ็›ด่ฟžๆญป่ทฏๅพ„ๆธ…็†**๏ผš`should_handle_root_heal_directly` ๆ’ false๏ผˆadmin/handlers/heal.rs:1200-1202๏ผŒๆต‹่ฏ•้”ๅฎš๏ผ‰๏ผŒstore.heal_format ็›ด่ฟžๅˆ†ๆ”ฏไธๅฏ่พพใ€‚ๅปบ่ฎฎ๏ผšๅˆ ้™คๆญปๅˆ†ๆ”ฏๆˆ–ๆขๅค็›ด่ฟž่ทฏๅพ„ไฝœไธบ้›†็พคๅ่ฐƒๅคฑ่ดฅ็š„้™็บงใ€‚ + +**HS-20 ๅ…ผๅฎนๆ——ๆ ‡ไธŽๆญปๆŒ‡ๆ ‡ๆธ…็†**๏ผš`RUSTFS_SCANNER_INLINE_HEAL_ENABLE`๏ผˆๅผ€ๅฏไป…ๅ‘Š่ญฆ๏ผ‰+ `rustfs_scanner_inline_heal_total` ๆญปๆŒ‡ๆ ‡ + `rustfs_common::metrics` ไธญ scanner ๅŸŸไปฃ็ ๅˆ†ๅฑ‚่ฟ็งป๏ผˆbacklog #1843 ๅทฒ็™ป่ฎฐ๏ผ‰ใ€‚ๅปบ่ฎฎ๏ผš้šๅˆ†ๅฑ‚่ฟ็งปไธ€ๅนถๆธ…็†ใ€‚ + +### ๆŒ‰่ฎพ่ฎกไธ่ฟฝๅนณ๏ผˆ7 ้กน๏ผŒ่ฎฐๅฝ•ไปฅ้˜ฒๅŽ็ปญ่ฏฏๅˆคไธบ็ผบๅฃ๏ผ‰ + +1. **bloom filter**๏ผšMinIO master ๅทฒๅˆ ้™ค๏ผ›RustFS `.bloomcycle.bin` ๅค็”จไธบ cycle/epoch ๅ›ดๆ ไธŽ MinIO ็Žฐ็Šถไธ€่‡ดใ€‚ +2. **scanner ้›†็พคๅ• leader**๏ผšๅŒๆ–นไธ€่‡ด๏ผ›RustFS ้ขๅค–ๆœ‰ epoch ๅ›ดๆ ใ€‚ +3. **heal ไธๅ‘ S3 bucket notification**๏ผšๅŒๆ–นไธ€่‡ด๏ผˆheal ็ป“ๆžœ่ตฐ admin status๏ผ‰ใ€‚ +4. **incomplete multipart ไธๅœจ scanner/ILM ๅ†…ๆ‰ง่กŒ**๏ผšๅŒๆ–นไธ€่‡ด๏ผˆ็‹ฌ็ซ‹ๅŽๅฐไพ‹็จ‹๏ผ‰ใ€‚ +5. **ๅ†…่” heal ็งป้™ค**๏ผšRustFS ๆœ‰ๆ„ไธบไน‹๏ผˆscanner ๅชๅ…ฅ้˜Ÿ๏ผ‰๏ผŒMinIO ็š„ applyHealing ๅ†…่”่ทฏๅพ„ไธๅšๅฏนๆ ‡ใ€‚ +6. **heal ๅบๅˆ—ๅธธ้ฉปไฟๆดป๏ผˆ10s ็ฉบ็™ฝๅ›žๅ†™๏ผ‰**๏ผšRustFS ๅฟซ็…งๅผๆŸฅ่ฏขๆจกๅž‹ไธๅŒ๏ผŒๆŒ‰ HS-06 ๅค„็†ๅขž้‡่ฏญไน‰ๅณๅฏ๏ผŒไธๅคๅˆถๆตๅผไฟๆดปใ€‚ +7. **`.trash`/`tmp-old` ่ทฏๅพ„ๅๅ…ผๅฎน**๏ผšRustFS ๅธƒๅฑ€ๅธธ้‡็‹ฌ็ซ‹๏ผŒไธ้€ๅญ—ๅฏน้ฝ MinIO ่ทฏๅพ„ใ€‚ + +--- + +## 6. ้…็ฝฎ้ป˜่ฎคๅ€ผๆ€ป่กจ๏ผˆRustFS๏ผ‰ + +heal๏ผˆenv ๅ‰็ผ€ `RUSTFS_HEAL_`๏ผŒ`crates/config/src/constants/heal.rs`๏ผŒๆถˆ่ดนไบŽ `manager.rs:724-800`๏ผ‰๏ผš + +| ้…็ฝฎ | ้ป˜่ฎค | ็ƒญๆ›ดๆ–ฐ | +|---|---|---| +| AUTO_HEAL_ENABLE | true | ๅฆ | +| QUEUE_SIZE | 10000 | ๅฆ | +| INTERVAL_SECS | 10 | ๅฆ๏ผˆๅฏๅŠจๆ—ถๅ›บๅฎš๏ผ‰ | +| TASK_TIMEOUT_SECS | 300 | ๅฆ | +| MAX_CONCURRENT_HEALS | 4 | ๅฆ | +| MAX_CONCURRENT_PER_SET | 1๏ผˆโ‰คmin(ๅ…จๅฑ€,ๅ€ผ)๏ผ‰ | ๅฆ | +| LOW_PRIORITY_MERGE_ENABLE | true | ๅฆ | +| LOW_PRIORITY_DROP_WHEN_FULL | true | ๅฆ | +| PAGE_OBJECT_CONCURRENCY | 8๏ผˆDeep/AutoHeal ๅผบๅˆถ 1๏ผ‰ | ๅฆ | +| EVENT_DRIVEN_SCHEDULER_ENABLE | true | ๅฆ | +| SET_BULKHEAD_ENABLE | true | ๅฆ | +| PAGE_PARALLEL_ENABLE | true | ๅฆ | +| MAINLINE_THROTTLE_ENABLE | true | ๅฆ | +| MAINLINE_READ/WRITE_UTILIZATION_HIGH_PERCENT | 80/80 | ๅฆ | +| MAINLINE_MAX_SLEEP_MS | 250 | ๅฆ | +| ๏ผˆๆ€ปๅผ€ๅ…ณ๏ผ‰RUSTFS_HEAL_ENABLED | true | ๅฆ | +| admin ๅญ็ณป็ปŸ heal.bitrot_cycle | 30d | ๆ˜ฏ๏ผˆ็ป scanner runtime config๏ผ‰ | + +scanner๏ผˆadmin ๅญ็ณป็ปŸ `scanner`๏ผŒ`crates/config/src/constants/scanner.rs` + `ecstore/src/config/scanner.rs` + `runtime_config.rs:527-673`๏ผ‰๏ผš + +| ้”ฎ | env | ้ป˜่ฎค | +|---|---|---| +| speed | RUSTFS_SCANNER_SPEED | default๏ผˆ2x/1s/60s๏ผ‰ | +| delay / max_wait / cycle / start_delay | RUSTFS_SCANNER_* | ๆดพ็”Ÿ/็ฉบ | +| cycle_max_duration/objects/directories | โ€ฆ_MAX_* | 0๏ผˆไธ้™๏ผ‰ | +| bitrot_cycle | โ€ฆ_BITROT_CYCLE_SECS | 2592000๏ผˆ30d๏ผ›0/on=ๆฏ่ฝฎ๏ผŒoff=็ฆ็”จ๏ผ‰ | +| idle_mode | โ€ฆ_IDLE_MODE | true | +| cache_save_timeout | โ€ฆ_CACHE_SAVE_TIMEOUT_SECS | 30s | +| max_concurrent_set_scans / disk_scans | โ€ฆ_MAX_CONCURRENT_* | 4/4 | +| yield_every_n_objects | โ€ฆ_YIELD_EVERY_N_OBJECTS | 128 | +| alert_excess_versions / version_size / folders | โ€ฆ_ALERT_* | 100 / 1TiB / 65538 | + +scanner ๅ†…้ƒจ env๏ผš`RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=16`ใ€`RUSTFS_HEAL_OBJECT_SELECT_PROB=1024`ใ€`RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS=60`ใ€`RUSTFS_DATA_USAGE_FAILED_OBJECT_TTL_SECS=86400`/`_MAX=10000`ใ€`RUSTFS_LOCK_ACQUIRE_TIMEOUT=5s`ใ€`RUSTFS_SCANNER_ENABLED=true`ใ€`RUSTFS_SCANNER_INLINE_HEAL_ENABLE=false`๏ผˆๅ…ผๅฎนๅ‘Š่ญฆ๏ผ‰ใ€‚ + +ๅ…จ้ƒจ 17 ไธช scanner ้”ฎๆ”ฏๆŒ env > config ๅŒ้€š้“ + admin PUT ็ƒญๆ›ด๏ผˆgeneration+Notify ๅณๆ—ถ็”Ÿๆ•ˆ๏ผ‰๏ผ›heal ่ฟ่กŒๆ—ถๅ‚ๆ•ฐ็›ฎๅ‰ไป… env๏ผˆๆ—  admin ็ƒญๆ›ดๅ…ฅๅฃ๏ผŒ`Arc>` ็ป“ๆž„ๅทฒ้ข„็•™๏ผ‰ใ€‚ + +--- + +## 7. ็›ธๅ…ณ backlog / ๅŽ†ๅฒ็ดขๅผ• + +- ๆข็›˜่‡ชๅŠจไฟฎๅค็ณปๅˆ—๏ผˆๅทฒ้—ญ็Žฏ๏ผ‰๏ผšbacklog #1786๏ผˆๅ†—ไฝ™ๅ‡็ปฟ็ฎ—ๆณ•๏ผ‰ใ€#1787๏ผˆ็›ฎๆ ‡ๆงฝไฝ้™ๅฎš๏ผ‰ใ€#1789๏ผˆresume ไธŽ healing marker ็ป‘ๅฎš replacement ๅฎžไพ‹๏ผ‰ใ€#1791๏ผˆ้ป‘็™ฝ็›’้ชŒๆ”ถ็Ÿฉ้˜ต๏ผ‰ใ€‚ +- #801 DiskInfo.healing ไปŽๆœช่ต‹ๅ€ผ๏ผˆๅทฒไฟฎๅค้—ญ็Žฏ๏ผŒ็Žฐ `set_disk/mod.rs:4988` ๆœ‰่ต‹ๅ€ผ้“พ๏ผ‰ใ€‚ +- #1651 Scanner ๆŒ‡ๆ ‡่Š‚็‚น/source/bucket-drive ็ปดๅบฆ๏ผˆOPEN๏ผŒๆœฌๅˆ†ๆž ยง3.8/ยง4.6 ็›ธๅ…ณ๏ผ‰ใ€‚ +- #1843 crates/common 83% scanner/heal ๅŸŸไปฃ็ ๅˆ†ๅฑ‚่ฟ็งป๏ผˆOPEN๏ผŒๅซ HS-20๏ผ‰ใ€‚ +- ไปฃ็ ๆณจ้‡Šๅผ•็”จ็š„ๅŽ†ๅฒ็ผบ้™ท๏ผˆ็Žฐๅทฒๆœ‰้˜ฒๆŠคไธŽๅ›žๅฝ’ๆต‹่ฏ•๏ผ‰๏ผš#856/#799 B7๏ผˆ็ฆป็บฟ็›˜่ฏฏ่ฎฐ healed๏ผ‰ใ€#855/B6/#1033๏ผˆskip ไธๅพ—ๆ ‡่ฎฐๅฎŒๆˆ๏ผ‰ใ€#920๏ผˆsub-quorum ๅนถ้›†ๆžšไธพ๏ผ‰ใ€#856 B5๏ผˆๆŒ‰็‰ˆๆœฌ็ปญๆ‰ซ๏ผ‰ใ€#5173๏ผˆbitrot trailing bytes๏ผ‰ใ€#5029๏ผˆๅ›žๅฝ’่Š‚็‚น stale ็‰ˆๆœฌๅˆๅนถ๏ผ‰ใ€‚ +- v1 ๅฏนๆ ‡ๆ–‡ๆกฃ๏ผš`docs/rustfs-heal-scanner-vs-minio-parity-assessment.md`๏ผˆๆœฌๆ–‡ๅ–ไปฃ๏ผ‰ใ€่ฝๅœฐๆ‰‹ๅ†Œ `docs/rustfs-heal-scanner-vs-minio-improvement-playbook.md`๏ผˆ้ƒจๅˆ†ๆก็›ฎๅทฒ่ขซๅŽ็ปญๅฎž็Žฐ่ถ…่ถŠ๏ผ‰ใ€‚ +- ๆข็›˜ๆทฑๅบฆๅˆ†ๆž๏ผš`docs/new-disk-replacement-and-healing-deep-analysis-zh.md`ใ€`docs/node-disk-identity-and-healing-analysis-zh.md`ใ€‚ + +## 8. ๅฎก่ฎกๆ–นๆณ•ไธŽๅฑ€้™ + +- ๅ››่ทฏๅนถ่กŒๅฎก่ฎก๏ผˆheal crate ้€ๆ–‡ไปถใ€scanner crate ้€ๆ–‡ไปถใ€ecstore ้›†ๆˆๅฑ‚ wiringใ€MinIO master ๆบ็ ็ ”็ฉถ๏ผ‰+ ไธปไผš่ฏๅฏนๅ…ณ้”ฎ"็ผบๅคฑ"็ป“่ฎบ้€ๆกไบฒ้ชŒ๏ผˆget_disk_status TODOใ€HealEvent ้›ถๅค–้ƒจๅผ•็”จใ€.bloomcycle.bin ๆ—  bloom ๅฎž็Žฐใ€check_abandoned_parts ไธ‰ๅฑ‚ NotImplementedใ€ETag ๅ…œๅบ•ๅทฒๅฎž็Žฐใ€trace ้€š้“้›ถๅ‘ฝไธญใ€already_running ่ฏญไน‰๏ผ‰ใ€‚ +- ๆœช้€่กŒๆ ธๅฎž็š„็‚น๏ผˆๅทฒๅœจๆ–‡ไธญๆ ‡ๆณจ"ๆœช็กฎ่ฎค/ๆœช้€่กŒๆ ธ"๏ผ‰๏ผšDeleteAllVersions ๅ‰็ผ€ๅ•่ฐƒ็”จไผ˜ๅŒ–๏ผˆHS-17๏ผ‰ใ€trash ไบŒๆฎตๆธ…็†็ป†่Š‚๏ผˆHS-18๏ผ‰ใ€ilm worker ้ป˜่ฎคๅ€ผๅฏน็…งใ€stale multipart ้ป˜่ฎคๅ€ผๅฏน็…งใ€mc CLI flag ้€ๅญ—ๆ‹ผๅ†™๏ผˆMinIO ไพง๏ผ‰ใ€‚ๅ…ถไธญ HS-17 ไธŽ HS-18 ๅทฒไบŽ 2026-08-19 ๅฎŒๆˆ้€่กŒๆ ธๅฎž๏ผŒ็ป“่ฎบ่ง ยง9.2/ยง9.3ใ€‚ +- MinIO ไพงๅผ•็”จไปฅๅ…ถ master `7aac2a2c5b` ไธบๅ‡†๏ผ›RustFS ไพง่กŒๅทไปฅ 2026-08-16 ๅทฅไฝœๅŒบไธบๅ‡†๏ผŒๅŽ็ปญๆผ”่ฟ›่ฏทไปฅ็ฌฆๅทๅๆฃ€็ดขไธบๅ‡†ใ€‚ + +## 9. ่ฝๅœฐ็ป“ๆžœ๏ผˆ2026-08-19 ๆ›ดๆ–ฐ๏ผ‰ + +ๆœฌๅฎก่ฎก่ก็”Ÿ็š„ 14 ไธชๅญ issue๏ผˆbacklog #1865~#1878๏ผ‰ๅทฒๅ…จ้ƒจ้—ญ็Žฏใ€‚ๆœฌ่Š‚ไธบๅทฎ่ทๆธ…ๅ• HS-01~HS-20 ็š„ๆœ€็ปˆๅค„็ฝฎ่ฎฐๅฝ•๏ผŒไนŸๆ˜ฏไธ‹ไธ€่ฝฎๅฏนๆ ‡้‡ๅฎก็š„ๅขž้‡ๅŸบ็บฟใ€‚ + +### 9.1 ๅทฒ่ฝๅœฐ๏ผˆPR ๅ‡ๅทฒๅˆๅนถ main๏ผ‰ + +- HS-01 MRF ๆŽฅ็บฟ + ๆŒไน…ๅŒ–ไฟฎๅค่ดฆๆœฌ๏ผˆ#1865๏ผŒPR #6189๏ผ‰๏ผšๅ†ณ็ญ–้€‰ (a)ใ€‚common MRF channel๏ผˆbounded 8192ใ€try_send ๆฐธไธ้˜ปๅกž๏ผ‰+ heal mrf_queue๏ผˆ100k ๆก / 8MiB ๅŒ้™็Žฏๅฝข๏ผ‰+ `buckets/.heal/mrf/journal.bin` CRC ๆŒไน…ๅŒ–ๅ›žๆ”พ๏ผˆtorn tail ๆˆชๆ–ญใ€ๅ›žๆ”พๅŽๅˆ ้™ค๏ผ‰+ ไธ‰ๆŠ•้€’็‚น๏ผˆread decode_errorโ†’Urgent ECDecodeใ€scanner ๅ…ƒๆ•ฐๆฎๆŸๅโ†’High Metadataใ€add_partialโ†’Normal๏ผ‰+ `RUSTFS_HEAL_MRF_ENABLE` ไธ€้”ฎๅ›ž้€€ใ€‚ +- HS-02 abandoned parts/data-dir ๅฏน่ดฆ๏ผˆ#1866๏ผŒPR #6179๏ผ‰๏ผšๆŽฅ้€š abandoned ๆฃ€ๆŸฅๅ…ฅๅฃ๏ผŒไฟ็•™ dry-run / reclaim ่ฎกๆ•ฐใ€‚ +- HS-03 heal/scanner trace ้€š้“๏ผˆ#1867๏ผŒPR #6179๏ผ‰๏ผš่ฟ›็จ‹ๅ†… trace bus + `/v3/trace` admin ๆตๅผ่ฎข้˜… + heal task / abandoned-parts / scanner folder / ILM / heal-candidate trace producerใ€‚ +- HS-04 scanner ่ถ…้™ S3 ไบ‹ไปถ๏ผˆ#1868๏ผŒPR #6176๏ผ‰๏ผš`s3:Scanner:ManyVersions/LargeVersions/BigPrefix` ไธ‰ไบ‹ไปถ + 24h ่พนๆฒฟๅ†ทๅด๏ผ›HS-15 ้˜ˆๅ€ผๅทฎๅผ‚ๆ–‡ๆกฃๅŒ–๏ผˆ`docs/operations/scanner-excess-alerts.md`๏ผ‰ใ€‚ +- HS-05 madmin ๅฎขๆˆท็ซฏไธ€ๆœŸ๏ผˆ#1869๏ผŒPR #6166๏ผ‰๏ผšSigV4 admin ๅฎขๆˆท็ซฏ heal/scanner ๆ–นๆณ•๏ผ›ๅขž้‡ๆถˆ่ดนๆ–นๆณ•ๅพ… follow-up๏ผˆๅ่ฎฎๅทฒ็”ฑ HS-06 ๅนถๅ…ฅ๏ผ‰ใ€‚ +- HS-06 admin heal ๅขž้‡่ฏญไน‰ไธŽ็ฑปๅž‹ๅŒ–้‡ๅ ๏ผˆ#1870๏ผŒPR #6206๏ผ‰๏ผš`sinceSeq/nextSeq/minSeq` ๅขž้‡ๆธธๆ ‡๏ผˆwire additiveใ€็ผบ็œ=ๅ…จ้‡ๅฟซ็…ง๏ผ‰+ `RUSTFS_HEAL_OVERLAP_POLICY`๏ผˆ้ป˜่ฎค merge ไธๅ˜๏ผ›minio_error ไธ‹ AlreadyRunning/OverlappingPaths ็ฑปๅž‹ๅŒ–ๆ‹’็ป๏ผ‰+ forceStart ๅ…ˆๅœๆ—งๅ†ๅฏๆ–ฐใ€‚ +- HS-07 healing ่ฟ›ๅบฆๅฏ่งๆ€ง๏ผˆ#1871๏ผŒPR #6179๏ผ‰๏ผšdata-usage ๆ€ป้‡ๅŸบ็บฟ + baseline/current/healed ่ฎกๆ•ฐใ€‚ +- HS-08 prefix usage๏ผˆ#1872๏ผŒPR #6171๏ผ‰๏ผš`GET /v3/usage/{bucket}`ใ€‚ +- HS-11 bitrot ๅฏๅŠจ่‡ชๆฃ€๏ผˆ#1873๏ผŒPR #6165๏ผ‰ใ€‚ +- HS-13 heal ่ทณ่ฟ‡่ฟ‡ๆปค๏ผˆ#1875๏ผŒPR #6179๏ผ‰๏ผš่ฟ‡ๆปคๅ‘ฝไธญ็‰ˆๆœฌไธๅ†่ฎกไธบๅคฑ่ดฅใ€‚ +- HS-16 ๅ•ๆœบๅ‘จๆœŸ้’ฉๅญ๏ผˆ#1878๏ผŒPR #6250๏ผ‰๏ผšๅˆ ๆ’ None ้’ฉๅญ๏ผŒๅ†ณ็ญ–่ฎฐๅฝ•่ง `docs/operations/heal-scanner-parity-notes-zh.md`ใ€‚ +- HS-09/10/19/20 ๆญปไปฃ็ ๆธ…็†ๆ‰น๏ผˆ#1877๏ผŒPR #6256๏ผ‰๏ผšๅ‡€ โˆ’911 ่กŒ้›ถ่กŒไธบๅ˜ๆ›ด๏ผ›`get_disk_status` TODO๏ผˆๅ…จไป“ๅบ“ๅ”ฏไธ€ไบงๅ“ TODO๏ผ‰ๆธ…้›ถ๏ผ›HS-01 ่”ๅŠจ็š„ `ec_decode_rebuild`/`get_object_meta` ไฟ็•™ๅนถๅŠ  Reserved ๆณจ้‡Š๏ผˆMRF ๅฝ“ๅ‰็ป `heal_object` ๆ‰ง่กŒ๏ผ‰ใ€‚ + +### 9.2 ๆ ธๅฏนๅŽ็กฎ่ฎค"ๅทฒๅฎž็Žฐ / ้ž็ผบๅฃ"๏ผˆๅฎก่ฎกๆœŸ่ฏฏๅˆคไฟฎๆญฃ๏ผŒ็ดฏ่ฎกๅ››ไพ‹๏ผ‰ + +- bloom filter๏ผˆยง0 ๅทฒไฟฎๆญฃ๏ผ‰๏ผšMinIO master ๅทฒๅˆ ้™ค๏ผŒๅŒๆ–น็Žฐ็Šถไธ€่‡ดใ€‚ +- ETag ๅ…œๅบ•ไปฒ่ฃ๏ผˆยง0 ๅทฒไฟฎๆญฃ๏ผ‰๏ผšRustFS ๅทฒๆœ‰ๅฎž็Žฐ๏ผˆ`set_disk/ops/heal.rs`๏ผ‰ใ€‚ +- HS-17๏ผˆ#1876๏ผŒ2026-08-19 ้€่กŒๆ ธๅฎžๅŽๅ…ณ้—ญ๏ผ‰๏ผšDeleteAllVersions ๅ‰็ผ€ๅ•่ฐƒ็”จไผ˜ๅŒ– RustFS ๅทฒๅฎŒๆ•ดๅฎž็Žฐโ€”โ€”`apply_expiry_on_non_transitioned_objects` ๅฏน `delete_all()` ไธค action ่ฎพ `delete_prefix + delete_prefix_object` ๅŽๅ•ๆฌก `delete_object`๏ผˆ`bucket_lifecycle_ops.rs:5047-5056`๏ผ‰๏ผŒSetDisks ๅˆ†ๆ”ฏไธ€ๆฌกๅ†™้” + ไธ€ๆฌกๅ…จ็‰ˆๆœฌ quorum ่ฏป + ๅ†…่”้€็‰ˆๆœฌ object-lock ๆฃ€ๆŸฅ๏ผˆ`set_disk/ops/object.rs:5566-5612`๏ผ‰๏ผŒไธŽ MinIO `expire.go` ็š„ `applyExpiryOnNonTransitionedObjects` ้€่กŒๅฏน้ฝใ€‚ยง8 ๅŽŸๅˆ—"ๆœช้€่กŒๆ ธๅฎž"็š„ๆœฌ้กนๅทฒๆœ‰็ป“่ฎบ๏ผš็Žฐ็Šถๅณไผ˜ๅŒ–่ทฏๅพ„๏ผŒๆ— ้œ€ๅฎž็Žฐใ€‚ +- HS-14๏ผˆ#1878๏ผŒPR #6250 ้™„ๅธฆๆ ธๅฏน๏ผ‰๏ผšMinIO"idle=็ฉบ้—ฒๆ‰่Š‚ๆต"ๆ˜ฏ 2024-01 minio/minio#18734 ไน‹ๅ‰็š„่กŒไธบ๏ผˆ`scannerIdleMode` ็Žฐไธบ้™ๆ€้…็ฝฎ๏ผŒ`idle_speed=on` ้ป˜่ฎคๅณๅง‹็ปˆๆŒ‰้€Ÿๅบฆๆกฃ่Š‚ๆต๏ผŒ"idle"ๅ‘ฝๅๆ˜ฏๅŽ†ๅฒๆฎ‹็•™๏ผ‰๏ผ›RustFS `RUSTFS_SCANNER_IDLE_MODE` ไธŽ MinIO ๅฝ“ๅ‰่ฏญไน‰ๆ–นๅ‘ไธ€่‡ด๏ผŒไธ”ๅฆๆœ‰ MinIO ๆฒกๆœ‰็š„ๅ‰ๅฐ่ฏป้€€้ฟไธ‹้™ใ€‚็œŸๅฎž่ฟ็งป้™ท้˜ฑ๏ผˆๅ˜้‡้กป `RUSTFS_` ๅ‰็ผ€ใ€`on/off` vs `true/false` ่ฏ่กจใ€`false` ่ฟžๅ‰ๅฐไฟๆŠคไธ€่ตทๅ…ณ๏ผ‰ๅทฒๆ–‡ๆกฃๅŒ–ไบŽ `docs/operations/heal-scanner-parity-notes-zh.md`ใ€‚ + +### 9.3 ๅฎก่ฎกๅž‹็ป“่ฎบ๏ผˆๆ— ้œ€ๆ”นไปฃ็ ๏ผ‰ + +- HS-12๏ผˆ#1874๏ผŒPR #6183๏ผ‰๏ผšไธๅญ˜ๅœจ MinIO ็”จ `x-minio-healing` ้˜ฒๅพก็š„้‚ฃ็ฑป็ซžไบ‰โ€”โ€”ๆ‰€ๆœ‰ๅŒ (bucket, object) ๆไบค้ขๅœจๅŒไธ€ๆŠŠๅฏน่ฑก็บง ns ๅ†™้”ไบ’ๆ–ฅ๏ผŒheal ้” guard ่ฆ†็›– rename ๆไบคๅ…จ็จ‹๏ผ›ไบคไป˜ 2 ไธชๅนถๅ‘ไธๅ˜้‡ๅ›žๅฝ’ๆต‹่ฏ• + `docs/operations/heal-concurrency-safety-notes-zh.md` ไบค็‚น็Ÿฉ้˜ตใ€‚ +- HS-18๏ผˆ#1878๏ผŒ2026-08-19 ้€่กŒๆ ธๅฎž๏ผ‰๏ผštrash/tmp ไธ‰ๆฎตๆธ…็†ๅ…จๅฏน้ฝโ€”โ€”stale multipart ้š”็ฆป-ๆธ…็†็ญ‰ไปทไธ”ๆ›ดๅฎ‰ๅ…จ๏ผˆ`delete_all_with_quorum` ้€็›˜้€’ๅฝ’ๅˆ ๅณ `move_to_trash` rename ่ฟ› `.rustfs.sys/tmp/.trash`๏ผŒๅฆๆœ‰้” + fence๏ผ‰ใ€trash ๆŽ’็ฉบๅŸบๆœฌ็ญ‰ไปท๏ผˆๆ— ้€ๆก sleeper ่Š‚ๆต๏ผŒ5m ๅ‘จๆœŸๅคฉ็„ถ้™้ข‘๏ผ‰ใ€tmp ้ž trash 24h ๅ›žๆ”ถ็ญ‰ไปท๏ผˆRustFS 5m ๆฏ” MinIO 6h ๆ›ดๅŠๆ—ถ๏ผ‰๏ผ›ๅ‘จๆœŸ้ป˜่ฎค 24h/6h/5m ไธ‰้กนๅ…จๅฏน้ฝใ€‚ยง8 ๅŽŸๅˆ—"ๆœช้€่กŒๆ ธๅฎž"็š„ๆœฌ้กนๅทฒๆœ‰็ป“่ฎบใ€‚ + +### 9.4 ็งปไบค follow-up๏ผˆๆฑ‡ๆ€ปไบŽ backlog#1862 ่ฏ„่ฎบๅŒบ๏ผ‰ + +HS-01 bitrot GETโ†’MRF ๅ…จ้“พ่ทฏ e2eใ€kill -9 journal ๅ›žๆ”พ e2eใ€้˜Ÿๅˆ—ๆปกๅŽ‹ๆต‹ RSS๏ผˆโ‰ค ้ข„็ฎ—+10%๏ผ‰๏ผ›HS-05/06 madmin ๅขž้‡ๆถˆ่ดนๆ–นๆณ• + wire ๅ•ไธ€ๆฅๆบๅŒ– + embedded e2e + ๅคš่ฝฎ่ฝฎ่ฏข soak๏ผ›HS-08 ๅคš็›˜ scanner ๅ‘จๆœŸ e2e๏ผ›HS-04 ่ถ…้™ๅฎก่ฎกๆก็›ฎ๏ผ›HS-18 ไฝŽไบŽ quorum ็š„ stale-multipart ๅดฉๆบƒๆฎ‹็•™็ช—ๅฃ๏ผˆๆ‰‡ๅ‡บไธญ้€”ๅดฉๆบƒไธ”ๅทฒๆธ…็›˜ๆ•ฐ > parity ๆ—ถ FileNotFound ไธๅœจๅฟฝ็•ฅ้›†ๅฏผ่‡ดไธ่‡ช็„ถๆ”ถๆ•›๏ผŒไฟฎๅค้œ€ไธ“็”จ quorum ๅ˜ไฝ“๏ผ‰ใ€‚ + +ไธ‹ไธ€่ฝฎ้‡ๅฎกๅปบ่ฎฎ๏ผš่ทŸ้š heal/scanner ไธ‹ไธ€ไธชๅคง็‰นๆ€ง่ฝๅœฐๅŽ่งฆๅ‘๏ผŒไปฅๆœฌ่Š‚ไธบๅขž้‡ๅŸบ็บฟใ€‚ diff --git a/helm/README.md b/helm/README.md index 79b7c6fd6..c246a3bac 100644 --- a/helm/README.md +++ b/helm/README.md @@ -267,7 +267,7 @@ uer. `ClusterIssuer` or `Issuer`. | | topologySpreadConstraints.enabled | bool | `false` | Enable custom topology spread constraints on distributed-mode StatefulSet pods. | | topologySpreadConstraints.constraints | list | `[]` | Raw `spec.template.spec.topologySpreadConstraints` entries applied to the distributed StatefulSet when enabled. | | gatewayApi.enabled | bool | `false` | To enable/disable gateway api support. | -| gatewayApi.gatewayClass | string | `traefik` | Gateway class implementation. | +| gatewayApi.gatewayClass | string | `traefik` | Gateway class implementation (traefik, contour, istio). | | gatewayApi.httpToHttpsRedirect | bool | `true` | To enable/disable the redirect httproute. | | gatewayApi.listeners.http.name | string | `web` | Gateway API http listener name. | | gatewayApi.listeners.http.port| int | `8000` | Gateway API http listener port. | @@ -437,7 +437,7 @@ helm install rustfs rustfs/rustfs -n rustfs --set tls.enabled=true,--set-file tl # Gateway API support (alpha) -Due to [ingress nginx retirement](https://kubernetes.io/blog/2025/11/11/ingress-nginx-retirement/) in March 2026, so RustFS adds support for [gateway api](https://gateway-api.sigs.k8s.io/). Currently, RustFS only supports traefik as gateway class, more and more gateway class support will be added in the future after those classes are tested. If you want to enable gateway api, specify `gatewayApi.enabled` to `true` while specify `ingress.enabled` to `false`. After installation, you can find the `Gateway` and `HttpRoute` resources, +Due to [ingress nginx retirement](https://kubernetes.io/blog/2025/11/11/ingress-nginx-retirement/) in March 2026, so RustFS adds support for [gateway api](https://gateway-api.sigs.k8s.io/). Currently, RustFS supports traefik, contour, and istio as gateway classes. If you want to enable gateway api, specify `gatewayApi.enabled` to `true` while specify `ingress.enabled` to `false`. After installation, you can find the `Gateway` and `HttpRoute` resources, ``` $ kubectl -n rustfs get gateway diff --git a/helm/rustfs/templates/gateway-api/istio-destinationrule.yaml b/helm/rustfs/templates/gateway-api/istio-destinationrule.yaml new file mode 100644 index 000000000..3c4b5cd43 --- /dev/null +++ b/helm/rustfs/templates/gateway-api/istio-destinationrule.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.gatewayApi.enabled (eq .Values.gatewayApi.gatewayClass "istio") }} +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + name: {{ include "rustfs.fullname" . }}-sticky + namespace: {{ .Release.Namespace }} +spec: + host: {{ include "rustfs.fullname" . }}-svc.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }} + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: rustfs + path: / + ttl: 0s +{{- end }} diff --git a/helm/rustfs/values.yaml b/helm/rustfs/values.yaml index d57317243..365a1a57c 100644 --- a/helm/rustfs/values.yaml +++ b/helm/rustfs/values.yaml @@ -361,7 +361,7 @@ ingress: gatewayApi: enabled: false - gatewayClass: traefik # Only support for traefik and contour gatewayClass at the moment. + gatewayClass: traefik # Supported gatewayClass: traefik, contour, istio. listeners: # Specify which listeners to create on the Gateway. http: name: web diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 4528e30e4..206d9dfcc 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -354,7 +354,7 @@ proptest = "1" tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] } temp-env = { workspace = true, features = ["async_closure"] } tracing-subscriber = { workspace = true, features = ["env-filter", "time"] } -metrics-util = { version = "0.20", features = ["debugging"] } +metrics-util = { workspace = true, features = ["debugging"] } opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] } rsa = { workspace = true } rcgen = { workspace = true } diff --git a/rustfs/src/admin/console.rs b/rustfs/src/admin/console.rs index acf820e6b..5ee314044 100644 --- a/rustfs/src/admin/console.rs +++ b/rustfs/src/admin/console.rs @@ -119,6 +119,10 @@ async fn static_handler(uri: Uri) -> impl IntoResponse { #[derive(Debug, Serialize, Clone)] pub(crate) struct Config { #[serde(skip)] + #[allow( + dead_code, + reason = "reachable only from this file's tests: no route registers config_handler (backlog#1823)" + )] port: u16, api: Api, s3: S3, @@ -176,11 +180,14 @@ impl Config { } } + #[allow( + dead_code, + reason = "reachable only from this file's tests: no route registers config_handler (backlog#1823)" + )] fn to_json(&self) -> String { serde_json::to_string(self).unwrap_or_default() } - #[allow(dead_code)] pub(crate) fn version_info(&self) -> String { format!( "RELEASE.{}@{} (rust {} {})", @@ -190,21 +197,6 @@ impl Config { build::BUILD_TARGET ) } - - #[allow(dead_code)] - pub(crate) fn version(&self) -> String { - self.release.version.clone() - } - - #[allow(dead_code)] - pub(crate) fn license(&self) -> String { - format!("{} {}", self.license.name.clone(), self.license.url.clone()) - } - - #[allow(dead_code)] - pub(crate) fn doc(&self) -> String { - self.doc.clone() - } } fn build_console_api_base_url(base_url: &str) -> String { @@ -353,7 +345,10 @@ async fn version_handler() -> impl IntoResponse { /// - 200 OK with JSON body containing the console configuration if initialized. /// - 500 Internal Server Error if configuration is not initialized. #[instrument(fields(uri))] -#[allow(dead_code)] +#[allow( + dead_code, + reason = "reachable only from this file's tests: no route registers it (backlog#1823)" +)] async fn config_handler(uri: Uri, headers: HeaderMap) -> impl IntoResponse { // Get the scheme from the headers or use the URI scheme let scheme = headers diff --git a/rustfs/src/admin/handlers/account_info.rs b/rustfs/src/admin/handlers/account_info.rs index ca2bda6ea..4a3b45952 100644 --- a/rustfs/src/admin/handlers/account_info.rs +++ b/rustfs/src/admin/handlers/account_info.rs @@ -42,7 +42,6 @@ fn map_data_usage_result(result: Result) -> S3Result, } +/// The pre-check keeps this endpoint's historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize_cluster_snapshot_request(req: &S3Request) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "authentication required")); - }; + } - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]).await?; + Ok(()) } fn build_json_response( @@ -953,6 +944,30 @@ mod tests { ); } + /// This endpoint authorizes through the shared admin gate, which reports + /// "get cred failed" for a credential-less request. The pre-check keeps the + /// message it has always returned (rustfs/backlog#1829). + #[tokio::test] + async fn cluster_snapshot_gate_keeps_its_missing_credentials_message() { + let req = s3s::S3Request { + input: s3s::Body::from(String::new()), + method: http::Method::GET, + uri: http::Uri::from_static("/rustfs/admin/v4/cluster/snapshot"), + headers: http::HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + let err = super::authorize_cluster_snapshot_request(&req) + .await + .expect_err("a request without credentials must be rejected"); + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("authentication required")); + } + #[test] fn cluster_snapshot_response_serializes_none_snapshot() { let value = serde_json::to_value(ClusterSnapshotResponse { snapshot: None }).expect("serialize response"); diff --git a/rustfs/src/admin/handlers/diagnostics.rs b/rustfs/src/admin/handlers/diagnostics.rs index 523df19db..83c3625fe 100644 --- a/rustfs/src/admin/handlers/diagnostics.rs +++ b/rustfs/src/admin/handlers/diagnostics.rs @@ -28,18 +28,21 @@ use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::storage_api::access::spawn_traced; use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use crate::storage::storage_api::get_global_lock_clients; use bytes::Bytes; -use futures::{Stream, StreamExt}; +use futures::{Stream, StreamExt, future::join_all}; use http::{HeaderMap, HeaderValue, Uri, header::CONTENT_LENGTH}; use hyper::{Method, StatusCode}; use matchit::Params; -use rustfs_lock::{LockMode, ObjectKey, get_global_lock_manager}; +use rustfs_lock::{LockLeaseInfo, LockMode, LockType, ObjectKey, get_global_lock_manager}; use rustfs_policy::policy::action::{Action, AdminAction}; use s3s::header::CONTENT_TYPE; use s3s::stream::{ByteStream, DynByteStream}; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, StdError, s3_error}; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::pin::Pin; +use std::sync::Arc; use std::task::{Context, Poll}; use std::time::{Duration, SystemTime}; use tokio::sync::{Semaphore, SemaphorePermit, mpsc}; @@ -212,8 +215,156 @@ fn system_time_to_rfc3339(t: SystemTime) -> Option { dt.format(&time::format_description::well_known::Rfc3339).ok() } -fn collect_top_locks(limit: usize) -> TopLocksResponse { - let manager = get_global_lock_manager(); +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum TopLockMode { + Read, + Write, +} + +impl TopLockMode { + fn label(self) -> &'static str { + match self { + Self::Read => "READ", + Self::Write => "WRITE", + } + } +} + +#[derive(Debug, PartialEq, Eq, Hash)] +struct LockHolderKey { + resource: ObjectKey, + mode: TopLockMode, + owner: String, +} + +#[derive(Debug)] +struct TopLockState { + acquired_at: SystemTime, + ttl_secs: u64, + priority: &'static str, +} + +#[derive(Debug)] +struct LeaseHolderState { + acquired_at: SystemTime, + ttl_secs: u64, + holder_count: u32, + guard_ids: Option>, +} + +fn build_top_locks_response( + limit: usize, + now: SystemTime, + lease_infos: Vec, + fast_infos: Vec<(rustfs_lock::ObjectLockInfo, u32, Option>)>, +) -> TopLocksResponse { + let mut lease_holders = HashMap::with_capacity(lease_infos.len()); + + for info in lease_infos { + let mode = match info.lock_type { + LockType::Shared => TopLockMode::Read, + LockType::Exclusive => TopLockMode::Write, + }; + let key = LockHolderKey { + resource: info.resource, + mode, + owner: info.owner, + }; + let ttl_secs = info.remaining_ttl.as_secs(); + lease_holders + .entry(key) + .and_modify(|state: &mut LeaseHolderState| { + if info.acquired_at < state.acquired_at { + state.acquired_at = info.acquired_at; + } + state.ttl_secs = state.ttl_secs.max(ttl_secs); + state.holder_count = state.holder_count.saturating_add(1); + match (state.guard_ids.as_mut(), info.guard_id) { + (Some(guard_ids), Some(guard_id)) => guard_ids.push(guard_id), + _ => state.guard_ids = None, + } + }) + .or_insert(LeaseHolderState { + acquired_at: info.acquired_at, + ttl_secs, + holder_count: 1, + guard_ids: info.guard_id.map(|guard_id| vec![guard_id]), + }); + } + for state in lease_holders.values_mut() { + if let Some(guard_ids) = &mut state.guard_ids { + guard_ids.sort_unstable(); + } + } + + let mut infos: Vec<_> = fast_infos + .into_iter() + .map(|(info, holder_count, guard_ids)| { + let mode = match info.mode { + LockMode::Shared => TopLockMode::Read, + LockMode::Exclusive => TopLockMode::Write, + }; + let key = LockHolderKey { + resource: info.key, + mode, + owner: info.owner.to_string(), + }; + let priority = lock_priority_label(info.priority); + // Match the complete holder cohort so replacements cannot reuse stale lease data. + let state = match lease_holders.remove(&key) { + Some(lease) + if lease.holder_count == holder_count && lease.guard_ids.is_some() && lease.guard_ids == guard_ids => + { + TopLockState { + acquired_at: lease.acquired_at, + ttl_secs: lease.ttl_secs, + priority, + } + } + _ => TopLockState { + acquired_at: info.acquired_at, + ttl_secs: info.expires_at.duration_since(now).unwrap_or(Duration::ZERO).as_secs(), + priority, + }, + }; + (key, state) + }) + .collect(); + // Longest-held first, matching MinIO's `top locks` ordering intent. + infos.sort_by_key(|(_, state)| state.acquired_at); + let total = infos.len(); + let truncated = total > limit; + + let locks = infos + .into_iter() + .take(limit) + .map(|(holder, state)| LockEntry { + resource: format!("{}/{}", holder.resource.bucket, holder.resource.object), + bucket: holder.resource.bucket.to_string(), + object: holder.resource.object.to_string(), + version: holder.resource.version.as_ref().map(|version| version.to_string()), + lock_type: holder.mode.label(), + owner: holder.owner, + priority: state.priority, + since: system_time_to_rfc3339(state.acquired_at), + elapsed_secs: now.duration_since(state.acquired_at).unwrap_or(Duration::ZERO).as_secs(), + ttl_secs: state.ttl_secs, + }) + .collect(); + + TopLocksResponse { + total, + truncated, + locks, + capability_note: None, + } +} + +async fn collect_top_locks_with_clients( + limit: usize, + manager: Arc, + clients: Vec>, +) -> TopLocksResponse { let Some(fast) = manager.as_fast_lock_manager() else { return TopLocksResponse { total: 0, @@ -225,43 +376,27 @@ fn collect_top_locks(limit: usize) -> TopLocksResponse { }; }; - let now = SystemTime::now(); - let mut infos = fast.list_locks(); - // Longest-held first, matching MinIO's `top locks` ordering intent. - infos.sort_by_key(|i| i.acquired_at); - let total = infos.len(); - let truncated = total > limit; + let lease_infos = if clients.is_empty() { + Vec::new() + } else { + join_all(clients.iter().map(|client| client.list_lock_leases())) + .await + .into_iter() + .flatten() + .collect() + }; + // Capture holders last so released or replaced lease guards fail the merge checks. + let fast_infos = fast.list_locks_with_holder_generations(); - let locks = infos - .into_iter() - .take(limit) - .map(|info| { - let elapsed_secs = now.duration_since(info.acquired_at).unwrap_or(Duration::ZERO).as_secs(); - let ttl_secs = info.expires_at.duration_since(now).unwrap_or(Duration::ZERO).as_secs(); - LockEntry { - resource: format!("{}/{}", info.key.bucket, info.key.object), - bucket: info.key.bucket.to_string(), - object: info.key.object.to_string(), - version: info.key.version.as_ref().map(|v| v.to_string()), - lock_type: match info.mode { - LockMode::Exclusive => "WRITE", - LockMode::Shared => "READ", - }, - owner: info.owner.to_string(), - priority: lock_priority_label(info.priority), - since: system_time_to_rfc3339(info.acquired_at), - elapsed_secs, - ttl_secs, - } - }) - .collect(); + build_top_locks_response(limit, SystemTime::now(), lease_infos, fast_infos) +} - TopLocksResponse { - total, - truncated, - locks, - capability_note: None, - } +async fn collect_top_locks(limit: usize) -> TopLocksResponse { + let manager = get_global_lock_manager(); + let clients = get_global_lock_clients() + .map(|clients| clients.values().cloned().collect()) + .unwrap_or_default(); + collect_top_locks_with_clients(limit, manager, clients).await } fn parse_top_locks_limit(uri: &Uri) -> usize { @@ -279,7 +414,7 @@ impl Operation for TopLocksHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { authorize(&req, AdminAction::TopLocksAdminAction).await?; let limit = parse_top_locks_limit(&req.uri); - let response = collect_top_locks(limit); + let response = collect_top_locks(limit).await; json_response(StatusCode::OK, &response) } } @@ -1107,6 +1242,254 @@ mod tests { assert_eq!(parse_top_locks_limit(&Uri::from_static("/x?count=999999")), TOP_LOCKS_MAX_LIMIT); } + #[test] + fn top_locks_prefers_renewable_lease_deadlines() { + let now = SystemTime::UNIX_EPOCH + Duration::from_secs(100); + let leased_resource = ObjectKey::new("bucket", "shared-object"); + let exclusive_resource = ObjectKey::new("bucket", "write-object"); + let direct_resource = ObjectKey::new("bucket", "direct-object"); + let mixed_resource = ObjectKey::new("bucket", "mixed-object"); + let replaced_resource = ObjectKey::new("bucket", "replaced-object"); + let remaining_shared_resource = ObjectKey::new("bucket", "remaining-shared-object"); + let opaque_resource = ObjectKey::new("bucket", "opaque-object"); + + let response = build_top_locks_response( + TOP_LOCKS_DEFAULT_LIMIT, + now, + vec![ + LockLeaseInfo { + resource: leased_resource.clone(), + lock_type: LockType::Shared, + owner: "owner-a".to_string(), + acquired_at: now - Duration::from_secs(50), + guard_id: Some(11), + remaining_ttl: Duration::from_secs(5), + }, + LockLeaseInfo { + resource: leased_resource.clone(), + lock_type: LockType::Shared, + owner: "owner-a".to_string(), + acquired_at: now - Duration::from_secs(40), + guard_id: Some(18), + remaining_ttl: Duration::from_secs(20), + }, + LockLeaseInfo { + resource: mixed_resource.clone(), + lock_type: LockType::Shared, + owner: "owner-c".to_string(), + acquired_at: now - Duration::from_secs(30), + guard_id: Some(13), + remaining_ttl: Duration::from_secs(25), + }, + LockLeaseInfo { + resource: exclusive_resource.clone(), + lock_type: LockType::Exclusive, + owner: "owner-d".to_string(), + acquired_at: now - Duration::from_secs(15), + guard_id: Some(12), + remaining_ttl: Duration::from_secs(18), + }, + LockLeaseInfo { + resource: replaced_resource.clone(), + lock_type: LockType::Exclusive, + owner: "owner-e".to_string(), + acquired_at: now - Duration::from_secs(30), + guard_id: Some(14), + remaining_ttl: Duration::from_secs(25), + }, + LockLeaseInfo { + resource: remaining_shared_resource.clone(), + lock_type: LockType::Shared, + owner: "owner-f".to_string(), + acquired_at: now - Duration::from_secs(30), + guard_id: Some(16), + remaining_ttl: Duration::from_secs(22), + }, + LockLeaseInfo { + resource: opaque_resource.clone(), + lock_type: LockType::Exclusive, + owner: "owner-g".to_string(), + acquired_at: now - Duration::from_secs(30), + guard_id: None, + remaining_ttl: Duration::from_secs(30), + }, + ], + vec![ + ( + rustfs_lock::ObjectLockInfo { + key: replaced_resource, + mode: LockMode::Exclusive, + owner: "owner-e".into(), + acquired_at: now - Duration::from_secs(5), + expires_at: now + Duration::from_secs(4), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 1, + Some(vec![15]), + ), + ( + rustfs_lock::ObjectLockInfo { + key: remaining_shared_resource, + mode: LockMode::Shared, + owner: "owner-f".into(), + acquired_at: now - Duration::from_secs(5), + expires_at: now + Duration::from_secs(3), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 1, + Some(vec![16]), + ), + ( + rustfs_lock::ObjectLockInfo { + key: exclusive_resource, + mode: LockMode::Exclusive, + owner: "owner-d".into(), + acquired_at: now - Duration::from_secs(15), + expires_at: now + Duration::from_secs(2), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 1, + Some(vec![12]), + ), + ( + rustfs_lock::ObjectLockInfo { + key: leased_resource, + mode: LockMode::Shared, + owner: "owner-a".into(), + acquired_at: now - Duration::from_secs(50), + expires_at: now + Duration::from_secs(1), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 2, + Some(vec![11, 18]), + ), + ( + rustfs_lock::ObjectLockInfo { + key: direct_resource, + mode: LockMode::Exclusive, + owner: "owner-b".into(), + acquired_at: now - Duration::from_secs(10), + expires_at: now + Duration::from_secs(7), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 1, + Some(vec![17]), + ), + ( + rustfs_lock::ObjectLockInfo { + key: mixed_resource, + mode: LockMode::Shared, + owner: "owner-c".into(), + acquired_at: now - Duration::from_secs(30), + expires_at: now + Duration::from_secs(9), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 2, + None, + ), + ( + rustfs_lock::ObjectLockInfo { + key: opaque_resource, + mode: LockMode::Exclusive, + owner: "owner-g".into(), + acquired_at: now - Duration::from_secs(4), + expires_at: now + Duration::from_secs(6), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 1, + None, + ), + ], + ); + + assert_eq!(response.total, 7); + let leased = response + .locks + .iter() + .find(|entry| entry.object == "shared-object") + .expect("lease-backed shared owner should be listed once"); + assert_eq!(leased.lock_type, "READ"); + assert_eq!(leased.elapsed_secs, 50); + assert_eq!(leased.ttl_secs, 20); + + let exclusive = response + .locks + .iter() + .find(|entry| entry.object == "write-object") + .expect("lease-backed exclusive holder should be listed"); + assert_eq!(exclusive.lock_type, "WRITE"); + assert_eq!(exclusive.ttl_secs, 18); + + let direct = response + .locks + .iter() + .find(|entry| entry.object == "direct-object") + .expect("direct fast lock should remain visible"); + assert_eq!(direct.ttl_secs, 7); + + let mixed = response + .locks + .iter() + .find(|entry| entry.object == "mixed-object") + .expect("mixed direct and leased shared holders should remain visible"); + assert_eq!(mixed.ttl_secs, 9); + + let replaced = response + .locks + .iter() + .find(|entry| entry.object == "replaced-object") + .expect("a replaced lease holder should remain visible"); + assert_eq!(replaced.ttl_secs, 4); + + let remaining_shared = response + .locks + .iter() + .find(|entry| entry.object == "remaining-shared-object") + .expect("an older surviving shared lease should remain lease-backed"); + assert_eq!(remaining_shared.ttl_secs, 22); + + let opaque = response + .locks + .iter() + .find(|entry| entry.object == "opaque-object") + .expect("generation-less holder should remain visible"); + assert_eq!(opaque.ttl_secs, 6); + } + + #[test] + fn top_locks_rejects_replaced_shared_generation() { + let now = SystemTime::UNIX_EPOCH + Duration::from_secs(100); + let resource = ObjectKey::new("bucket", "replaced-shared-object"); + let response = build_top_locks_response( + TOP_LOCKS_DEFAULT_LIMIT, + now, + vec![LockLeaseInfo { + resource: resource.clone(), + lock_type: LockType::Shared, + owner: "owner-a".to_string(), + acquired_at: now - Duration::from_secs(30), + guard_id: Some(1), + remaining_ttl: Duration::from_secs(20), + }], + vec![( + rustfs_lock::ObjectLockInfo { + key: resource, + mode: LockMode::Shared, + owner: "owner-a".into(), + acquired_at: now - Duration::from_secs(2), + expires_at: now + Duration::from_secs(4), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 1, + Some(vec![2]), + )], + ); + + let entry = response.locks.first().expect("replacement remains visible"); + assert_eq!(entry.ttl_secs, 4); + assert_eq!(entry.elapsed_secs, 2); + } + #[tokio::test] async fn collect_top_locks_reports_live_lock() { // Acquire a real lock through the global manager and confirm it surfaces. @@ -1115,20 +1498,20 @@ mod tests { // The fast-lock manager exposes the acquire API; if the lock subsystem is // disabled in this environment, the response must carry a capability note. let Some(fast) = manager.as_fast_lock_manager() else { - let response = collect_top_locks(TOP_LOCKS_DEFAULT_LIMIT); + let response = collect_top_locks(TOP_LOCKS_DEFAULT_LIMIT).await; assert!(response.capability_note.is_some() || response.locks.is_empty()); return; }; let guard = match fast.acquire_write_lock(key.clone(), "diag-owner").await { Ok(g) => g, Err(_) => { - let response = collect_top_locks(TOP_LOCKS_DEFAULT_LIMIT); + let response = collect_top_locks(TOP_LOCKS_DEFAULT_LIMIT).await; assert!(response.capability_note.is_some() || response.locks.is_empty()); return; } }; - let response = collect_top_locks(TOP_LOCKS_DEFAULT_LIMIT); + let response = collect_top_locks(TOP_LOCKS_DEFAULT_LIMIT).await; let found = response .locks .iter() @@ -1144,4 +1527,36 @@ mod tests { drop(guard); } + + #[tokio::test(start_paused = true)] + async fn collect_top_locks_uses_refreshed_local_lease() { + use rustfs_lock::{FastObjectLockManager, GlobalLockManager, LocalClient, LockClient, LockRequest}; + + let manager = Arc::new(GlobalLockManager::Enabled(Arc::new(FastObjectLockManager::new()))); + let client = Arc::new(LocalClient::with_manager(manager.clone())); + let request = LockRequest::new(ObjectKey::new("diag-bucket", "renewed-object"), LockType::Exclusive, "diag-owner") + .with_ttl(Duration::from_secs(30)); + let lock_id = request.lock_id.clone(); + assert!( + client + .acquire_lock(&request) + .await + .expect("local lock acquisition should succeed") + .success + ); + + tokio::time::advance(Duration::from_secs(20)).await; + assert!(client.refresh(&lock_id).await.expect("local lease refresh should succeed")); + + let clients: Vec> = vec![client.clone()]; + let response = collect_top_locks_with_clients(TOP_LOCKS_DEFAULT_LIMIT, manager, clients).await; + let entry = response + .locks + .iter() + .find(|entry| entry.bucket == "diag-bucket" && entry.object == "renewed-object") + .expect("refreshed local lock should be listed"); + assert!(entry.ttl_secs >= 29, "collector must use the refreshed lease deadline"); + + assert!(client.release(&lock_id).await.expect("local lock release should succeed")); + } } diff --git a/rustfs/src/admin/handlers/extensions.rs b/rustfs/src/admin/handlers/extensions.rs index bc7ec9dc5..6b3e07b26 100644 --- a/rustfs/src/admin/handlers/extensions.rs +++ b/rustfs/src/admin/handlers/extensions.rs @@ -14,7 +14,7 @@ use crate::admin::storage_api::cluster::CapabilityStatus; use crate::admin::{ - auth::validate_admin_request, + auth::authorize_admin_request, handlers::{cluster_snapshot, plugins_instances, system}, plugin_contract::{ PluginContractDomain, PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount, PluginInstanceEntry, @@ -22,8 +22,7 @@ use crate::admin::{ }, router::{AdminOperation, Operation, S3Router}, }; -use crate::auth::{check_key_valid, get_session_token}; -use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use crate::server::ADMIN_PREFIX; use http::{HeaderMap, HeaderValue, StatusCode}; use hyper::Method; use matchit::Params; @@ -183,42 +182,26 @@ fn map_extension_instance(instance: PluginInstanceEntry) -> ExtensionInstanceEnt } } +/// The pre-check keeps this endpoint's historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize_extension_catalog_request(req: &S3Request) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "authentication required")); - }; + } - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]).await?; + Ok(()) } +/// The pre-check keeps this endpoint's historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize_extension_instance_request(req: &S3Request) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "authentication required")); - }; + } - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(AdminAction::GetBucketTargetAction)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::GetBucketTargetAction)]).await?; + Ok(()) } fn build_json_response( @@ -320,6 +303,36 @@ mod tests { ); } + /// Both extension gates authorize through the shared admin gate, which reports + /// "get cred failed" for a credential-less request. The pre-check keeps the + /// message these endpoints have always returned (rustfs/backlog#1829). + #[tokio::test] + async fn extension_gates_keep_their_missing_credentials_message() { + let credential_less_request = || s3s::S3Request { + input: s3s::Body::from(String::new()), + method: http::Method::GET, + uri: http::Uri::from_static("/rustfs/admin/v4/extensions/catalog"), + headers: http::HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + for err in [ + super::authorize_extension_catalog_request(&credential_less_request()) + .await + .expect_err("a request without credentials must be rejected"), + super::authorize_extension_instance_request(&credential_less_request()) + .await + .expect_err("a request without credentials must be rejected"), + ] { + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("authentication required")); + } + } + #[test] fn builtin_ops_schemas_register_cleanly_in_runtime_registries() { let mut diagnostics_registry = rustfs_targets::OpsDiagnosticsRegistry::new(); diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 3d98e966a..9ae59353b 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -14,10 +14,9 @@ use crate::admin::auth::{authenticate_request, validate_admin_request}; use crate::admin::router::{AdminOperation, Operation, S3Router}; -use crate::admin::runtime_sources::{app_context_from_req, object_store_from_extensions}; +use crate::admin::runtime_sources::app_context_from_req; use crate::admin::storage_api::bucket::is_reserved_or_invalid_bucket; use crate::admin::storage_api::bucket::utils::is_valid_object_prefix; -use crate::admin::storage_api::contract::heal::HealOperations as _; use crate::server::ADMIN_PREFIX; use crate::server::RemoteAddr; use crate::storage::rpc::node_service::heal::{ @@ -1219,41 +1218,6 @@ fn validate_heal_request_mode(hip: &HealInitParams) -> S3Result<()> { Ok(()) } -fn should_handle_root_heal_directly(_hip: &HealInitParams) -> bool { - false -} - -fn map_root_heal_status(heal_err: Option) -> S3Result<()> { - match heal_err { - None => Ok(()), - Some(crate::admin::storage_api::error::StorageError::NoHealRequired) => { - info!( - event = EVENT_ADMIN_RESPONSE_EMITTED, - component = LOG_COMPONENT_ADMIN_API, - subsystem = LOG_SUBSYSTEM_HEAL_ADMIN, - operation = "root_heal", - result = "success", - state = "no_heal_required", - "admin response emitted" - ); - Ok(()) - } - Some(err) => { - warn!( - event = EVENT_ADMIN_REQUEST_FAILED, - component = LOG_COMPONENT_ADMIN_API, - subsystem = LOG_SUBSYSTEM_HEAL_ADMIN, - operation = "root_heal", - result = "failed", - reason = "root_heal_failed", - error = %err, - "admin request failed" - ); - Err(s3_error!(InternalError, "root heal failed: {err}")) - } - } -} - fn json_response(status: StatusCode, body: Vec) -> S3Response<(StatusCode, Body)> { let mut headers = HeaderMap::new(); headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); @@ -1358,50 +1322,6 @@ impl Operation for HealHandler { } }; let hip = extract_heal_init_params(&bytes, &req.uri, params)?; - // The heal channel currently models bucket/object work. Root heal reuses the - // existing format-heal path directly so `/v3/heal/` is accepted intentionally. - if should_handle_root_heal_directly(&hip) { - let Some(store) = object_store_from_extensions(&req.extensions) else { - warn!( - event = EVENT_ADMIN_REQUEST_FAILED, - component = LOG_COMPONENT_ADMIN_API, - subsystem = LOG_SUBSYSTEM_HEAL_ADMIN, - operation = "root_heal", - result = "failed", - reason = "server_not_initialized", - "admin request failed" - ); - return Err(s3_error!(InternalError, "server not initialized")); - }; - - let (_, heal_err) = store.heal_format(hip.hs.dry_run).await.map_err(|e| { - warn!( - event = EVENT_ADMIN_REQUEST_FAILED, - component = LOG_COMPONENT_ADMIN_API, - subsystem = LOG_SUBSYSTEM_HEAL_ADMIN, - operation = "root_heal", - result = "failed", - reason = "heal_format_failed", - error = %e, - "admin request failed" - ); - s3_error!(InternalError, "root heal failed: {e}") - })?; - - map_root_heal_status(heal_err)?; - let body = encode_heal_start_success("root-heal".to_string(), client_address)?; - info!( - event = EVENT_ADMIN_RESPONSE_EMITTED, - component = LOG_COMPONENT_ADMIN_API, - subsystem = LOG_SUBSYSTEM_HEAL_ADMIN, - operation = "root_heal", - result = "success", - state = "started", - "admin response emitted" - ); - - return Ok(json_response(StatusCode::OK, body)); - } validate_heal_request_mode(&hip)?; let response_operation = if hip.force_stop { "cancel_heal" @@ -1614,11 +1534,9 @@ mod tests { build_replacement_recovery_status_response, encode_background_heal_status, encode_heal_control_path, encode_heal_start_success, encode_heal_task_status, execute_after_heal_control_capability, heal_channel_response_items, heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id, json_response, - map_heal_response, map_root_heal_status, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status, - query_peer_replacement_recovery_status, reject_heal_admission, should_handle_root_heal_directly, - validate_heal_request_mode, validate_heal_target, + map_heal_response, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status, + query_peer_replacement_recovery_status, reject_heal_admission, validate_heal_request_mode, validate_heal_target, }; - use crate::admin::storage_api::error::StorageError; use crate::storage::rpc::node_service::heal::{ NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status, }; @@ -2086,48 +2004,63 @@ mod tests { } #[test] - fn test_should_handle_root_heal_directly_is_disabled_for_root_start_modes() { - assert!(!should_handle_root_heal_directly(&HealInitParams::default())); - assert!(!should_handle_root_heal_directly(&HealInitParams { - force_start: true, - ..Default::default() - })); - } - - #[test] - fn test_should_handle_root_heal_directly_skips_query_cancel_and_bucket_targets() { - assert!(!should_handle_root_heal_directly(&HealInitParams { - client_token: "heal-token".to_string(), - ..Default::default() - })); - assert!(!should_handle_root_heal_directly(&HealInitParams { - force_stop: true, - ..Default::default() - })); - assert!(!should_handle_root_heal_directly(&HealInitParams { - bucket: "bucket".to_string(), - ..Default::default() - })); - assert!(!should_handle_root_heal_directly(&HealInitParams { - hs: HealOpts { - pool: Some(1), - set: Some(2), + fn test_root_heal_shapes_route_through_cluster_coordination() { + // Root heal has no direct local store path: every start shape is either + // rejected by validate_heal_request_mode or submitted to the cluster + // heal channel as an Admin-sourced request (see HealHandler::call). + let accepted_root_starts = [ + HealInitParams { + hs: HealOpts { + recursive: true, + ..Default::default() + }, ..Default::default() }, - ..Default::default() - })); - } + HealInitParams { + force_start: true, + hs: HealOpts { + recursive: true, + ..Default::default() + }, + ..Default::default() + }, + HealInitParams { + hs: HealOpts { + pool: Some(1), + set: Some(2), + ..Default::default() + }, + ..Default::default() + }, + ]; + for hip in accepted_root_starts { + validate_heal_request_mode(&hip).expect("accepted root heal start must reach cluster coordination"); + let request = build_heal_channel_request(&hip); + assert_eq!(request.bucket, "", "root heal must stay cluster-scoped"); + assert_eq!(request.source, HealRequestSource::Admin); + assert!(!request.id.is_empty(), "cluster heal requests carry a dedup id"); + } - #[test] - fn test_map_root_heal_status_allows_no_heal_required() { - map_root_heal_status(Some(StorageError::NoHealRequired)).expect("NoHealRequired should stay non-fatal"); - } - - #[test] - fn test_map_root_heal_status_rejects_fatal_errors() { - let err = map_root_heal_status(Some(StorageError::Unexpected)).expect_err("fatal status must fail"); - assert_eq!(err.code(), &S3ErrorCode::InternalError); - assert!(err.to_string().contains("root heal failed: Unexpected error")); + // Shapes that cannot start a tracked heal (plain start, bare force_start + // without recursive, bare pool) are rejected instead of falling back to + // a direct local path. + for hip in [ + HealInitParams::default(), + HealInitParams { + force_start: true, + ..Default::default() + }, + HealInitParams { + hs: HealOpts { + pool: Some(1), + ..Default::default() + }, + ..Default::default() + }, + ] { + let err = validate_heal_request_mode(&hip).expect_err("unscoped root heal start must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } } #[test] diff --git a/rustfs/src/admin/handlers/is_admin.rs b/rustfs/src/admin/handlers/is_admin.rs index 4bbc1026a..7e29c4bff 100644 --- a/rustfs/src/admin/handlers/is_admin.rs +++ b/rustfs/src/admin/handlers/is_admin.rs @@ -46,6 +46,12 @@ impl Operation for IsAdminHandler { let access_key_to_check = input_cred.access_key.clone(); + // This endpoint reports a capability; it does not gate on one. The + // `is_allowed` result below becomes the `is_admin` field of a 200 + // response โ€” a caller without admin rights gets `{"is_admin": false}`, + // not a 403. Turning this into a rejection would change the API + // contract, so it must stay out of any shared-gate normalisation + // (backlog#1886). // Check if the user is admin: root user check, then evaluate through the policy engine let is_admin = if let Some(sys_cred) = current_action_credentials() { constant_time_eq(&access_key_to_check, &sys_cred.access_key) diff --git a/rustfs/src/admin/handlers/object_data_cache.rs b/rustfs/src/admin/handlers/object_data_cache.rs index d470e8b56..6e45e7cb6 100644 --- a/rustfs/src/admin/handlers/object_data_cache.rs +++ b/rustfs/src/admin/handlers/object_data_cache.rs @@ -21,12 +21,11 @@ //! that bucket, and with `bucket`+`object` it flushes that one identity โ€” the //! only remediation for a poisoned entry short of a node restart. -use crate::admin::auth::validate_admin_request; +use crate::admin::auth::authorize_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::current_object_data_cache; use crate::app::object_data_cache::ObjectDataCacheAdapter; -use crate::auth::{check_key_valid, get_session_token}; -use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use crate::server::ADMIN_PREFIX; use http::{HeaderMap, HeaderValue}; use hyper::{Method, StatusCode}; use matchit::Params; @@ -76,17 +75,14 @@ pub fn register_object_data_cache_route(r: &mut S3Router) -> std Ok(()) } +/// The pre-check keeps these endpoints' historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize(req: &S3Request, action: AdminAction) -> S3Result<()> { - let Some(input_cred) = req.credentials.as_ref() else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "missing credentials")); - }; - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - let remote_addr = req - .extensions - .get::>() - .and_then(|opt| opt.map(|addr| addr.0)); - validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await + } + authorize_admin_request(req, vec![Action::AdminAction(action)]).await?; + Ok(()) } fn json_response(body: &T) -> S3Result> { @@ -208,6 +204,30 @@ mod tests { assert_eq!(invalidation_outcome(&ObjectDataCacheInvalidationResult::NoOp), ("noop", 0)); } + /// These endpoints authorize through the shared admin gate, which reports + /// "get cred failed" for a credential-less request. The pre-check keeps the + /// message they have always returned (rustfs/backlog#1829). + #[tokio::test] + async fn authorize_keeps_its_missing_credentials_message() { + let req = S3Request { + input: Body::from(String::new()), + method: Method::GET, + uri: "/rustfs/admin/v3/object-data-cache/stats".parse().expect("uri should parse"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + let err = authorize(&req, AdminAction::ServerInfoAdminAction) + .await + .expect_err("a request without credentials must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("missing credentials")); + } + #[test] fn stats_handler_requires_server_info_action() { // Guard the auth contract: the stats endpoint is a read, the flush diff --git a/rustfs/src/admin/handlers/plugins_catalog.rs b/rustfs/src/admin/handlers/plugins_catalog.rs index fc25e9508..8045ce0e1 100644 --- a/rustfs/src/admin/handlers/plugins_catalog.rs +++ b/rustfs/src/admin/handlers/plugins_catalog.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::admin::{ - auth::validate_admin_request, + auth::authorize_admin_request, plugin_contract::{ PluginCatalogAdminDiscovery, PluginCatalogDomainEntry, PluginCatalogEntry, PluginCatalogResponse, PluginContractDomain, PluginContractEntrypointKind, PluginContractPackaging, PluginDistributionContract, PluginRuntimeContract, @@ -21,8 +21,7 @@ use crate::admin::{ router::{AdminOperation, Operation, S3Router}, runtime_sources::default_admin_usecase, }; -use crate::auth::{check_key_valid, get_session_token}; -use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use crate::server::ADMIN_PREFIX; use http::{HeaderMap, HeaderValue, StatusCode}; use hyper::Method; use matchit::Params; @@ -114,23 +113,15 @@ fn merge_catalog_descriptor(plugins: &mut HashMap<&'static str, PluginCatalogEnt } } +/// The pre-check keeps this endpoint's historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize_plugin_catalog_request(req: &S3Request) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "authentication required")); - }; + } - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]).await?; + Ok(()) } fn build_json_response( @@ -175,6 +166,30 @@ mod tests { ); } + /// This endpoint authorizes through the shared admin gate, which reports + /// "get cred failed" for a credential-less request. The pre-check keeps the + /// message it has always returned (rustfs/backlog#1829). + #[tokio::test] + async fn plugin_catalog_gate_keeps_its_missing_credentials_message() { + let req = s3s::S3Request { + input: s3s::Body::from(String::new()), + method: http::Method::GET, + uri: http::Uri::from_static("/rustfs/admin/v4/plugins/catalog"), + headers: http::HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + let err = super::authorize_plugin_catalog_request(&req) + .await + .expect_err("a request without credentials must be rejected"); + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("authentication required")); + } + #[test] fn plugin_catalog_contains_representative_builtin_targets() { let response = build_catalog_response(); diff --git a/rustfs/src/admin/handlers/plugins_instances.rs b/rustfs/src/admin/handlers/plugins_instances.rs index f66f5b693..548720651 100644 --- a/rustfs/src/admin/handlers/plugins_instances.rs +++ b/rustfs/src/admin/handlers/plugins_instances.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::admin::{ - auth::validate_admin_request, + auth::authorize_admin_request, handlers::audit_runtime_config::{load_server_config_from_store, remove_audit_target_config, set_audit_target_config}, handlers::notify_runtime_access::{ load_notification_config_snapshot, remove_notification_target_config, set_notification_target_config, @@ -29,10 +29,9 @@ use crate::admin::{ }, router::{AdminOperation, Operation, S3Router}, }; -use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ - ADMIN_PREFIX, RemoteAddr, is_audit_module_enabled, is_notify_module_enabled, refresh_audit_module_enabled, - refresh_notify_module_enabled, refresh_persisted_module_switches_from_store, + ADMIN_PREFIX, is_audit_module_enabled, is_notify_module_enabled, refresh_audit_module_enabled, refresh_notify_module_enabled, + refresh_persisted_module_switches_from_store, }; use hyper::{Method, StatusCode}; use matchit::Params; @@ -563,42 +562,26 @@ fn plugin_instance_matches_query(instance: &PluginInstanceEntry, query: &str) -> .any(|field| field.to_ascii_lowercase().contains(&query)) } +/// The pre-check keeps this endpoint's historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize_plugin_instance_request(req: &S3Request) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "authentication required")); - }; + } - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(AdminAction::GetBucketTargetAction)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::GetBucketTargetAction)]).await?; + Ok(()) } +/// The pre-check keeps this endpoint's historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize_plugin_instance_write_request(req: &S3Request) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "authentication required")); - }; + } - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(AdminAction::SetBucketTargetAction)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::SetBucketTargetAction)]).await?; + Ok(()) } fn plugin_instance_mutation_block_reason( @@ -942,6 +925,36 @@ mod tests { ); } + /// Both instance gates authorize through the shared admin gate, which reports + /// "get cred failed" for a credential-less request. The pre-check keeps the + /// message these endpoints have always returned (rustfs/backlog#1829). + #[tokio::test] + async fn plugin_instance_gates_keep_their_missing_credentials_message() { + let credential_less_request = || S3Request { + input: Body::from(String::new()), + method: Method::GET, + uri: Uri::from_static("/rustfs/admin/v4/plugins/instances"), + headers: HeaderMap::new(), + extensions: Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + for err in [ + super::authorize_plugin_instance_request(&credential_less_request()) + .await + .expect_err("a request without credentials must be rejected"), + super::authorize_plugin_instance_write_request(&credential_less_request()) + .await + .expect_err("a request without credentials must be rejected"), + ] { + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("authentication required")); + } + } + #[test] fn configured_instance_without_runtime_appears_offline() { let config = Config(HashMap::from([( diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index 890a11800..106c25790 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -455,12 +455,6 @@ pub fn register_replication_route(r: &mut S3Router) -> std::io:: async fn validate_replication_admin_request(req: &S3Request, action: AdminAction) -> S3Result { authorize_admin_request(req, vec![Action::AdminAction(action)]).await } - -#[allow(dead_code)] -fn is_local_host(_host: String) -> bool { - false -} - pub(crate) async fn cluster_replication_stats(bucket: &str, context: Option>) -> BucketStats { let Some(stats) = current_replication_stats_handle_for_context(context.clone()) else { return BucketStats::default(); diff --git a/rustfs/src/admin/handlers/service_account.rs b/rustfs/src/admin/handlers/service_account.rs index ac5075068..938ba7836 100644 --- a/rustfs/src/admin/handlers/service_account.rs +++ b/rustfs/src/admin/handlers/service_account.rs @@ -359,6 +359,13 @@ impl Operation for AddServiceAccount { return Err(s3_error!(InvalidRequest, "iam not init")); }; + // This family deliberately calls `is_allowed` directly instead of going + // through `validate_admin_request`, and must keep doing so + // (backlog#1886). The shared helper returns as soon as *any* candidate + // action is allowed โ€” an OR. The checks here are an AND: each one must + // pass, and a later stage additionally needs `owner` for the GHSA-5354 + // parent-scope guard and drives `deny_only` dynamically. Replacing these + // with the shared gate would widen authorization. if !iam_store .is_allowed(&Args { account: &cred.access_key, diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 219ebbffe..1bb0379df 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -126,6 +126,10 @@ const SITE_REPLICATION_JOIN_ADMISSION_LOCK_PATH: &str = "config/site-replication const SITE_REPL_ADD_SUCCESS: &str = "Requested sites were configured for replication successfully."; const SITE_REPL_EDIT_SUCCESS: &str = "Requested site was updated successfully."; const SITE_REPL_REMOVE_SUCCESS: &str = "Requested site(s) were removed from cluster replication successfully."; +/// Local removal committed, but at least one peer could not be told. The +/// cluster is diverged until the removal finishes โ€” the reconcile tick keeps +/// retrying it, and `replicate info` reports the pending operation meanwhile. +const SITE_REPL_REMOVE_PARTIAL: &str = "Partial"; const SITE_REPL_RESYNC_START: &str = "start"; const SITE_REPL_RESYNC_CANCEL: &str = "cancel"; const SITE_REPL_RESYNC_STATUS: &str = "status"; @@ -713,6 +717,16 @@ struct SRPeerJoinResponse { peer: PeerInfo, #[serde(rename = "initialSyncErrorMessage", default, skip_serializing_if = "String::is_empty")] initial_sync_error_message: String, + /// Whether the receiving site actually applied this join. + /// + /// Three-valued on purpose. `None` means the peer did not report โ€” MinIO + /// answers a successful `SRPeerJoin` with an empty body, and RustFS peers + /// older than this field say nothing either โ€” so the initiator must NOT + /// read it as a failure. `Some(false)` is an explicit no-op: the peer had + /// already moved past the snapshot it was sent and wrote nothing, which + /// used to be indistinguishable from success (rustfs/rustfs#5963). + #[serde(default, skip_serializing_if = "Option::is_none")] + applied: Option, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -2568,6 +2582,21 @@ fn apply_peer_join( state.peers = normalize_join_peers_for_local(local_peer, join_req.peers); initialize_join_peer_sync_state(&mut state.peers, defer_sync_state_enable); state.sync_state_initialized = true; + // An accepted join supersedes a half-finished removal this site started: + // the sender's snapshot IS the new topology, while the pending record only + // exists to keep notifying peers about the OLD one. Leaving it set is what + // kept a recovered site rejecting every peer bucket-op forever โ€” + // `SRPeerBucketOpsHandler` short-circuits on `pending_remove` BEFORE it + // consults `enabled()`, so a successful re-add restored the topology on + // both sides while replication stayed dead (rustfs/rustfs#5963). + // + // Safe against a concurrent removal: `SiteReplicationRemoveHandler` and + // the join admission both hold the lifecycle guard, so a join is only ever + // admitted before that handler starts or after it has returned. + // + // Deliberately NOT cleared here: the peer-edit high-water marks (see this + // function's doc comment) โ€” those fence edit ordering, not lifecycle. + state.pending_remove = None; state.name = state .peers .get(&local_peer.deployment_id) @@ -3002,8 +3031,17 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin { - if state.pending_endpoint_refresh.is_some() || state.pending_remove.is_some() || state.pending_rotation.is_some() - { + if state.pending_endpoint_refresh.is_some() || state.pending_rotation.is_some() { + return; + } + // A removal whose peers were unreachable is the one pending + // marker that nothing else re-drives, and it wedges the site + // while it sits there. Push it forward here rather than giving + // up the round (rustfs/rustfs#5963). The reconcilers below + // still skip this round either way: the topology is only + // settled once the removal clears, and the next tick sees it. + if let Some(pending_remove) = state.pending_remove.clone() { + resume_pending_remove(&state, &pending_remove).await; return; } } @@ -7041,15 +7079,31 @@ async fn dequeue_site_replication_retry_event_for_generation(peer: &PeerInfo, pa } } +/// The removal's client-facing verdict. +/// +/// A fully-notified removal keeps answering with the historical success string, +/// byte for byte, so healthy runs stay wire-identical for every existing +/// client. Only the path that used to LIE โ€” peers that could not be notified, +/// reported as unqualified success while the cluster silently diverged +/// (rustfs/rustfs#5963) โ€” now says `Partial`, matching the vocabulary +/// `SRRotateServiceAccountHandler` already uses for the same situation. fn site_replication_remove_status(peer_errors: &[String]) -> ReplicateRemoveStatus { + if peer_errors.is_empty() { + return ReplicateRemoveStatus { + status: SITE_REPL_REMOVE_SUCCESS.to_string(), + err_detail: String::new(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }; + } + + let summaries: Vec = peer_errors.iter().map(|error| summarize_peer_error_detail(error)).collect(); ReplicateRemoveStatus { - status: SITE_REPL_REMOVE_SUCCESS.to_string(), - err_detail: if peer_errors.is_empty() { - String::new() - } else { - let summaries: Vec = peer_errors.iter().map(|error| summarize_peer_error_detail(error)).collect(); - summarize_peer_error_detail(&format!("failed to notify {} peer(s): {}", summaries.len(), summaries.join("; "))) - }, + status: SITE_REPL_REMOVE_PARTIAL.to_string(), + err_detail: summarize_peer_error_detail(&format!( + "failed to notify {} peer(s): {}", + summaries.len(), + summaries.join("; ") + )), api_version: Some(SITE_REPL_API_VERSION.to_string()), } } @@ -7226,6 +7280,137 @@ async fn clear_pending_remove(remove_id: &str) -> S3Result<()> { .await } +/// Push a half-finished removal one step forward: notify every peer that has +/// not acked yet, then finalize locally if that completed the set. Returns the +/// per-peer failures and whether the removal is now finished. +/// +/// Shared by the operator-driven `SiteReplicationRemoveHandler` and the +/// reconcile tick. The tick is what makes this self-healing: a removal whose +/// peers were unreachable used to sit in `pending_remove` forever, and that one +/// field gates every peer bucket-op (`SRPeerBucketOpsHandler` checks it first) +/// plus every reconciler โ€” so the site stayed wedged until an operator happened +/// to re-run `replicate remove` (rustfs/rustfs#5963). +/// +/// Callers must hold the lifecycle guard: this both notifies peers and, on the +/// final step, takes the bucket-op write lock to clean up local rules. +async fn drive_pending_remove(pending_remove: &PendingRemove, local_peer: &PeerInfo) -> S3Result<(Vec, bool)> { + let mut peer_errors = Vec::new(); + let mut secret_candidates = pending_remove.secret_candidates.clone(); + if pending_remove.service_account_access_key.is_empty() { + peer_errors.push("site replication service account unavailable".to_string()); + } else if let Ok(service_account_secret_key) = + site_replicator_service_account_secret(&pending_remove.service_account_access_key).await + { + record_pending_remove_secret_candidate(&pending_remove.id, service_account_secret_key.clone()).await?; + push_unique_secret_candidate(&mut secret_candidates, service_account_secret_key); + } + + if secret_candidates.is_empty() { + peer_errors.push("site replication service account secret unavailable".to_string()); + } else { + for peer in pending_remove.original_peers.values() { + if same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) + || pending_remove.acked_deployment_ids.contains(&peer.deployment_id) + { + continue; + } + if let Err(err) = send_peer_admin_request_with_secret_candidates( + &runtime_peer_connection(peer)?, + SITE_REPLICATION_PEER_REMOVE_PATH, + &pending_remove.service_account_access_key, + &secret_candidates, + &pending_remove.req, + ) + .await + { + let err_detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint)); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + peer = %peer.endpoint, + result = "peer_remove_notification_failed", + error = %err_detail, + "admin site replication state" + ); + peer_errors.push(err_detail); + } else { + mark_pending_remove_peer_acked(&pending_remove.id, &peer.deployment_id).await?; + } + } + } + + let finalize_candidate = pending_remove_ready_to_finalize(&pending_remove.id, local_peer).await?; + let complete = if let Some(finalized_remove) = finalize_candidate { + let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.write().await; + let removed_deployment_ids = removed_deployment_ids_for_pending_remove(&finalized_remove, local_peer); + match cleanup_removed_site_replication_buckets(&removed_deployment_ids).await { + Ok(removed) => { + if removed > 0 { + info!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + removed, + result = "remove_cleanup_completed", + "admin site replication state" + ); + } + clear_pending_remove(&pending_remove.id).await?; + true + } + Err(err) => { + peer_errors.push(summarize_peer_error_detail(&format!("local remove cleanup failed: {err}"))); + false + } + } + } else { + false + }; + + Ok((peer_errors, complete)) +} + +/// The reconcile tick's half of [`drive_pending_remove`]: resume the removal +/// this site could not finish, and report the outcome. Runs under the tick's +/// lifecycle guard, which is what keeps it from racing an operator re-running +/// `replicate remove` (that handler takes the same guard). +async fn resume_pending_remove(state: &SiteReplicationState, pending_remove: &PendingRemove) { + let local_peer = current_local_runtime_peer(state); + match drive_pending_remove(pending_remove, &local_peer).await { + Ok((peer_errors, complete)) => { + if complete && peer_errors.is_empty() { + info!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "pending_remove_resumed", + "admin site replication state" + ); + } else { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "pending_remove_still_pending", + error_count = peer_errors.len(), + "admin site replication state" + ); + } + } + Err(err) => { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "pending_remove_resume_failed", + error = ?err, + "admin site replication state" + ); + } + } +} + fn removed_deployment_ids_for_pending_remove(pending: &PendingRemove, local_peer: &PeerInfo) -> HashSet { if pending.req.remove_all || pending.req.site_names.iter().any(|name| name == &local_peer.name) { return pending @@ -9656,9 +9841,12 @@ pub struct SiteReplicationAddHandler {} /// peer identity from the add preflight metainfo in that case. fn parse_peer_join_response(body: &[u8], fallback_peer: PeerInfo) -> Result { if body.iter().all(u8::is_ascii_whitespace) { + // MinIO's empty-body success. `applied` stays `None`: the peer told us + // nothing, which must not be reported as a no-op join. return Ok(SRPeerJoinResponse { peer: fallback_peer, initial_sync_error_message: String::new(), + applied: None, }); } serde_json::from_slice(body) @@ -9761,6 +9949,19 @@ impl Operation for SiteReplicationAddHandler { if !join_response.initial_sync_error_message.is_empty() { initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message)); } + // An explicit no-op join. The peer answered 200 but wrote nothing โ€” + // its persisted state is already newer than the snapshot it was + // sent โ€” so the add is only PARTIALLY configured and saying + // "configured successfully" would be a lie (rustfs/rustfs#5963). + // `None` (a MinIO peer, or one older than the field) is not a + // no-op signal and is deliberately not reported. + if join_response.applied == Some(false) { + initial_sync_errors.push(format!( + "{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \ + the site is not configured against this peer", + site.endpoint + )); + } state = reconcile_peer_with_actual_identity(state, join_response.peer); let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| { S3Error::with_message( @@ -9933,79 +10134,7 @@ impl Operation for SiteReplicationRemoveHandler { .await? }; - let mut peer_errors = Vec::new(); - let mut secret_candidates = pending_remove.secret_candidates.clone(); - if pending_remove.service_account_access_key.is_empty() { - peer_errors.push("site replication service account unavailable".to_string()); - } else if let Ok(service_account_secret_key) = - site_replicator_service_account_secret(&pending_remove.service_account_access_key).await - { - record_pending_remove_secret_candidate(&pending_remove.id, service_account_secret_key.clone()).await?; - push_unique_secret_candidate(&mut secret_candidates, service_account_secret_key); - } - - if secret_candidates.is_empty() { - peer_errors.push("site replication service account secret unavailable".to_string()); - } else { - for peer in pending_remove.original_peers.values() { - if same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) - || pending_remove.acked_deployment_ids.contains(&peer.deployment_id) - { - continue; - } - if let Err(err) = send_peer_admin_request_with_secret_candidates( - &runtime_peer_connection(peer)?, - SITE_REPLICATION_PEER_REMOVE_PATH, - &pending_remove.service_account_access_key, - &secret_candidates, - &pending_remove.req, - ) - .await - { - let err_detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint)); - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - peer = %peer.endpoint, - result = "peer_remove_notification_failed", - error = %err_detail, - "admin site replication state" - ); - peer_errors.push(err_detail); - } else { - mark_pending_remove_peer_acked(&pending_remove.id, &peer.deployment_id).await?; - } - } - } - - let finalize_candidate = pending_remove_ready_to_finalize(&pending_remove.id, &local_peer).await?; - let complete = if let Some(finalized_remove) = finalize_candidate { - let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.write().await; - let removed_deployment_ids = removed_deployment_ids_for_pending_remove(&finalized_remove, &local_peer); - match cleanup_removed_site_replication_buckets(&removed_deployment_ids).await { - Ok(removed) => { - if removed > 0 { - info!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - removed, - result = "remove_cleanup_completed", - "admin site replication state" - ); - } - clear_pending_remove(&pending_remove.id).await?; - true - } - Err(err) => { - peer_errors.push(summarize_peer_error_detail(&format!("local remove cleanup failed: {err}"))); - false - } - } - } else { - false - }; + let (mut peer_errors, complete) = drive_pending_remove(&pending_remove, &local_peer).await?; if !complete && peer_errors.is_empty() { peer_errors.push("site replication remove is still pending".to_string()); } @@ -10019,6 +10148,25 @@ impl Operation for SiteReplicationRemoveHandler { } } +/// The `replicate info` projection. +/// +/// Carries the peer-facing health this endpoint used to omit entirely: a peer +/// rejecting every operation, or a removal stuck mid-flight, left `info` +/// reporting a perfectly healthy cluster while replication was dead โ€” both were +/// only visible through `replicate status --json` (rustfs/rustfs#5963). Split +/// out so that omission is a test failure rather than an invisible regression. +fn site_replication_info_for(state: &SiteReplicationState, local_peer: &PeerInfo) -> SiteReplicationInfo { + SiteReplicationInfo { + enabled: state.enabled(), + name: local_peer.name.clone(), + sites: state.peers.values().cloned().collect(), + service_account_access_key: state.service_account_access_key.clone(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + retry_stats: retry_stats_for_state(state), + pending_operation: pending_operation_for_state(state, local_peer), + } +} + pub struct SiteReplicationInfoHandler {} #[async_trait::async_trait] @@ -10027,14 +10175,7 @@ impl Operation for SiteReplicationInfoHandler { validate_site_replication_admin_request(&req, AdminAction::SiteReplicationInfoAction).await?; let state = load_site_replication_state().await?; let local_peer = current_local_peer(&req, &state); - let info = SiteReplicationInfo { - enabled: state.enabled(), - name: local_peer.name, - sites: state.peers.values().cloned().collect(), - service_account_access_key: state.service_account_access_key, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - }; - json_response(&info) + json_response(&site_replication_info_for(&state, &local_peer)) } } @@ -10253,6 +10394,28 @@ async fn apply_peer_join_service_account(join_req: SRPeerJoinReq) -> S3Result<() Ok(()) } +/// The answer to a join this site refused to apply because it had already +/// moved past the sender's snapshot. Split out so the verdict itself is +/// testable: answering `applied: Some(true)` here (or omitting the field) is +/// exactly the silent no-op that made `replicate add` report success against a +/// peer that wrote nothing (rustfs/rustfs#5963). +fn superseded_join_response(peer: PeerInfo) -> SRPeerJoinResponse { + SRPeerJoinResponse { + peer, + initial_sync_error_message: String::new(), + applied: Some(false), + } +} + +/// The answer to a join this site committed. +fn applied_join_response(peer: PeerInfo, initial_sync_error_message: String) -> SRPeerJoinResponse { + SRPeerJoinResponse { + peer, + initial_sync_error_message, + applied: Some(true), + } +} + #[async_trait::async_trait] impl Operation for SRPeerJoinHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { @@ -10275,10 +10438,14 @@ impl Operation for SRPeerJoinHandler { let (state, local_peer) = match committed { PeerJoinOutcome::Applied(state, local_peer) => (*state, local_peer), PeerJoinOutcome::Superseded(peer) => { - return json_response(&SRPeerJoinResponse { - peer, - ..Default::default() - }); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "join_superseded", + "admin site replication state" + ); + return json_response(&superseded_join_response(peer)); } }; // Fix 1 (receiving side): ensure the joining peer also sets up replication for any @@ -10297,10 +10464,10 @@ impl Operation for SRPeerJoinHandler { "admin site replication state" ); } - json_response(&SRPeerJoinResponse { - peer: state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer), - initial_sync_error_message: backfill_errors.render(), - }) + json_response(&applied_join_response( + state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer), + backfill_errors.render(), + )) } } @@ -11344,7 +11511,11 @@ impl Operation for SRRotateServiceAccountHandler { { continue; } - if let Err(err) = send_peer_admin_request_with_secret_candidates( + // A superseded join returns BEFORE `apply_iam`, so a no-op answer + // means the peer never installed the new secret. Acking it would + // finalize a rotation half the mesh cannot authenticate against + // (rustfs/rustfs#5963). + let rotation_error = match send_peer_admin_request_with_secret_candidates( &runtime_peer_connection(peer)?, SITE_REPLICATION_PEER_JOIN_PATH, &pending_rotation.access_key, @@ -11353,7 +11524,20 @@ impl Operation for SRRotateServiceAccountHandler { ) .await { - let detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint)); + Err(err) => Some(summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint))), + Ok(body) => match parse_peer_join_response(&body, peer.clone()) { + Ok(response) if response.applied == Some(false) => Some(summarize_peer_error_detail(&format!( + "{}: peer did not apply the rotation join (its site replication state is newer than the snapshot it \ + was sent); the new service account secret was not installed", + peer.endpoint + ))), + // Unparseable bodies keep the pre-existing behaviour: the + // transport succeeded, and MinIO peers answer with an empty + // body this helper already tolerates. + Ok(_) | Err(_) => None, + }, + }; + if let Some(detail) = rotation_error { warn!( event = EVENT_ADMIN_SITE_REPLICATION_STATE, component = LOG_COMPONENT_ADMIN, @@ -15639,9 +15823,17 @@ mod tests { site_replication_remove_status(&["peer request to https://remote.example.com failed with 403 Forbidden".to_string()]); assert!(state.peers.is_empty()); - assert_eq!(status.status, SITE_REPL_REMOVE_SUCCESS); + assert_eq!( + status.status, SITE_REPL_REMOVE_PARTIAL, + "a removal whose peer could not be notified must not report unqualified success" + ); assert!(status.err_detail.contains("failed to notify 1 peer")); assert!(status.err_detail.contains("403 Forbidden")); + + // The fully-notified path stays byte-identical for existing clients. + let clean = site_replication_remove_status(&[]); + assert_eq!(clean.status, SITE_REPL_REMOVE_SUCCESS); + assert!(clean.err_detail.is_empty()); } #[test] @@ -16982,16 +17174,22 @@ mod tests { assert_eq!(response.peer.deployment_id, "remote-deployment"); assert_eq!(response.peer.endpoint, "https://remote.example.com"); assert!(response.initial_sync_error_message.is_empty()); + assert_eq!( + response.applied, None, + "a MinIO empty-body success reports nothing; it must not read as a no-op join" + ); } let json = serde_json::to_vec(&SRPeerJoinResponse { peer: peer("actual", "https://actual.example.com"), initial_sync_error_message: "sync failed".to_string(), + applied: Some(true), }) .expect("serialize join response"); let response = parse_peer_join_response(&json, fallback.clone()).expect("parse join response body"); assert_eq!(response.peer.endpoint, "https://actual.example.com"); assert_eq!(response.initial_sync_error_message, "sync failed"); + assert_eq!(response.applied, Some(true)); assert!(parse_peer_join_response(b"not-json", fallback).is_err()); } @@ -17712,13 +17910,319 @@ mod tests { .expect("parse legacy peer join response"); assert!(response.initial_sync_error_message.is_empty()); + assert_eq!( + response.applied, None, + "a peer older than the field says nothing about whether it applied the join" + ); let value = serde_json::to_value(SRPeerJoinResponse { peer: peer("remote", "https://remote.example.com"), initial_sync_error_message: "bucket setup failed".to_string(), + applied: Some(true), }) .expect("serialize peer join response"); assert_eq!(value.get("initialSyncErrorMessage").and_then(Value::as_str), Some("bucket setup failed")); + assert_eq!(value.get("applied").and_then(Value::as_bool), Some(true)); + + // An unset verdict must not appear on the wire, so a peer that never + // learned the field keeps deserializing byte-identical payloads. + let value = serde_json::to_value(SRPeerJoinResponse { + peer: peer("remote", "https://remote.example.com"), + initial_sync_error_message: String::new(), + applied: None, + }) + .expect("serialize peer join response"); + assert!(value.get("applied").is_none(), "an unset verdict must be omitted: {value}"); + } + + /// rustfs/rustfs#5963: a removal that could not notify its peers leaves + /// `pending_remove` set, and that field alone makes `SRPeerBucketOpsHandler` + /// reject every peer operation โ€” before it ever consults `enabled()`. A + /// later join restored the topology but left the marker, so a "successful" + /// re-add produced a cluster that reported Enabled/2-sites on both sides + /// while replication stayed dead. The join must clear it. + #[test] + fn peer_join_clears_a_stuck_pending_remove() { + let local = PeerInfo { + deployment_id: "site-b".to_string(), + ..peer("site-b", "https://site-b.example.com") + }; + let remote = PeerInfo { + deployment_id: "site-a".to_string(), + ..peer("site-a", "https://site-a.example.com") + }; + let mut state = SiteReplicationState { + peers: BTreeMap::from([(local.deployment_id.clone(), local.clone())]), + pending_remove: Some(PendingRemove { + id: "stuck-remove".to_string(), + req: SRRemoveReq { + remove_all: true, + ..Default::default() + }, + service_account_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + secret_candidates: Vec::new(), + original_peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + acked_deployment_ids: BTreeSet::new(), + updated_at: Some(OffsetDateTime::now_utc()), + }), + ..Default::default() + }; + + apply_peer_join( + &mut state, + &local, + SRPeerJoinReq { + svc_acct_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + svc_acct_secret_key: "svc-secret".to_string(), + svc_acct_parent: "root".to_string(), + peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote), + ]), + updated_at: Some(OffsetDateTime::now_utc()), + }, + false, + ); + + assert!( + state.pending_remove.is_none(), + "an accepted join supersedes the half-finished removal it lands on" + ); + assert!(state.enabled(), "the join restores the two-site topology"); + // The guard `SRPeerBucketOpsHandler` evaluates, asserted directly: with + // the marker cleared and the topology back, peer bucket-ops are + // admitted again. + assert!( + state.pending_remove.is_none() && state.enabled(), + "the bucket-ops admission predicate must now pass" + ); + } + + /// The fence marks are lifecycle-independent and must survive the clearing + /// above โ€” wiping them would reopen the rollback window the fence closes. + #[test] + fn peer_join_clearing_pending_remove_keeps_edit_generation_marks() { + let local = PeerInfo { + deployment_id: "site-b".to_string(), + ..peer("site-b", "https://site-b.example.com") + }; + let remote = PeerInfo { + deployment_id: "site-a".to_string(), + ..peer("site-a", "https://site-a.example.com") + }; + let mut state = SiteReplicationState { + peers: BTreeMap::from([(local.deployment_id.clone(), local.clone())]), + applied_edit_generations: BTreeMap::from([(remote.deployment_id.clone(), 7)]), + pending_remove: Some(PendingRemove { + id: "stuck-remove".to_string(), + req: SRRemoveReq { + remove_all: true, + ..Default::default() + }, + service_account_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + secret_candidates: Vec::new(), + original_peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + acked_deployment_ids: BTreeSet::new(), + updated_at: Some(OffsetDateTime::now_utc()), + }), + ..Default::default() + }; + + apply_peer_join( + &mut state, + &local, + SRPeerJoinReq { + svc_acct_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + svc_acct_secret_key: "svc-secret".to_string(), + svc_acct_parent: "root".to_string(), + peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + updated_at: Some(OffsetDateTime::now_utc()), + }, + false, + ); + + assert!(state.pending_remove.is_none()); + assert_eq!( + state.applied_edit_generations.get(&remote.deployment_id), + Some(&7), + "clearing the lifecycle marker must not touch the ordering fence" + ); + } + + /// rustfs/rustfs#5963: the two join verdicts must be distinguishable on the + /// wire. `Some(true)`/`Some(false)` is what lets the initiator tell a real + /// configuration from a 200 that wrote nothing; flipping either one back to + /// an unset verdict re-hides the no-op. + #[test] + fn join_verdicts_are_distinguishable_on_the_wire() { + let remote = peer("remote", "https://remote.example.com"); + + let superseded = superseded_join_response(remote.clone()); + assert_eq!( + superseded.applied, + Some(false), + "a join this site refused to apply must say so explicitly" + ); + assert!(superseded.initial_sync_error_message.is_empty()); + + let applied = applied_join_response(remote, "bucket setup failed".to_string()); + assert_eq!(applied.applied, Some(true)); + assert_eq!(applied.initial_sync_error_message, "bucket setup failed"); + + // Round-tripping through the wire keeps the two apart โ€” the initiator + // only ever sees the serialized form. + let decoded: SRPeerJoinResponse = + serde_json::from_slice(&serde_json::to_vec(&superseded_join_response(peer("r", "https://r.example.com"))).unwrap()) + .expect("round-trip superseded verdict"); + assert_eq!(decoded.applied, Some(false)); + } + + /// rustfs/rustfs#5963: a stuck removal must be visible on the endpoint + /// operators actually run. `replicate info` used to report only + /// `enabled: false`, which reads as "never configured" rather than "a + /// removal is wedged here and this site rejects every peer operation". + #[test] + fn site_replication_info_reports_a_wedged_removal() { + let local = PeerInfo { + deployment_id: "site-b".to_string(), + ..peer("site-b", "https://site-b.example.com") + }; + let remote = PeerInfo { + deployment_id: "site-a".to_string(), + ..peer("site-a", "https://site-a.example.com") + }; + let state = SiteReplicationState { + name: "site-b".to_string(), + peers: BTreeMap::from([(local.deployment_id.clone(), local.clone())]), + pending_remove: Some(PendingRemove { + id: "stuck-remove".to_string(), + req: SRRemoveReq { + remove_all: true, + ..Default::default() + }, + service_account_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + secret_candidates: Vec::new(), + original_peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + acked_deployment_ids: BTreeSet::new(), + updated_at: Some(OffsetDateTime::now_utc()), + }), + ..Default::default() + }; + + let info = site_replication_info_for(&state, &local); + assert!(!info.enabled, "the peer set is already torn down"); + let pending = info + .pending_operation + .as_ref() + .expect("a wedged removal must surface as a pending operation"); + assert_eq!(pending.operation, "remove"); + assert!( + pending.pending_peers.contains(&remote.deployment_id), + "the peer that was never notified must be named: {pending:?}" + ); + } + + /// The source side of the same failure: peer operations are being rejected, + /// the topology still looks like a healthy two-site cluster, and `info` has + /// to say the deliveries are failing. + #[test] + fn site_replication_info_reports_failing_peer_deliveries() { + let local = PeerInfo { + deployment_id: "site-a".to_string(), + ..peer("site-a", "https://site-a.example.com") + }; + let remote = PeerInfo { + deployment_id: "site-b".to_string(), + ..peer("site-b", "https://site-b.example.com") + }; + let state = SiteReplicationState { + name: "site-a".to_string(), + peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + retry_queue: vec![SiteReplicationRetryEvent { + id: "evt".to_string(), + peer_deployment_id: remote.deployment_id.clone(), + peer_endpoint: remote.endpoint, + path: "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=demo&operation=make-with-versioning".to_string(), + retry_count: 9, + failed: true, + last_error: "site replication is not enabled".to_string(), + updated_at: Some(OffsetDateTime::now_utc()), + edit_generation: None, + }], + ..Default::default() + }; + + let info = site_replication_info_for(&state, &local); + assert!(info.enabled, "the topology still reports two sites โ€” that was the trap"); + let stats = info + .retry_stats + .as_ref() + .expect("a peer rejecting every delivery must be visible in `info`"); + assert_eq!(stats.failed, 1); + assert_eq!(stats.last_error, "site replication is not enabled"); + + // A healthy site must stay wire-identical to before the field existed. + let healthy = SiteReplicationState { + retry_queue: Vec::new(), + ..state + }; + let info = site_replication_info_for(&healthy, &local); + assert!(info.retry_stats.is_none()); + assert!(info.pending_operation.is_none()); + } + + /// rustfs/rustfs#5963: `replicate info` reported a healthy cluster while + /// every peer operation was failing. The health it used to omit now rides + /// along, and a healthy site still serializes without the new fields. + #[test] + fn site_replication_info_health_fields_are_absent_when_healthy() { + let healthy = SiteReplicationInfo { + enabled: true, + name: "site-a".to_string(), + sites: vec![peer("site-a", "https://site-a.example.com")], + service_account_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + retry_stats: None, + pending_operation: None, + }; + let value = serde_json::to_value(&healthy).expect("serialize info"); + assert!(value.get("retryStats").is_none(), "a healthy site must not grow fields: {value}"); + assert!(value.get("pendingOperation").is_none(), "a healthy site must not grow fields: {value}"); + + let degraded = SiteReplicationInfo { + retry_stats: Some(SRRetryStats { + pending: 1, + failed: 4, + last_error: "site replication is not enabled".to_string(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + ..healthy + }; + let value = serde_json::to_value(°raded).expect("serialize info"); + assert_eq!( + value.pointer("/retryStats/failed").and_then(Value::as_u64), + Some(4), + "a source site whose peer rejects everything must say so in `info`" + ); + assert_eq!( + value.pointer("/retryStats/lastError").and_then(Value::as_str), + Some("site replication is not enabled") + ); } // Fix 5: remove --all must purge local state unconditionally even when peer errors occur @@ -17766,12 +18270,14 @@ mod tests { assert!(state.peers.is_empty(), "peers must be cleared on remove --all"); assert!(state.resync_status.is_empty(), "resync_status must be cleared on remove --all"); - // Even if peers returned 403 (desynced account), status still reports success + // The local side is torn down either way, but a peer that returned 403 + // (desynced account) leaves the cluster diverged โ€” the response must + // say so instead of reporting unqualified success (rustfs/rustfs#5963). let status = site_replication_remove_status(&["https://remote.example.com: peer/remove returned 403 Forbidden".to_string()]); assert_eq!( - status.status, SITE_REPL_REMOVE_SUCCESS, - "local remove reports success even when peer notifications fail" + status.status, SITE_REPL_REMOVE_PARTIAL, + "local remove must report a partial result when peer notifications fail" ); assert!( status.err_detail.contains("403 Forbidden"), diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index 0290ebbcb..03cd43714 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -53,25 +53,18 @@ const EVENT_ADMIN_TIER_STATE: &str = "admin_tier_state"; #[derive(Debug, Clone, serde::Deserialize, Default)] pub struct AddTierQuery { #[serde(rename = "accessKey")] - #[allow(dead_code)] pub access_key: Option, - #[allow(dead_code)] pub status: Option, #[serde(rename = "secretKey")] - #[allow(dead_code)] pub secret_key: Option, #[serde(rename = "serviceName")] - #[allow(dead_code)] pub service_name: Option, #[serde(rename = "sessionToken")] - #[allow(dead_code)] pub session_token: Option, pub tier: Option, #[serde(rename = "tierName")] - #[allow(dead_code)] pub tier_name: Option, #[serde(rename = "tierType")] - #[allow(dead_code)] pub tier_type: Option, pub force: Option, } @@ -532,7 +525,6 @@ impl Operation for EditTier { #[derive(Debug, Clone, serde::Deserialize, Default)] pub struct BucketQuery { #[serde(rename = "bucket")] - #[allow(dead_code)] pub bucket: String, } pub struct ListTiers {} diff --git a/rustfs/src/admin/handlers/trace.rs b/rustfs/src/admin/handlers/trace.rs index 17daa67bf..6c22eeec7 100644 --- a/rustfs/src/admin/handlers/trace.rs +++ b/rustfs/src/admin/handlers/trace.rs @@ -21,7 +21,6 @@ use matchit::Params; use rustfs_madmin::service_commands::ServiceTraceOpts; use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; -#[allow(dead_code)] fn extract_trace_options(uri: &Uri) -> S3Result { let mut st_opts = ServiceTraceOpts::default(); st_opts @@ -31,7 +30,6 @@ fn extract_trace_options(uri: &Uri) -> S3Result { Ok(st_opts) } -#[allow(dead_code)] pub struct Trace {} #[async_trait::async_trait] diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index fe07d4056..37c4b34c9 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -19,7 +19,6 @@ pub mod handlers; mod plugin_contract; pub(crate) mod replication_metrics_wire; // Contract inventory is validated by tests before later runtime integration. -#[allow(dead_code)] pub(crate) mod route_policy; pub mod router; pub(crate) mod runtime_sources; diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 2c1ad5e1b..ccb013211 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -1598,6 +1598,10 @@ pub const DEFERRED_ADMIN_ROUTE_POLICIES: &[DeferredAdminRoutePolicy] = &[ ), ]; +#[allow( + dead_code, + reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)" +)] pub fn validate_admin_route_policy_specs() -> Result<(), AdminRouteMatrixError> { validate_admin_route_specs(ADMIN_ROUTE_POLICY_SPECS) } diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 14ad1fb5e..498069eb4 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -5800,7 +5800,6 @@ mod tests { } } -#[allow(dead_code)] #[derive(Debug, Clone)] pub struct Extra { pub credentials: Option, diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index eb38e9640..318718b49 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -909,10 +909,6 @@ pub(crate) mod contract { }; } - pub(crate) mod heal { - pub(crate) use super::super::storage_contracts::HealOperations; - } - pub(crate) mod list { pub(crate) use super::super::storage_contracts::ListOperations; } diff --git a/rustfs/src/app/context/global.rs b/rustfs/src/app/context/global.rs index 50c79498a..b26151223 100644 --- a/rustfs/src/app/context/global.rs +++ b/rustfs/src/app/context/global.rs @@ -45,7 +45,6 @@ pub struct AppContext { object_store: Arc, iam: Arc, federated_identity: Arc, - #[allow(dead_code)] kms: Arc, kms_runtime: Arc, outbound_tls_runtime: Arc, @@ -162,7 +161,6 @@ impl AppContext { self.federated_identity.publish_handle(service) } - #[allow(dead_code)] pub fn kms(&self) -> Arc { self.kms.clone() } diff --git a/rustfs/src/app/context/handles.rs b/rustfs/src/app/context/handles.rs index 9e376e1a5..8a2bdc29c 100644 --- a/rustfs/src/app/context/handles.rs +++ b/rustfs/src/app/context/handles.rs @@ -49,7 +49,6 @@ use tokio::sync::RwLock; /// Default IAM interface adapter. pub struct IamHandle { - #[allow(dead_code)] iam: Arc>, } @@ -110,7 +109,6 @@ impl FederatedIdentityInterface for FederatedIdentityHandle { } /// Default KMS interface adapter. -#[allow(dead_code)] pub struct KmsHandle { kms: Arc, } diff --git a/rustfs/src/app/context/interfaces.rs b/rustfs/src/app/context/interfaces.rs index 17e029b76..b36a560dd 100644 --- a/rustfs/src/app/context/interfaces.rs +++ b/rustfs/src/app/context/interfaces.rs @@ -36,7 +36,6 @@ use tokio::sync::RwLock; /// IAM interface for application-layer use-cases. pub trait IamInterface: Send + Sync { - #[allow(dead_code)] fn handle(&self) -> Arc>; fn is_ready(&self) -> bool; fn token_signing_key(&self) -> Option { @@ -53,7 +52,6 @@ pub trait FederatedIdentityInterface: Send + Sync { } /// KMS interface for application-layer use-cases. -#[allow(dead_code)] pub trait KmsInterface: Send + Sync { fn handle(&self) -> Arc; } diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index b63110d39..2c7405c62 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -97,7 +97,7 @@ use super::storage_api::object_usecase::set_disk::{ }; use super::storage_api::object_usecase::sse::{ DecryptionRequest, EncryptionRequest, SSEType, SseKmsPrincipal, apply_bucket_default_lock_retention, - authorize_sse_kms_object_read, build_ssec_read_headers, encryption_material_to_metadata, + authorize_sse_kms_object_read, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in, load_bucket_object_lock_config_state, map_get_object_reader_error, sse_decryption, sse_encryption, validate_bucket_object_lock_enabled_state, @@ -169,7 +169,7 @@ use s3s::dto::{ ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, ObjectPart, PutObjectInput, PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreStatus, SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput, ServerSideEncryption, - ServerSideEncryptionByDefault, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader, Timestamp, TimestampFormat, + ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader, Timestamp, TimestampFormat, WebsiteRedirectLocation, }; use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH}; @@ -738,7 +738,7 @@ struct GetObjectPreparedRead { } struct GetObjectStrategyContext { - #[allow(dead_code)] + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] io_strategy: concurrency::IoStrategy, optimal_buffer_size: usize, enable_readahead: bool, @@ -2676,23 +2676,37 @@ fn has_put_sse_request_headers(headers: &HeaderMap) -> bool { || headers.get(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID).is_some() } -/// Managed SSE resolved from a bucket default encryption rule on the copy path. +/// Resolve the effective server-side encryption for a write against the bucket's +/// default encryption configuration. /// -/// Unknown algorithms fall back to AES256, the same total mapping as the PUT and -/// extract paths and the storage-layer resolver (`prepare_sse_configuration`), which -/// `sse_encryption` re-runs when it mints the destination DEK. Resolving `None` here -/// instead lets a same-name copy under a malformed bucket default pass the -/// `copy_changes_encryption` guard and take the metadata-only shortcut while the -/// storage layer still encrypts: fresh DEK metadata is committed beside the untouched -/// plaintext blocks and the object becomes unreadable. Reachable only via corrupt or -/// hand-edited bucket metadata โ€” PutBucketEncryption rejects unknown algorithms -/// (backlog#1826). -fn bucket_default_write_sse(sse: &ServerSideEncryptionByDefault) -> ServerSideEncryption { - match sse.sse_algorithm.as_str() { - "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), - _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - } +/// A request-level value always wins; the bucket default only fills a gap, and +/// the unknown-algorithm fallback lives once in [`bucket_default_write_sse`]. +/// +/// `has_explicit_ssec` suppresses the default entirely. Only COPY passes `true` +/// today: its destination may carry SSE-C, which must not also be given managed +/// encryption. PUT and extract pass `false`, matching their current behaviour โ€” +/// see backlog#1826 for the divergence that leaves. +/// +/// Callers layering further overrides (PUT's `ciphertext_passthrough`) apply +/// them to the returned pair. +fn resolve_bucket_default_sse( + bucket_sse_config: Option<&ServerSideEncryptionConfiguration>, + requested_sse: Option, + requested_kms_key_id: Option, + has_explicit_ssec: bool, +) -> (Option, Option) { + let bucket_default = || { + if has_explicit_ssec { + return None; + } + bucket_sse_config + .and_then(|config| config.rules.first()) + .and_then(|rule| rule.apply_server_side_encryption_by_default.as_ref()) + }; + + let effective_sse = requested_sse.or_else(|| bucket_default().map(bucket_default_write_sse)); + let effective_kms_key_id = requested_kms_key_id.or_else(|| bucket_default().and_then(|sse| sse.kms_master_key_id.clone())); + (effective_sse, effective_kms_key_id) } fn should_use_small_eager_put_path( @@ -5823,19 +5837,12 @@ impl DefaultObjectUsecase { ); let original_sse = server_side_encryption.clone(); - let mut effective_sse = server_side_encryption.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default.as_ref().map(|sse| { - match sse.sse_algorithm.as_str() { - "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), - _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), // fallback to AES256 - } - }) - }) - }) - }); + let (mut effective_sse, mut effective_kms_key_id) = resolve_bucket_default_sse( + bucket_sse_config.as_ref().map(|(config, _timestamp)| config), + server_side_encryption, + ssekms_key_id, + false, + ); debug!( target: "rustfs::app::object_usecase", component = "app", @@ -5847,16 +5854,6 @@ impl DefaultObjectUsecase { "Resolved effective SSE configuration" ); - let mut effective_kms_key_id = ssekms_key_id.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .and_then(|sse| sse.kms_master_key_id.clone()) - }) - }) - }); - if ciphertext_passthrough { // The replica keeps the source's SSE-C metadata; the bucket // default must not claim managed encryption on it. @@ -7658,30 +7655,12 @@ impl DefaultObjectUsecase { } }; - let mut effective_sse = requested_sse.or_else(|| { - if has_explicit_ssec { - return None; - } - bucket_sse_config.as_ref().and_then(|(config, _)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .map(bucket_default_write_sse) - }) - }) - }); - let mut effective_kms_key_id = requested_kms_key_id.or_else(|| { - if has_explicit_ssec { - return None; - } - bucket_sse_config.as_ref().and_then(|(config, _)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .and_then(|sse| sse.kms_master_key_id.clone()) - }) - }) - }); + let (mut effective_sse, mut effective_kms_key_id) = resolve_bucket_default_sse( + bucket_sse_config.as_ref().map(|(config, _)| config), + requested_sse, + requested_kms_key_id, + has_explicit_ssec, + ); let h = build_ssec_read_headers( copy_source_sse_customer_algorithm.as_ref(), @@ -9587,28 +9566,12 @@ impl DefaultObjectUsecase { let original_sse = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?); let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); - let mut effective_sse = original_sse.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .map(|sse| match sse.sse_algorithm.as_str() { - "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), - _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - }) - }) - }) - }); - let mut effective_kms_key_id = ssekms_key_id.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .and_then(|sse| sse.kms_master_key_id.clone()) - }) - }) - }); + let (mut effective_sse, mut effective_kms_key_id) = resolve_bucket_default_sse( + bucket_sse_config.as_ref().map(|(config, _timestamp)| config), + original_sse, + ssekms_key_id, + false, + ); if effective_sse .as_ref() .is_some_and(|sse| sse.as_str().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) @@ -10133,8 +10096,8 @@ mod tests { DefaultRetention, Delete, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination, ExistingObjectReplication, ExistingObjectReplicationStatus, ObjectIdentifier, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRule, ReplicaModifications, ReplicaModificationsStatus, - ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, RestoreRequest, ServerSideEncryptionConfiguration, - ServerSideEncryptionRule, SourceSelectionCriteria, + ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, RestoreRequest, ServerSideEncryptionByDefault, + ServerSideEncryptionConfiguration, ServerSideEncryptionRule, SourceSelectionCriteria, }; use std::pin::Pin; use std::sync::Arc; @@ -10397,6 +10360,74 @@ mod tests { } } + fn bucket_sse_config_with(algorithm: &str, kms_key_id: Option<&str>) -> ServerSideEncryptionConfiguration { + ServerSideEncryptionConfiguration { + rules: vec![ServerSideEncryptionRule { + apply_server_side_encryption_by_default: Some(ServerSideEncryptionByDefault { + sse_algorithm: ServerSideEncryption::from(String::from(algorithm)), + kms_master_key_id: kms_key_id.map(|id| SSEKMSKeyId::from(id.to_string())), + }), + bucket_key_enabled: None, + }], + } + } + + #[test] + fn resolve_bucket_default_sse_prefers_the_request_over_the_bucket_default() { + let config = bucket_sse_config_with(ServerSideEncryption::AWS_KMS, Some("bucket-key")); + + let (sse, kms_key_id) = resolve_bucket_default_sse( + Some(&config), + Some(ServerSideEncryption::from_static(ServerSideEncryption::AES256)), + Some(SSEKMSKeyId::from("request-key".to_string())), + false, + ); + + assert_eq!(sse.as_ref().map(|sse| sse.as_str()), Some(ServerSideEncryption::AES256)); + assert_eq!(kms_key_id.as_deref(), Some("request-key")); + } + + #[test] + fn resolve_bucket_default_sse_fills_gaps_from_the_bucket_default() { + let config = bucket_sse_config_with(ServerSideEncryption::AWS_KMS, Some("bucket-key")); + + let (sse, kms_key_id) = resolve_bucket_default_sse(Some(&config), None, None, false); + + assert_eq!(sse.as_ref().map(|sse| sse.as_str()), Some(ServerSideEncryption::AWS_KMS)); + assert_eq!(kms_key_id.as_deref(), Some("bucket-key")); + } + + #[test] + fn resolve_bucket_default_sse_falls_back_to_aes256_for_an_unknown_algorithm() { + // Reachable only through corrupt or hand-edited bucket metadata; + // PutBucketEncryption rejects unknown algorithms. All three call sites + // now share this single decision (backlog#1826). + let config = bucket_sse_config_with("garbage", None); + + let (sse, kms_key_id) = resolve_bucket_default_sse(Some(&config), None, None, false); + + assert_eq!(sse.as_ref().map(|sse| sse.as_str()), Some(ServerSideEncryption::AES256)); + assert!(kms_key_id.is_none()); + } + + #[test] + fn resolve_bucket_default_sse_suppresses_the_default_for_explicit_ssec() { + let config = bucket_sse_config_with(ServerSideEncryption::AES256, Some("bucket-key")); + + let (sse, kms_key_id) = resolve_bucket_default_sse(Some(&config), None, None, true); + + assert!(sse.is_none(), "an SSE-C destination must not also get managed encryption"); + assert!(kms_key_id.is_none()); + } + + #[test] + fn resolve_bucket_default_sse_returns_nothing_without_a_bucket_default() { + let (sse, kms_key_id) = resolve_bucket_default_sse(None, None, None, false); + + assert!(sse.is_none()); + assert!(kms_key_id.is_none()); + } + #[test] fn put_request_user_metadata_cannot_suppress_bucket_default_retention() { let mut metadata = diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 0c2b3d4d4..d90cfb6a8 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -1032,8 +1032,9 @@ pub(crate) mod sse { validate_bucket_object_lock_enabled_state, }; pub(crate) use crate::storage::storage_api::sse_consumer::{ - EncryptionKeyKind, SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers, - extract_ssekms_context_from_headers, map_get_object_reader_error, mark_encrypted_multipart_metadata, + EncryptionKeyKind, SSEType, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata, + extract_ssec_params_from_headers, extract_ssekms_context_from_headers, map_get_object_reader_error, + mark_encrypted_multipart_metadata, }; } diff --git a/rustfs/src/auth.rs b/rustfs/src/auth.rs index dc254afa8..fa12a564e 100644 --- a/rustfs/src/auth.rs +++ b/rustfs/src/auth.rs @@ -812,12 +812,10 @@ fn is_reserved_condition_key(key: &str, server_derived: &HashMap AuthType { get_request_auth_type_with_query(header, None) } -#[allow(dead_code)] pub(crate) fn get_request_auth_type_with_query(header: &HeaderMap, query: Option<&str>) -> AuthType { if is_request_signature_v2(header) { AuthType::SignedV2 @@ -846,20 +844,6 @@ pub(crate) fn get_request_auth_type_with_query(header: &HeaderMap, query: Option } } -/// Helper function to determine auth type and signature version -/// -/// # Arguments -/// * `header` - HTTP headers of the request -/// -/// # Returns -/// * `(String, String)` - Tuple of auth type and signature version -/// -#[allow(dead_code)] -fn determine_auth_type_and_version(header: &HeaderMap) -> (String, String) { - determine_auth_type_and_version_with_query(header, None) -} - -#[allow(dead_code)] fn determine_auth_type_and_version_with_query(header: &HeaderMap, query: Option<&str>) -> (String, String) { match get_request_auth_type_with_query(header, query) { AuthType::JWT => ("JWT".to_string(), String::new()), @@ -925,18 +909,6 @@ fn is_request_signature_v2(header: &HeaderMap) -> bool { false } -/// Verify if request has AWS PreSign Version '4' -/// -/// # Arguments -/// * `header` - HTTP headers of the request -/// -/// # Returns -/// * `bool` - True if request has AWS PreSign Version '4', false otherwise -#[allow(dead_code)] -pub(crate) fn is_request_presigned_signature_v4(header: &HeaderMap) -> bool { - is_request_presigned_signature_v4_with_query(header, None) -} - pub(crate) fn is_request_presigned_signature_v4_with_query(header: &HeaderMap, query: Option<&str>) -> bool { if let Some(credential) = header.get(AMZ_CREDENTIAL) { return !credential.to_str().unwrap_or("").is_empty(); diff --git a/rustfs/src/capacity/capacity_integration.rs b/rustfs/src/capacity/capacity_integration.rs index 9389049bf..927838f3d 100644 --- a/rustfs/src/capacity/capacity_integration.rs +++ b/rustfs/src/capacity/capacity_integration.rs @@ -31,7 +31,6 @@ pub async fn init_capacity_management_managed() -> Option Option<(u64, String)> { get_cached_capacity_with_metrics() .await diff --git a/rustfs/src/init.rs b/rustfs/src/init.rs index 34a1d6ca0..e76dca19b 100644 --- a/rustfs/src/init.rs +++ b/rustfs/src/init.rs @@ -765,7 +765,6 @@ fn resolve_buffer_profile_config( /// Parse and normalize server address for FTP/FTPS /// Forces IPv4 binding to avoid libunftp IPv6 compatibility issues -#[allow(dead_code)] async fn parse_and_normalize_server_address( address_str: &str, ) -> Result> { @@ -781,45 +780,6 @@ async fn parse_and_normalize_server_address( Ok(normalized_addr) } - -/// Start FTP/FTPS server in background with shutdown support -/// # Arguments -/// * `server` - The FTP/FTPS server instance -/// * `protocol_name` - Name of the protocol (e.g., "FTP", "FTPS") -#[allow(dead_code)] -fn spawn_server(server: S, protocol_name: &'static str) -> tokio::sync::broadcast::Sender<()> -where - S: std::future::Future>> + Send + 'static, -{ - let (shutdown_tx, _) = tokio::sync::broadcast::channel(1); - - tokio::spawn(async move { - if let Err(e) = server.await { - error!( - target: "rustfs::init", - event = "protocol_server_state", - component = LOG_COMPONENT_INIT, - subsystem = LOG_SUBSYSTEM_PROTOCOL, - protocol = protocol_name, - state = "runtime_failed", - error = %e, - "Protocol server failed" - ); - } - info!( - target: "rustfs::init", - event = "protocol_server_state", - component = LOG_COMPONENT_INIT, - subsystem = LOG_SUBSYSTEM_PROTOCOL, - protocol = protocol_name, - state = "stopped", - "Protocol server stopped" - ); - }); - - shutdown_tx -} - /// Starts the auto-tuner for performance optimization if enabled via environment variable. /// /// The auto-tuner reads `RUSTFS_AUTOTUNER_ENABLED` to decide whether to run. diff --git a/rustfs/src/license.rs b/rustfs/src/license.rs index 5163e58bf..75638eaeb 100644 --- a/rustfs/src/license.rs +++ b/rustfs/src/license.rs @@ -211,7 +211,6 @@ fn apply_valid_status(state: &mut LicenseState, token: Token) { /// /// This is the extension point for OEM/build-time overlays. /// Returns `false` if the verifier was already initialized. -#[allow(dead_code)] pub fn set_license_verifier(verifier: SharedLicenseVerifier) -> bool { LICENSE_VERIFIER.set(verifier).is_ok() } diff --git a/rustfs/src/startup_iam.rs b/rustfs/src/startup_iam.rs index 6220e1c12..fe28108b8 100644 --- a/rustfs/src/startup_iam.rs +++ b/rustfs/src/startup_iam.rs @@ -352,17 +352,6 @@ fn should_fail_test_init_attempt() -> bool { false } } - -/// Reset the test failure counter so the next `should_fail_test_init_attempt` -/// call re-reads the environment variable by restoring the sentinel value. -/// Intended for use in integration tests that share a process. -#[doc(hidden)] -#[allow(dead_code)] -pub(crate) fn reset_test_failure_counter() { - use std::sync::atomic::Ordering; - TEST_REMAINING_FAILURES.store(u64::MAX, Ordering::SeqCst); -} - async fn attempt_init_iam_sys( store: Arc, ) -> std::result::Result>, std::io::Error> { diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index d69bde85f..690c2abdd 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -61,7 +61,7 @@ pub(crate) struct ReqInfo { pub object: Option, pub version_id: Option, pub replication_request_authorized: bool, - #[allow(dead_code)] + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] pub region: Option, pub request_context: Option, /// Set by probe-style callers that treat AccessDenied as an expected filter diff --git a/rustfs/src/storage/concurrency/io_schedule.rs b/rustfs/src/storage/concurrency/io_schedule.rs index 8d202708a..80ebf7c62 100644 --- a/rustfs/src/storage/concurrency/io_schedule.rs +++ b/rustfs/src/storage/concurrency/io_schedule.rs @@ -72,7 +72,6 @@ impl IoLoadLevel { } /// Get the load level as a string for metrics labels. - #[allow(dead_code)] pub fn as_str(&self) -> &'static str { match self { IoLoadLevel::Low => "low", @@ -83,7 +82,6 @@ impl IoLoadLevel { } /// Get the load level as a numeric index (0=Low, 1=Medium, 2=High, 3=Critical). - #[allow(dead_code)] pub fn level_index(&self) -> u8 { match self { IoLoadLevel::Low => 0, @@ -118,7 +116,6 @@ pub enum IoPriority { impl IoPriority { /// Determine priority from request size using scheduler config thresholds. - #[allow(dead_code)] pub fn from_size(size: i64) -> Self { Self::from_size_with_thresholds( size, @@ -152,19 +149,16 @@ impl IoPriority { } /// Check if this is high priority. - #[allow(dead_code)] pub fn is_high(&self) -> bool { matches!(self, IoPriority::High) } /// Check if this is normal priority. - #[allow(dead_code)] pub fn is_normal(&self) -> bool { matches!(self, IoPriority::Normal) } /// Check if this is low priority. - #[allow(dead_code)] pub fn is_low(&self) -> bool { matches!(self, IoPriority::Low) } @@ -403,7 +397,6 @@ impl IoSchedulerConfig { /// I/O queue status for monitoring. #[derive(Debug, Clone, Default)] -#[allow(dead_code)] pub struct IoQueueStatus { /// Total permits available. pub total_permits: usize, @@ -520,7 +513,6 @@ pub struct IoStrategyCore { impl IoStrategyCore { /// Create a minimal IoStrategyCore with essential fields only. - #[allow(dead_code)] pub fn new(storage_media: StorageMedia, access_pattern: AccessPattern, buffer_size: usize) -> Self { Self { storage_media, @@ -1194,7 +1186,6 @@ impl IoStrategy { } /// Get a human-readable description of the current I/O strategy. - #[allow(dead_code)] pub fn description(&self) -> String { format!( "IoStrategy[{:?}]: buffer={}KB, multiplier={:.2}, readahead={}, wait={:?}", @@ -1282,14 +1273,6 @@ impl IoLoadMetrics { IoLoadLevel::from_wait_duration(self.average_wait()) } - /// Get the overall average wait since startup - #[allow(dead_code)] - pub(crate) fn lifetime_average_wait(&self) -> Duration { - let total = self.total_wait_ns.load(Ordering::Relaxed); - let count = self.observation_count.load(Ordering::Relaxed); - total.checked_div(count).map(Duration::from_nanos).unwrap_or(Duration::ZERO) - } - /// Get the total observation count pub(crate) fn observation_count(&self) -> u64 { self.observation_count.load(Ordering::Relaxed) @@ -1450,13 +1433,13 @@ use tracing::warn; /// Queued I/O request with metadata. #[derive(Debug)] -#[allow(dead_code)] struct QueuedRequest { /// The actual request payload. request: T, /// Time when the request was enqueued. enqueue_time: Instant, /// Original priority assigned to the request. + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] original_priority: IoPriority, /// Current priority (may be boosted for starvation prevention). current_priority: IoPriority, @@ -1466,7 +1449,6 @@ struct QueuedRequest { /// Queue statistics for monitoring. #[derive(Debug, Clone, Default)] -#[allow(dead_code)] struct QueueStats { /// Number of high priority requests processed. high_processed: u64, @@ -1552,7 +1534,6 @@ impl Default for IoPriorityQueueConfig { impl IoPriorityQueueConfig { /// Load configuration from environment. - #[allow(dead_code)] pub fn from_env() -> Self { Self { queue_high_capacity: rustfs_utils::get_env_usize( @@ -1603,7 +1584,6 @@ impl IoPriorityQueueConfig { impl IoPriorityQueue { /// Create a new priority queue with the given configuration. - #[allow(dead_code)] pub fn new(config: IoPriorityQueueConfig) -> Self { let config_clone = config.clone(); Self { @@ -1617,7 +1597,6 @@ impl IoPriorityQueue { } /// Enqueue a request with the given priority. - #[allow(dead_code)] pub async fn enqueue(&self, priority: IoPriority, request: T) { let queued = QueuedRequest { request, @@ -1638,7 +1617,6 @@ impl IoPriorityQueue { /// /// This method performs starvation prevention checks before dequeuing. /// Returns `None` if all queues are empty. - #[allow(dead_code)] pub async fn dequeue(&self) -> Option<(T, IoPriority)> { // 1. Check for starvation prevention self.check_starvation().await; @@ -1716,7 +1694,6 @@ impl IoPriorityQueue { } /// Get current queue status for monitoring. - #[allow(dead_code)] pub async fn status(&self) -> IoQueueStatus { let high_queue = self.high_queue.lock().await; let normal_queue = self.normal_queue.lock().await; @@ -1737,7 +1714,6 @@ impl IoPriorityQueue { } /// Get the total number of queued requests. - #[allow(dead_code)] pub async fn len(&self) -> usize { let high_queue = self.high_queue.lock().await; let normal_queue = self.normal_queue.lock().await; @@ -1747,7 +1723,6 @@ impl IoPriorityQueue { } /// Check if all queues are empty. - #[allow(dead_code)] pub async fn is_empty(&self) -> bool { self.len().await == 0 } @@ -1766,11 +1741,9 @@ mod tests { }; use rustfs_io_core::io_profile::{AccessPattern, StorageMedia}; use rustfs_io_metrics::bandwidth::{BandwidthSnapshot, BandwidthTier}; - use serial_test::serial; use std::time::Duration; #[tokio::test] - #[serial] async fn test_io_priority_queue_basic() { let config = IoPriorityQueueConfig::default(); let queue = IoPriorityQueue::new(config); @@ -1789,7 +1762,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_priority_queue_dequeue_order() { let config = IoPriorityQueueConfig::default(); let queue = IoPriorityQueue::new(config); @@ -1817,7 +1789,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_priority_queue_status() { let config = IoPriorityQueueConfig::default(); let queue = IoPriorityQueue::new(config); @@ -1835,7 +1806,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_priority_queue_starvation_prevention() { let config = IoPriorityQueueConfig { starvation_threshold_secs: 1, @@ -1859,7 +1829,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_priority_from_size() { // High priority: < 1MB assert_eq!(IoPriority::from_size(100 * 1024), IoPriority::High); @@ -1875,7 +1844,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_load_level_from_wait_duration() { use std::time::Duration; @@ -1893,7 +1861,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_scheduler_config_default() { let config = IoSchedulerConfig::default(); @@ -1907,7 +1874,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_scheduler_config_to_core_config() { let config = IoSchedulerConfig::default(); let core = config.to_core_config(); @@ -1923,7 +1889,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_priority_queue_config_to_core_config() { let config = IoPriorityQueueConfig::default(); let core = config.to_core_config(); @@ -1935,7 +1900,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_priority_queue_config_from_scheduler_config() { let scheduler_config = IoSchedulerConfig { queue_high_capacity: 128, @@ -1958,7 +1922,6 @@ mod tests { // ============================================ #[tokio::test] - #[serial] async fn test_multi_factor_strategy_nvme_sequential_low_load() { // NVMe + Sequential + Low load = maximum buffer size let context = IoSchedulingContext { @@ -1985,7 +1948,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_hdd_random_high_load() { // HDD + Random + High load = conservative buffer size let context = IoSchedulingContext { @@ -2012,7 +1974,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_ssd_mixed_medium_load() { // SSD + Mixed + Medium load = moderate buffer let context = IoSchedulingContext { @@ -2040,7 +2001,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_critical_load_disables_features() { // Any media + Critical load = minimal features let context = IoSchedulingContext { @@ -2065,7 +2025,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_buffer_cap_enforcement() { // Test that storage media caps are enforced let context = IoSchedulingContext { @@ -2090,7 +2049,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_applies_sequential_hint_when_pattern_unknown() { let context = IoSchedulingContext { file_size: 2 * 1024 * 1024 * 1024, // 2GiB @@ -2115,7 +2073,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_bandwidth_low_reduces_buffer() { // Low bandwidth should reduce buffer let context = IoSchedulingContext { @@ -2139,7 +2096,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_high_concurrency_reduction() { // High concurrency should reduce buffer let context = IoSchedulingContext { @@ -2162,7 +2118,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_sequential_boost() { // Sequential reads should get boost let sequential_context = IoSchedulingContext { @@ -2204,7 +2159,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_unknown_media_conservative() { // Unknown media should be conservative let context = IoSchedulingContext { @@ -2230,7 +2184,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_priority_classification() { // Test priority classification based on file size let small_context = IoSchedulingContext { @@ -2277,7 +2230,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_readahead_decision_matrix() { // Test readahead enable/disable logic let configs = vec![ @@ -2363,7 +2315,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_buffer_multiplier_stages() { // Test that all multiplier stages are applied let context = IoSchedulingContext { @@ -2398,7 +2349,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_compatibility_path() { // Test that compatibility path (from_wait_duration) still works let wait_duration = Duration::from_millis(50); diff --git a/rustfs/src/storage/concurrency/manager.rs b/rustfs/src/storage/concurrency/manager.rs index 5bc456148..bf51b55bf 100644 --- a/rustfs/src/storage/concurrency/manager.rs +++ b/rustfs/src/storage/concurrency/manager.rs @@ -50,7 +50,7 @@ pub struct ConcurrencyManager { /// I/O load metrics for adaptive strategy calculation io_metrics: Arc>, /// I/O priority queue for request scheduling - #[allow(dead_code)] + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] priority_queue: Arc>, /// Bytes pool for buffer allocation and reuse bytes_pool: Arc, @@ -131,7 +131,6 @@ pub enum PutObjectAdmission { Rejected, } -#[allow(dead_code)] impl ConcurrencyManager { /// Create a new concurrency manager with default settings /// diff --git a/rustfs/src/storage/concurrency/request_guard.rs b/rustfs/src/storage/concurrency/request_guard.rs index d435ef2bd..f1847a5c0 100644 --- a/rustfs/src/storage/concurrency/request_guard.rs +++ b/rustfs/src/storage/concurrency/request_guard.rs @@ -64,7 +64,6 @@ impl GetObjectGuard { } /// Get the elapsed time since this guard was created. - #[allow(dead_code)] // This helper is primarily used by unit tests to assert timing. // It's intentionally kept public for callers that may want to inspect // a guard's duration without dropping it. diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index 5d6fba452..9ecd207a9 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -254,7 +254,10 @@ pub(crate) fn apply_bucket_default_lock_retention( /// ); /// ``` /// -#[allow(dead_code)] +#[allow( + dead_code, + reason = "exercised by ecfs_test; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn get_adaptive_buffer_size_with_profile(file_size: i64, profile: Option) -> usize { let config = match profile { Some(p) => RustFSBufferConfig::new(p), @@ -798,26 +801,10 @@ fn cache_remove(bucket: &str) { map.remove(bucket); } } - -/// Clear all entries in the cache. -#[allow(dead_code)] -fn cache_clear() { - if let Ok(mut map) = small_cache().write() { - map.clear(); - } -} - /// Invalidate the validation cache for a specific bucket. pub fn invalidate_bucket_validation_cache(bucket: &str) { cache_remove(bucket); } - -/// Invalidate all bucket validation cache entries. -#[allow(dead_code)] -pub fn invalidate_all_bucket_validation_cache() { - cache_clear(); -} - /// Helper function to get store and validate bucket exists. /// /// Uses adaptive cache with 5s TTL to avoid repeated stat_volume() calls. diff --git a/rustfs/src/storage/head_prefix.rs b/rustfs/src/storage/head_prefix.rs index 49f53f452..89ae57a1f 100644 --- a/rustfs/src/storage/head_prefix.rs +++ b/rustfs/src/storage/head_prefix.rs @@ -15,15 +15,6 @@ use super::ECStore; use crate::storage::storage_api::head_prefix_consumer::contract::list::ListOperations as _; use std::sync::Arc; - -/// Determines if the key "looks like a prefix" (ends with `/`). -/// Note: No special handling for empty strings here; the caller must ensure the key has passed `validate_object_key`. -#[allow(dead_code)] -#[inline] -pub(crate) fn is_prefix_key(key: &str) -> bool { - key.ends_with('/') -} - /// Constructs a more explicit error message when `HEAD` is performed on a `prefix`-style key but the directory marker object is missing. /// /// `has_children`: diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index dc5a6ca19..701dbd38b 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -1018,12 +1018,6 @@ pub fn parse_copy_source_range(range_str: &str) -> S3Result { Err(s3_error!(InvalidArgument, "Invalid range format")) } } - -#[allow(dead_code)] -pub(crate) fn get_content_sha256(headers: &HeaderMap) -> Option { - get_content_sha256_with_query(headers, None) -} - pub(crate) fn get_content_sha256_with_query(headers: &HeaderMap, query: Option<&str>) -> Option { match get_request_auth_type_with_query(headers, query) { AuthType::Presigned | AuthType::Signed => { @@ -1036,14 +1030,6 @@ pub(crate) fn get_content_sha256_with_query(headers: &HeaderMap, qu _ => None, } } - -/// skip_content_sha256_cksum returns true if caller needs to skip -/// payload checksum, false if not. -#[allow(dead_code)] -fn skip_content_sha256_cksum(headers: &HeaderMap) -> bool { - skip_content_sha256_cksum_with_query(headers, None) -} - fn skip_content_sha256_cksum_with_query(headers: &HeaderMap, query: Option<&str>) -> bool { let include_query_values = matches!(get_request_auth_type_with_query(headers, query), AuthType::Presigned); let content_sha256 = get_content_sha256_value(headers, query, include_query_values); @@ -1138,12 +1124,6 @@ fn get_content_sha256_value( .and_then(|v| v.to_str().ok()) .map(str::to_owned) } - -#[allow(dead_code)] -fn get_content_sha256_cksum(headers: &HeaderMap, service_type: ServiceType) -> String { - get_content_sha256_cksum_with_query(headers, None, service_type) -} - #[cfg(test)] #[allow(unused_imports)] mod tests { diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 352bfbd58..2503e25c7 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -2206,7 +2206,7 @@ mod tests { previous_scanner_activity_response, remove_heal_control_replay, scanner_activity_response, stop_rebalance_response, }; use crate::storage::rpc::node_service::heal::heal_topology_fingerprint; - use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo, HealEndpoint}; + use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo}; use crate::storage::storage_api::set_tonic_canonical_body_digest; use crate::storage::storage_api::{ Endpoint, @@ -2334,42 +2334,14 @@ mod tests { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result>> { - Ok(None) - } - - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> rustfs_heal::Result<()> { - Ok(()) - } - - async fn delete_object(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<()> { - Ok(()) - } - - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result { - Ok(true) - } - async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { Ok(Vec::new()) } - async fn get_disk_status(&self, _endpoint: &HealEndpoint) -> rustfs_heal::Result { - Ok(rustfs_heal::heal::storage::DiskStatus::Ok) - } - - async fn format_disk(&self, _endpoint: &HealEndpoint) -> rustfs_heal::Result<()> { - Ok(()) - } - async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result> { Ok(None) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> { - Ok(()) - } - async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(Vec::new()) } @@ -2378,14 +2350,6 @@ mod tests { Ok(false) } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { - Ok(None) - } - - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { - Ok(None) - } - async fn heal_object( &self, _bucket: &str, @@ -2411,14 +2375,6 @@ mod tests { Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None)) } - async fn list_objects_for_heal( - &self, - _bucket: &str, - _prefix: &str, - ) -> rustfs_heal::Result> { - Ok(Vec::new()) - } - async fn list_objects_for_heal_page( &self, _bucket: &str, diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index ce0ba42e5..5d9fbc969 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -23,12 +23,15 @@ use bytes::Bytes; use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::{ INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_REQUEST, - INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC, - global_internode_metrics, + INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, + INTERNODE_STAGE_READ_VERSION_DISK_READ, INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, + INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE, + INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics, }; use rustfs_protos::proto_gen::node_service::*; use serde::de::DeserializeOwned; use std::io::Cursor; +use std::time::Instant; use tonic::{Request, Response, Status}; use tracing::debug; @@ -201,6 +204,21 @@ fn encode_read_multiple_response_payloads( Ok((read_multiple_resps_json, read_multiple_resps_bin)) } +fn internode_stage_timer(attribution_enabled: bool) -> Option { + attribution_enabled.then(Instant::now) +} + +fn record_read_version_stage(stage: &'static str, started_at: Option) { + if let Some(started_at) = started_at { + global_internode_metrics().record_stage_duration_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + stage, + started_at.elapsed(), + ); + } +} + fn encode_batch_read_version_response_payloads( batch_read_version_resps: &[BatchReadVersionResp], request_decoded_from_msgpack: bool, @@ -685,11 +703,42 @@ impl NodeService { request: Request, ) -> Result, Status> { let request = request.into_inner(); + let metrics = global_internode_metrics(); + let read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + if read_version_attribution_enabled { + metrics.record_incoming_request_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + metrics.record_recv_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + request + .disk + .len() + .saturating_add(request.volume.len()) + .saturating_add(request.path.len()) + .saturating_add(request.version_id.len()) + .saturating_add(request.opts.len()) + .saturating_add(request.opts_bin.len()), + ); + } if let Some(disk) = self.find_disk(&request.disk).await { let request_had_msgpack_payload = !request.opts_bin.is_empty(); + let decode_started = internode_stage_timer(read_version_attribution_enabled); let opts = match decode_msgpack_or_json::(&request.opts_bin, &request.opts, "ReadOptions") { - Ok(options) => options, + Ok(options) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, decode_started); + options + } Err(err) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, decode_started); + if read_version_attribution_enabled { + metrics.record_error_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + } return Ok(Response::new(ReadVersionResponse { success: false, file_info: String::new(), @@ -698,42 +747,88 @@ impl NodeService { })); } }; + let disk_read_started = internode_stage_timer(read_version_attribution_enabled); match disk .read_version("", &request.volume, &request.path, &request.version_id, &opts) .await { Ok(file_info) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_DISK_READ, disk_read_started); + let json_encode_started = internode_stage_timer(read_version_attribution_enabled); let file_info_json = compat_response_json(&file_info, request_had_msgpack_payload); + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE, json_encode_started); + let msgpack_encode_started = internode_stage_timer(read_version_attribution_enabled); let file_info_bin = encode_file_info_msgpack(&file_info); + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE, msgpack_encode_started); match (file_info_json, file_info_bin) { - (Ok(file_info), Ok(file_info_bin)) => Ok(Response::new(ReadVersionResponse { - success: true, - file_info, - file_info_bin: file_info_bin.into(), - error: None, - })), - (Err(err), _) => Ok(Response::new(ReadVersionResponse { - success: false, - file_info: String::new(), - file_info_bin: Vec::new().into(), - error: Some(DiskError::other(format!("encode data failed: {err}")).into()), - })), - (_, Err(err)) => Ok(Response::new(ReadVersionResponse { - success: false, - file_info: String::new(), - file_info_bin: Vec::new().into(), - error: Some(DiskError::other(format!("encode data failed: {err}")).into()), - })), + (Ok(file_info), Ok(file_info_bin)) => { + if read_version_attribution_enabled { + metrics.record_sent_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + file_info.len().saturating_add(file_info_bin.len()), + ); + } + Ok(Response::new(ReadVersionResponse { + success: true, + file_info, + file_info_bin: file_info_bin.into(), + error: None, + })) + } + (Err(err), _) => { + if read_version_attribution_enabled { + metrics.record_error_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + } + Ok(Response::new(ReadVersionResponse { + success: false, + file_info: String::new(), + file_info_bin: Vec::new().into(), + error: Some(DiskError::other(format!("encode data failed: {err}")).into()), + })) + } + (_, Err(err)) => { + if read_version_attribution_enabled { + metrics.record_error_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + } + Ok(Response::new(ReadVersionResponse { + success: false, + file_info: String::new(), + file_info_bin: Vec::new().into(), + error: Some(DiskError::other(format!("encode data failed: {err}")).into()), + })) + } } } - Err(err) => Ok(Response::new(ReadVersionResponse { - success: false, - file_info: String::new(), - file_info_bin: Vec::new().into(), - error: Some(err.into()), - })), + Err(err) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_DISK_READ, disk_read_started); + if read_version_attribution_enabled { + metrics.record_error_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + } + Ok(Response::new(ReadVersionResponse { + success: false, + file_info: String::new(), + file_info_bin: Vec::new().into(), + error: Some(err.into()), + })) + } } } else { + if read_version_attribution_enabled { + metrics.record_error_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + } Ok(Response::new(ReadVersionResponse { success: false, file_info: String::new(), @@ -1520,12 +1615,16 @@ mod tests { encode_batch_read_version_response_payloads, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, encode_read_multiple_response_payloads, encode_rename_data_response_payloads, }; + use crate::storage::rpc::node_service::make_server; use crate::storage::storage_api::ReadMultipleResp; use crate::storage::storage_api::RenameDataResp; use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp; use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::global_internode_metrics; + use rustfs_protos::proto_gen::node_service::ReadVersionRequest; use serde::{Deserialize, Serialize}; + use serial_test::serial; + use tonic::Request; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct SamplePayload { @@ -1533,6 +1632,36 @@ mod tests { count: u32, } + #[tokio::test] + #[serial] + async fn handle_read_version_records_attribution_for_missing_disk() { + let metrics = global_internode_metrics(); + let previous_stage_metrics = rustfs_io_metrics::get_stage_metrics_enabled(); + metrics.reset_for_test(); + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + + let response = make_server() + .handle_read_version(Request::new(ReadVersionRequest { + disk: "missing-disk".to_string(), + volume: "bucket".to_string(), + path: "object".to_string(), + version_id: String::new(), + opts: String::new(), + opts_bin: Vec::new().into(), + })) + .await + .expect("ReadVersion handler should return a response") + .into_inner(); + + rustfs_io_metrics::set_get_stage_metrics_enabled(previous_stage_metrics); + let snapshot = metrics.snapshot(); + assert!(!response.success); + assert_eq!(snapshot.incoming_requests_total, 1); + assert_eq!(snapshot.errors_total, 1); + assert!(snapshot.recv_bytes_total > 0); + metrics.reset_for_test(); + } + #[test] fn decode_msgpack_or_json_prefers_binary_payload() { let payload = SamplePayload { diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index eda06a7dd..b3e4f839f 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -164,7 +164,7 @@ use rustfs_utils::http::headers::{ AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, }; use rustfs_utils::path::path_join_buf; -use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId}; +use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, ServerSideEncryptionByDefault}; use std::borrow::Cow; // ============================================================================ @@ -203,6 +203,24 @@ pub struct SseConfiguration { /// Effective KMS key ID (after considering bucket defaults) pub effective_kms_key_id: Option, } +/// Managed SSE resolved from a bucket default encryption rule on a write path. +/// +/// The single mapping shared by every writer: this resolver, and the PUT, COPY +/// and extract paths in `app::object_usecase`, which reach it through +/// `resolve_bucket_default_sse`. Unknown algorithms fall back to AES256 rather +/// than to `None`. Resolving `None` instead lets a same-name copy under a +/// malformed bucket default pass the `copy_changes_encryption` guard and take +/// the metadata-only shortcut while this layer still encrypts: fresh DEK +/// metadata is committed beside the untouched plaintext blocks and the object +/// becomes unreadable. Reachable only via corrupt or hand-edited bucket +/// metadata โ€” PutBucketEncryption rejects unknown algorithms (backlog#1826). +pub(crate) fn bucket_default_write_sse(sse: &ServerSideEncryptionByDefault) -> ServerSideEncryption { + match sse.sse_algorithm.as_str() { + "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), + _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + } +} /// Prepare SSE configuration by resolving request parameters with bucket defaults /// @@ -266,11 +284,7 @@ async fn prepare_sse_configuration( has_kms_key_id = sse.kms_master_key_id.is_some(), "Bucket SSE default resolved" ); - match sse.sse_algorithm.as_str() { - "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), - _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), // fallback - } + bucket_default_write_sse(sse) }) }) }); @@ -3354,7 +3368,6 @@ async fn get_local_sse_dek_provider() -> Result, ApiErro /// Clears GLOBAL_SSE_DEK_PROVIDER (local/test providers) and /// GLOBAL_KMS_DEK_PROVIDER (test-injected KMS providers). #[cfg(test)] -#[allow(dead_code)] pub fn reset_sse_dek_provider() { if let Ok(mut slot) = GLOBAL_SSE_DEK_PROVIDER.write() { *slot = None; @@ -3365,7 +3378,6 @@ pub fn reset_sse_dek_provider() { } #[cfg(test)] -#[allow(dead_code)] pub fn set_sse_dek_provider_for_test(provider: Arc) { if let Ok(mut slot) = GLOBAL_KMS_DEK_PROVIDER.write() { *slot = Some(provider.clone()); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 534269546..31668783a 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -252,8 +252,6 @@ pub(crate) mod rpc_consumer { }; pub(crate) type StorageResult = super::super::Result; - #[cfg(test)] - pub(crate) type HealEndpoint = super::super::ecstore_disk::endpoint::Endpoint; #[cfg(test)] pub(crate) type HealBucketInfo = super::super::contract::bucket::BucketInfo; @@ -346,9 +344,9 @@ pub(crate) mod s3_api_consumer { pub(crate) mod sse_consumer { pub(crate) use super::super::sse::{ - EncryptionKeyKind, SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers, - extract_ssekms_context_from_headers, log_sse_kms_key_policy_mode, map_get_object_reader_error, - mark_encrypted_multipart_metadata, + EncryptionKeyKind, SSEType, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata, + extract_ssec_params_from_headers, extract_ssekms_context_from_headers, log_sse_kms_key_policy_mode, + map_get_object_reader_error, mark_encrypted_multipart_metadata, }; pub(crate) use super::{ DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, diff --git a/rustfs/src/storage/tonic_service.rs b/rustfs/src/storage/tonic_service.rs index e56a8f235..539361507 100644 --- a/rustfs/src/storage/tonic_service.rs +++ b/rustfs/src/storage/tonic_service.rs @@ -16,5 +16,4 @@ pub(crate) use crate::storage::rpc::node_service::make_heal_control_server_with_ #[cfg(test)] pub(crate) use crate::storage::rpc::node_service::{heal::heal_topology_fingerprint, make_heal_control_server_for_source}; pub use crate::storage::rpc::{make_heal_control_server, make_server, make_tier_mutation_control_server}; -#[allow(dead_code)] pub type NodeService = crate::storage::rpc::NodeService; diff --git a/rustfs/src/update.rs b/rustfs/src/update.rs index 4ff089408..3b7a267cb 100644 --- a/rustfs/src/update.rs +++ b/rustfs/src/update.rs @@ -45,7 +45,6 @@ pub struct VersionInfo { } /// Update check result -#[allow(dead_code)] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct UpdateCheckResult { /// Whether update is available @@ -91,7 +90,6 @@ impl VersionChecker { } /// Create version checker with custom configuration - #[allow(dead_code)] pub fn with_config(url: String, timeout: Duration) -> Self { let client = reqwest::Client::builder() .timeout(timeout) @@ -175,7 +173,6 @@ pub async fn check_updates() -> Result { } /// Update check with custom URL -#[allow(dead_code)] pub async fn check_updates_with_url(url: String) -> Result { let checker = VersionChecker::with_config(url, Duration::from_secs(10)); checker.check_for_updates().await diff --git a/scripts/README.md b/scripts/README.md index 8f11d6931..3ba0b5499 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -28,6 +28,7 @@ their issue closes. | `check_architecture_migration_rules.sh` | ci-gate | Architecture-boundary anti-regression guard | ci.yml Quick Checks; `make pre-commit` | | `check_body_cache_whitelist.sh` | ci-gate | Keeps the app-layer body-cache eligibility gate fail-closed | ci.yml Quick Checks | | `check_doc_paths.sh` | ci-gate | Fails when instruction/architecture docs reference repo paths that no longer exist | `make pre-commit` / `pre-pr` | +| `check_embedded_secrets.sh` | ci-gate | Repo-wide scan blocking committed private key material and provider credential literals | ci.yml Quick Checks; `make pre-commit` / `pre-pr` | | `check_extension_schema_boundaries.sh` | ci-gate | Extension-schema crate boundary guard | ci.yml Quick Checks; `make pre-commit` | | `check_layer_dependencies.sh` | ci-gate | Crate-layering DAG guard (reads `layer-dependency-baseline.txt`) | ci.yml Quick Checks | | `check_logging_guardrails.sh` | ci-gate | Blocks legacy logging patterns from returning | `make pre-commit` / `pre-pr` | diff --git a/scripts/check_embedded_secrets.sh b/scripts/check_embedded_secrets.sh new file mode 100755 index 000000000..0634eb0f8 --- /dev/null +++ b/scripts/check_embedded_secrets.sh @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Guard: no private key material and no long-lived provider credential may +# exist as a literal anywhere in the repository โ€” see AGENTS.md "Security +# Baseline" ("Never commit secrets, credentials, or key material") and +# .agents/skills/security-advisory-lessons ("Do not ship hard-coded shared +# tokens, HMAC secrets, private keys, or production test keys"). +# +# This replaces the unit test `test_source_does_not_embed_private_key` that +# used to live in crates/crypto/src/license_token.rs (rustfs/backlog#1884). +# That test read its own file with include_str! and asserted the file did not +# contain a PEM private-key header, protecting exactly one invariant: the RSA +# key that signs license tokens must never be checked in, because verification +# only ever needs the public key (crates/crypto/src/license_token.rs exposes +# `parse_signed_license_token`, and rustfs/src/license.rs reads the public key +# from RUSTFS_LICENSE_PUBLIC_KEY at runtime โ€” no key material belongs in the +# tree at all). Its coverage was one file: moving the key one file sideways, +# even inside the same crate, passed silently, and renaming license_token.rs +# stopped the guard from compiling rather than reporting anything. +# +# This scan covers every tracked โ€” and every not-yet-added, non-ignored โ€” text +# file in the repository, so it is a strict superset of the retired assertion: +# the same needle, everywhere, plus the algorithm variants and the credential +# formats below. +# +# It deliberately does not exclude the paths .github/secret_scanning.yml tells +# GitHub push protection to ignore (crates/e2e_test, **/tests, **/benches, +# .docker, .vscode). Those exclusions exist because pasted test credentials are +# expected there, which is exactly where a real key is most likely to arrive +# unnoticed; this guard is the CI-side gate that still looks. +# +# Only literals are in scope. Key material injected at build time is out of +# scope on purpose: no build.rs in the workspace embeds key material and the +# license public key is read from the environment at startup, so an artifact +# scan would add a release build to a compile-free check job for no reachable +# failure mode today. Revisit if a build script ever bakes in key material. +# Binary files are skipped (`git grep -I`), and a key stored as bare base64 +# with its header stripped is not detected โ€” the same two blind spots the +# retired test had. +# +# Every needle below is assembled around a variable so that the script's own +# text does not match the pattern it defines (the retired test used the same +# trick with ["BEGIN", "PRIVATE KEY"].join(" ")). That is what lets this script +# scan itself along with everything else instead of carving out a blind spot. +# +# `--self-test` builds throwaway fixture repositories and asserts every pattern +# family fires and every exemption holds; it is wired into `make script-tests`. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${CHECK_EMBEDDED_SECRETS_ROOT:-$(cd "${SCRIPT_DIR}/.." && pwd)}" + +BEGIN_MARK="BEGIN" +KEY_MARK="Key" + +# The file the retired test pinned with include_str!. It stays listed so that +# renaming or moving it is reported here explicitly instead of quietly ending +# the license-key invariant, which is how the test failed. +PINNED_SOURCES=( + "crates/crypto/src/license_token.rs" +) + +# "|". The name is free of "|", so the first "|" splits. +# +# The private-key family matches the header phrase without requiring the PEM +# dashes, so a key pasted into a JSON/YAML string, a doc block, or a Rust +# string built without the delimiters is still caught. The credential family is +# format-anchored โ€” fixed prefix plus fixed-width charset โ€” so a match is a +# credential shape and not prose. +PATTERNS=( + "PEM private key header|${BEGIN_MARK}[[:space:]]+([A-Z0-9]+[[:space:]]+)*PRIVATE KEY" + "PuTTY private key file|PuTTY-User-${KEY_MARK}-File" + "AWS access key id|(A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}" + "GitHub token|gh[pousr]_[A-Za-z0-9]{36}" + "GitHub fine-grained token|github_pat_[A-Za-z0-9_]{22,}" + "Slack token|xox[abprs]-[A-Za-z0-9-]{10,}" + "Stripe live key|sk_live_[0-9a-zA-Z]{20,}" + "Google API key|AIza[0-9A-Za-z_-]{35}" + "SendGrid API key|SG\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}" + "npm access token|npm_[A-Za-z0-9]{36}" + "PyPI upload token|pypi-AgEIcHlwaS5vcmc[A-Za-z0-9_-]{50,}" +) + +# Exact strings that carry no secret wherever they appear. A hit is excused +# only if the line stops matching once these exact strings are removed, so a +# line holding both an example value and a real credential still fails, and +# editing an entry โ€” swapping a placeholder body for real key material โ€” makes +# the guard fire again. Entries that stop matching anything are reported as +# stale, so the list cannot decay into a blanket exclusion. +# +# 1-2: rustfs/src/admin/handlers/site_replication.rs negative fixtures for +# `validate_peer_connection_inner`, which must reject a private key +# submitted where a peer CA certificate is expected. Asserting on the +# rejection requires the header in the input; the key bodies are the +# literal word "secret". +# 3-4: AWS's own documented example access key id from the SigV4 test vectors, +# which this repository pairs with the equally documented example secret +# key across signer, IAM, madmin, and auth tests, plus the deliberate +# one-character variant rustfs/src/auth.rs uses to prove key comparison +# distinguishes near-identical ids. +AWS_EXAMPLE_STEM="AKIAIOSFODNN7EXAMPL" +NON_SECRET_LITERALS=( + "-----${BEGIN_MARK} PRIVATE KEY-----\\nsecret\\n-----END PRIVATE KEY-----" + "-----${BEGIN_MARK} RSA PRIVATE KEY-----\\nsecret\\n-----END RSA PRIVATE KEY-----" + "${AWS_EXAMPLE_STEM}E" + "${AWS_EXAMPLE_STEM}F" +) + +run_scan() { + cd "$ROOT_DIR" + + # Without this, a scan run outside a work tree would make every `git grep` + # fail and the guard would report success having read nothing. + if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + printf 'Embedded secret guard failed: %s is not a git work tree, so the scan cannot enumerate files\n' \ + "$ROOT_DIR" >&2 + return 1 + fi + + local literal_used=() + local i + for ((i = 0; i < ${#NON_SECRET_LITERALS[@]}; i++)); do + literal_used[i]="0" + done + + local status=0 + local source + for source in "${PINNED_SOURCES[@]}"; do + if [[ ! -f "$source" ]]; then + printf 'Embedded secret guard failed: %s is missing; update PINNED_SOURCES in scripts/check_embedded_secrets.sh after moving it\n' \ + "$source" >&2 + status=1 + fi + done + + local entry name pattern hits grep_status hit file rest line_no text trimmed sanitized + for entry in "${PATTERNS[@]}"; do + name="${entry%%|*}" + pattern="${entry#*|}" + + hits="" + grep_status=0 + hits="$(git grep --untracked -I -n -E -e "$pattern" -- .)" || grep_status=$? + if [[ "$grep_status" -gt 1 ]]; then + printf 'Embedded secret guard failed: git grep exited %s while scanning for %s\n' "$grep_status" "$name" >&2 + status=1 + continue + fi + + while IFS= read -r hit; do + [[ -z "$hit" ]] && continue + file="${hit%%:*}" + rest="${hit#*:}" + line_no="${rest%%:*}" + text="${rest#*:}" + trimmed="$(printf '%s' "$text" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + sanitized="$trimmed" + for ((i = 0; i < ${#NON_SECRET_LITERALS[@]}; i++)); do + if [[ "$sanitized" == *"${NON_SECRET_LITERALS[i]}"* ]]; then + sanitized="${sanitized//"${NON_SECRET_LITERALS[i]}"/}" + literal_used[i]="1" + fi + done + + if ! printf '%s' "$sanitized" | grep -q -E -e "$pattern"; then + continue + fi + + printf 'Embedded secret guard failed: %s at %s:%s\n %s\n' "$name" "$file" "$line_no" "$trimmed" >&2 + status=1 + done <<<"$hits" + done + + for ((i = 0; i < ${#NON_SECRET_LITERALS[@]}; i++)); do + if [[ "${literal_used[i]}" != "1" ]]; then + printf 'Embedded secret guard failed: stale exemption, nothing matches it any more: %s\n' \ + "${NON_SECRET_LITERALS[i]}" >&2 + status=1 + fi + done + + if [[ "$status" -ne 0 ]]; then + printf '\nRemove the key material or credential above, and rotate anything that was committed even briefly.\n' >&2 + printf 'A genuine non-secret match is excused by adding its exact text to NON_SECRET_LITERALS in scripts/check_embedded_secrets.sh with a reason.\n' >&2 + return 1 + fi + + printf 'Embedded secret guard passed (no private key material or provider credential literal in the tree).\n' + return 0 +} + +# One synthetic value per pattern family, assembled from a filler so this +# function does not match the patterns it exercises. +self_test_violation_lines() { + local fill="QWERTYUIOPASDFGHJKLZXCVBNM0123456789" + printf '%s\n' \ + "-----${BEGIN_MARK} PRIVATE KEY-----" \ + "PuTTY-User-${KEY_MARK}-File: ssh-rsa" \ + "AKIA${fill:0:16}" \ + "ghp_${fill:0:36}" \ + "github_pat_${fill:0:22}" \ + "xoxb-${fill:0:12}" \ + "sk_live_${fill:0:20}" \ + "AIza${fill:0:35}" \ + "SG.${fill:0:20}.${fill:0:20}" \ + "npm_${fill:0:36}" \ + "pypi-AgEIcHlwaS5vcmc${fill}${fill:0:14}" +} + +self_test_fixture() { + local dir="$1" i + mkdir -p "${dir}/crates/crypto/src" + : >"${dir}/crates/crypto/src/license_token.rs" + # Every exemption must appear, or the stale-exemption check fires and the + # fixture would fail for a reason the case under test is not about. + for ((i = 0; i < ${#NON_SECRET_LITERALS[@]}; i++)); do + printf 'excused %s\n' "${NON_SECRET_LITERALS[i]}" >>"${dir}/excused.txt" + done + git -C "$dir" init -q +} + +SELF_TEST_TMP="" + +self_test() { + SELF_TEST_TMP="$(mktemp -d)" + trap 'rm -rf "$SELF_TEST_TMP"' EXIT + + local failures=0 out scan_status + local clean="${SELF_TEST_TMP}/clean" dirty="${SELF_TEST_TMP}/dirty" renamed="${SELF_TEST_TMP}/renamed" + + self_test_fixture "$clean" + if out="$(CHECK_EMBEDDED_SECRETS_ROOT="$clean" "$0" 2>&1)"; then + printf 'self-test ok: clean fixture with every exemption present passes\n' + else + printf 'self-test FAILED: clean fixture should pass but reported:\n%s\n' "$out" >&2 + failures=$((failures + 1)) + fi + + self_test_fixture "$dirty" + self_test_violation_lines >"${dirty}/leaked.txt" + scan_status=0 + out="$(CHECK_EMBEDDED_SECRETS_ROOT="$dirty" "$0" 2>&1)" || scan_status=$? + if [[ "$scan_status" -eq 0 ]]; then + printf 'self-test FAILED: fixture holding one value per pattern family should fail\n' >&2 + failures=$((failures + 1)) + fi + local entry name + for entry in "${PATTERNS[@]}"; do + name="${entry%%|*}" + if ! printf '%s' "$out" | grep -q -F -- "$name"; then + printf 'self-test FAILED: pattern family "%s" did not fire on its own probe value\n' "$name" >&2 + failures=$((failures + 1)) + fi + done + [[ "$failures" -eq 0 ]] && printf 'self-test ok: all %s pattern families fire\n' "${#PATTERNS[@]}" + + self_test_fixture "$renamed" + rm -f "${renamed}/crates/crypto/src/license_token.rs" + if CHECK_EMBEDDED_SECRETS_ROOT="$renamed" "$0" >/dev/null 2>&1; then + printf 'self-test FAILED: a moved pinned source should be reported\n' >&2 + failures=$((failures + 1)) + else + printf 'self-test ok: moving a pinned source is reported\n' + fi + + if [[ "$failures" -ne 0 ]]; then + printf '%s self-test assertion(s) failed\n' "$failures" >&2 + return 1 + fi + printf 'Embedded secret guard self-test passed.\n' + return 0 +} + +if [[ "${1:-}" == "--self-test" ]]; then + self_test +else + run_scan +fi diff --git a/scripts/find_assertless_tests.py b/scripts/find_assertless_tests.py index dc73447de..094d051af 100755 --- a/scripts/find_assertless_tests.py +++ b/scripts/find_assertless_tests.py @@ -49,19 +49,47 @@ import sys from pathlib import Path VERIFY_SIGNALS = re.compile( - r"assert!|assert_eq!|assert_ne!|debug_assert|panic!\(|\.expect\(|\.unwrap\(|" + r"assert[a-z0-9_]*!|debug_assert|panic!\(|\.expect\(|\.unwrap\(|" r"unreachable!|matches!\(|insta::|proptest!|\.await\?|\)\?|\?;|should_panic" ) DELEGATION = re.compile( - r"\b(?:assert|verify|check|expect|ensure|run)_[a-z0-9_]*\s*\(|" - r"\b[a-z0-9_]+_(?:case|cases|harness|roundtrip|round_trip)\s*\(" + r"\b(?:assert|verify|check|expect|ensure|run)_[a-z0-9_]*(?:::<[^>]*>)?\s*\(|" + r"\b[a-z0-9_]+_(?:case|cases|harness|roundtrip|round_trip)(?:::<[^>]*>)?\s*\(" ) + +# A body whose whole content is one call delegates by construction, whatever the +# callee is named: `run(DurabilityMode::Strict).await` and +# `aborting_encode_drops_blocked_producer(EncodePipeline::Vec).await` both hand +# every assertion to a shared harness. +SINGLE_CALL_BODY = re.compile( + r"\A\s*[a-zA-Z_][a-zA-Z0-9_:]*(?:::<[^>]*>)?\s*\([^;]*\)\s*(?:\.await\s*)?;?\s*\Z", + re.S, +) + +# A nested `fn` that is only bound and discarded is a signature guard: the type +# system is the assertion, exactly like the `fn _name()` form below. +SIGNATURE_GUARD = re.compile(r"\bfn\s+[a-zA-Z0-9_]+\s*(?:<[^>]*>)?\s*\([^;]*\)[^;]*\{", re.S) +DISCARDED_BINDING = re.compile(r"\blet\s+_\s*=\s*[a-zA-Z_][a-zA-Z0-9_]*\s*;") +# `let _ = Type::::method;` โ€” a path item referenced but never called can only +# be a signature guard; the call form (`let _ = x.foo();`) is excluded by the +# absence of parens before the semicolon. +DISCARDED_PATH_ITEM = re.compile(r"\blet\s+_\s*=\s*[a-zA-Z_][a-zA-Z0-9_]*(?:::(?:<[^>]*>|[a-zA-Z_][a-zA-Z0-9_]*))+\s*;") COMPILE_TIME_CHECK = re.compile(r"\bfn\s+_[a-zA-Z0-9_]*\s*(?:<[^>]*>)?\s*\(") TEST_ATTR = re.compile(r"#\[(?:tokio::)?test[\](]") TEST_CASE_ATTR = re.compile(r"#\[test_case") FN_LINE = re.compile(r"^\s*(?:pub\s+)?(?:async\s+)?fn\s+([a-zA-Z0-9_]+)") + +def extract_body(text: str) -> str: + """Return what is between the outermost braces of a scanned function.""" + start = text.find("{") + end = text.rfind("}") + if start == -1 or end <= start: + return text + return text[start + 1 : end] + + def scan_file(path: Path): try: lines = path.read_text(encoding="utf-8").split("\n") @@ -105,7 +133,17 @@ def scan_file(path: Path): break k += 1 text = "\n".join(body) - if not VERIFY_SIGNALS.search(text) and not DELEGATION.search(text) and not COMPILE_TIME_CHECK.search(text): + # The attribute block carries verification too: `#[should_panic(expected + # = "...")]` makes the panic message the assertion. + attr_text = "\n".join(attrs) + inner = extract_body(text) + delegates = ( + DELEGATION.search(text) + or SINGLE_CALL_BODY.match(inner) + or (SIGNATURE_GUARD.search(inner) and DISCARDED_BINDING.search(inner)) + or DISCARDED_PATH_ITEM.search(inner) + ) + if not VERIFY_SIGNALS.search(text) and not VERIFY_SIGNALS.search(attr_text) and not delegates and not COMPILE_TIME_CHECK.search(text): print(f"{path}:{j + 1}: {name}") i = k + 1 diff --git a/scripts/test/site_replication_smoke.py b/scripts/test/site_replication_smoke.py index be6b94563..259a98e79 100755 --- a/scripts/test/site_replication_smoke.py +++ b/scripts/test/site_replication_smoke.py @@ -27,6 +27,7 @@ Usage: ./scripts/test/site_replication_smoke.py # up: start both + pair ./scripts/test/site_replication_smoke.py status # process + pair status ./scripts/test/site_replication_smoke.py smoke # bidirectional object check + ./scripts/test/site_replication_smoke.py diverge # rustfs/rustfs#5963 regression ./scripts/test/site_replication_smoke.py logs # tail both server logs ./scripts/test/site_replication_smoke.py down # stop both processes ./scripts/test/site_replication_smoke.py clean # down + wipe site data @@ -337,11 +338,12 @@ def ensure_pair(site_a: Site, site_b: Site) -> None: print(f"[ok] site replication configured: {result.get('status', '')}") -def remove_pair(site: Site) -> None: +def remove_pair(site: Site) -> dict: status, body = admin(site, "PUT", "site-replication/remove", payload={"all": True}) if status != 200: raise SystemExit(f"[fail] site-replication remove: HTTP {status} {body.decode(errors='replace')}") print(f"[ok] site replication removed: {body.decode(errors='replace')}") + return json.loads(body) # --------------------------------------------------------------------------- @@ -397,6 +399,112 @@ def smoke(site_a: Site, site_b: Site, timeout: float) -> None: print(f"[ok] bidirectional replication verified via bucket {bucket}") +# --------------------------------------------------------------------------- +# Divergence regression (rustfs/rustfs#5963) +# --------------------------------------------------------------------------- + + +def wait_for(description: str, probe, timeout: float): + """Poll `probe` until it returns a truthy value; return it. SystemExit on timeout.""" + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + try: + result = probe() + except (urllib.error.URLError, OSError, TimeoutError, SystemExit) as err: + last = err + result = None + if result: + return result + time.sleep(1.0) + raise SystemExit(f"[fail] {description} within {timeout:.0f}s (last: {last})") + + +def diverge(site_a: Site, site_b: Site, binary: Path, console: bool, timeout: float) -> None: + """Reproduce rustfs/rustfs#5963 end to end and assert the cluster recovers. + + Before the fix, step 7 left site-b rejecting every peer bucket-op forever: + `pending_remove` gates `SRPeerBucketOpsHandler` ahead of `enabled()`, and a + join never cleared it โ€” so a *successful* re-add produced a cluster that + reported Enabled/2-sites on both sides while replication stayed dead. + """ + ensure_pair(site_a, site_b) + + # 1. Take site-a down so it cannot be told about the removal. + print("[..] step 1: stopping site-a so it cannot be notified") + stop_site(site_a) + + # 2. Remove from site-b. The local teardown commits either way, but the + # response must NOT claim unqualified success (P2-5). + print("[..] step 2: removing site replication from site-b while site-a is down") + status = remove_pair(site_b) + if not status.get("errorDetail"): + raise SystemExit(f"[fail] remove hid the unreachable peer; expected errorDetail: {json.dumps(status)}") + if status.get("status") == "Requested site(s) were removed from cluster replication successfully.": + raise SystemExit(f"[fail] remove reported unqualified success despite an unnotified peer: {json.dumps(status)}") + print(f"[ok] remove reported a partial result: status={status.get('status')!r}") + + # 3. The wedged removal must be visible on `info`, not just in status --json (P1-4). + info_b = pair_state(site_b) + pending = info_b.get("pendingOperation") + if not pending or pending.get("operation") != "remove": + raise SystemExit(f"[fail] site-b hides the wedged removal in `info`: {json.dumps(info_b, indent=2)}") + print(f"[ok] site-b reports the wedged removal: pendingPeers={pending.get('pendingPeers')}") + + # 4. Bring site-a back. It still believes in a healthy 2-site cluster. + print("[..] step 4: restarting site-a") + start_site(site_a, binary, console) + wait_ready([site_a], timeout) + info_a = pair_state(site_a) + if not info_a.get("enabled"): + raise SystemExit(f"[fail] site-a lost its own state: {json.dumps(info_a, indent=2)}") + print("[ok] site-a still reports an enabled cluster (the divergence)") + + # 5. A bucket created on site-a cannot reach site-b. The failure must become + # visible on the SOURCE, which used to report a perfectly healthy cluster. + bucket = f"sr-diverge-{uuid.uuid4().hex[:8]}" + sig_status, body = signed_request(site_a, "PUT", f"/{bucket}") + if sig_status != 200: + raise SystemExit(f"[fail] create bucket {bucket} on site-a: HTTP {sig_status} {body.decode(errors='replace')}") + print(f"[ok] created {bucket} on site-a (locally succeeds, peer push is rejected)") + + stats = wait_for( + "site-a did not surface the failing peer deliveries in `info`", + lambda: pair_state(site_a).get("retryStats"), + timeout, + ) + print(f"[ok] site-a reports failing deliveries: pending={stats.get('pending')} failed={stats.get('failed')} " + f"lastError={stats.get('lastError')!r}") + + # 6. Re-add. This is the operator's natural recovery move. + print("[..] step 6: re-adding the pair from site-a") + peers = [ + {"name": s.name, "endpoints": s.endpoint, "accessKey": s.access_key, "secretKey": s.secret_key} + for s in (site_a, site_b) + ] + add_status, add_body = admin(site_a, "PUT", "site-replication/add", "replicateILMExpiry=false", peers) + if add_status != 200: + raise SystemExit(f"[fail] re-add: HTTP {add_status} {add_body.decode(errors='replace')}") + print(f"[ok] re-add accepted: {add_body.decode(errors='replace')}") + + # 7. The join must have cleared site-b's pending_remove (P0-1). Without the + # fix this assertion is exactly what fails while everything above passes. + info_b = pair_state(site_b) + if info_b.get("pendingOperation"): + raise SystemExit( + "[fail] the join did not clear site-b's wedged removal; peer bucket-ops stay rejected forever: " + f"{json.dumps(info_b, indent=2)}" + ) + if not info_b.get("enabled"): + raise SystemExit(f"[fail] site-b did not rejoin: {json.dumps(info_b, indent=2)}") + print("[ok] site-b cleared the wedged removal and rejoined") + + # 8. The symptom the issue actually reported: replication works again. + print("[..] step 8: verifying replication actually flows again") + smoke(site_a, site_b, timeout) + print("[ok] rustfs/rustfs#5963 regression passed") + + # --------------------------------------------------------------------------- # Commands # --------------------------------------------------------------------------- @@ -464,7 +572,7 @@ def main() -> None: "command", nargs="?", default="up", - choices=["up", "down", "restart", "status", "logs", "smoke", "info", "remove", "clean"], + choices=["up", "down", "restart", "status", "logs", "smoke", "diverge", "info", "remove", "clean"], ) parser.add_argument("--port-a", type=int, default=9000, help="site A S3 port (default: %(default)s)") parser.add_argument("--port-b", type=int, default=9020, help="site B S3 port (default: %(default)s)") @@ -495,6 +603,8 @@ def main() -> None: cmd_logs(sites, args.lines) elif args.command == "smoke": smoke(site_a, site_b, args.timeout) + elif args.command == "diverge": + diverge(site_a, site_b, args.binary, args.console, args.timeout) elif args.command == "info": print(json.dumps(pair_state(site_a), indent=2, ensure_ascii=False)) elif args.command == "remove":