From 950db7412c229a2d2c0498e436fbbd121f091a02 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 18 Sep 2026 21:53:31 +0800 Subject: [PATCH 01/30] fix(heal): inspect stale members during deep scans (#8009) * fix(heal): inspect stale members during deep scans * test(heal): prove convergence outcomes --- crates/ecstore/src/set_disk/ops/heal.rs | 98 ++++++++++++++++++- .../heal_b5_versioned_regression_test.rs | 86 ++++++++++++++++ 2 files changed, 183 insertions(+), 1 deletion(-) diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 82cc66610..6235cb7d1 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -978,12 +978,25 @@ impl SetDisks { result.parity_blocks = result.disk_count - read_quorum as usize; result.data_blocks = read_quorum as usize; - let ((mut online_disks, quorum_mod_time, quorum_etag), disk_len) = { + let ((quorum_disks, quorum_mod_time, quorum_etag), disk_len) = { let disks = self.disks.read().await; let disk_len = disks.len(); (Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum as usize), disk_len) }; + // A deep heal must inspect every reachable member, not only the + // quorum that agreed on the canonical metadata. A returning disk + // can legitimately carry an older xl.meta (or an explicit-version + // absence) while still being the exact stale target that needs + // repair. Restricting the repair set to quorum members silently + // turns that stale member into `verified_healthy`/`unchanged` + // (backlog#2612, #2596). Normal scans retain the quorum fast path. + let mut online_disks = if opts.scan_mode == HealScanMode::Deep { + self.get_disks_internal().await + } else { + quorum_disks + }; + trace!( event = EVENT_SET_DISK_HEAL, component = LOG_COMPONENT_ECSTORE, @@ -4087,6 +4100,89 @@ mod heal_result_report_tests { ); } + #[tokio::test] + async fn deep_heal_rebuilds_near_tail_truncated_part() { + let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "deep-heal-near-tail-truncation"; + let object = "object.bin"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let expected_payload = vec![0x6d; 1024 * 1024]; + set.put_object( + bucket, + object, + &mut PutObjReader::from_vec(expected_payload.clone()), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("source object should be written before shard truncation"); + let source = disks[2] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("source metadata should be readable"); + let data_dir = source.data_dir.expect("non-inline source should have a data directory"); + let truncated_part = temp_dirs[1] + .path() + .join(bucket) + .join(object) + .join(data_dir.to_string()) + .join("part.1"); + let original_len = tokio::fs::metadata(&truncated_part) + .await + .expect("target shard should exist") + .len(); + assert!(original_len > 1, "test shard must be large enough to truncate"); + let file = tokio::fs::OpenOptions::new() + .write(true) + .open(&truncated_part) + .await + .expect("target shard should be writable"); + file.set_len(original_len - 1) + .await + .expect("target shard should be truncated"); + + let mut reader = set + .get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default()) + .await + .expect("GET should remain readable after a one-byte shard truncation"); + let mut read_back = Vec::new(); + tokio::io::copy(&mut reader, &mut read_back) + .await + .expect("GET should reconstruct the truncated shard through EC"); + assert_eq!(read_back, expected_payload, "EC GET must preserve the object bytes"); + + let (result, error) = set + .heal_object( + bucket, + object, + "", + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("deep heal should finish after a one-byte tail truncation"); + + assert!(error.is_none(), "deep heal should recover the truncated shard: {error:?}"); + assert_eq!(result.drives_healed(), Some(1)); + assert_eq!(result.before.drives[1].state, DriveState::Corrupt.to_string()); + assert_eq!( + tokio::fs::metadata(&truncated_part) + .await + .expect("repaired shard should exist") + .len(), + original_len, + "deep heal must restore the complete shard length" + ); + } + #[tokio::test] async fn deep_heal_inline_bitrot_rebuilds_physical_data_and_parity_shards() { use crate::storage_api_contracts::range::HTTPRangeSpec; diff --git a/crates/heal/tests/heal_b5_versioned_regression_test.rs b/crates/heal/tests/heal_b5_versioned_regression_test.rs index 2eb5a315a..b790f20f0 100644 --- a/crates/heal/tests/heal_b5_versioned_regression_test.rs +++ b/crates/heal/tests/heal_b5_versioned_regression_test.rs @@ -36,6 +36,7 @@ use rustfs_heal::heal::{ }; use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode}; use serial_test::serial; +use sha2::{Digest, Sha256}; use std::{ path::{Path, PathBuf}, sync::Arc, @@ -722,6 +723,91 @@ mod serial_tests { ); } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial] + async fn test_recursive_deep_heal_reports_stale_delete_marker_repair() { + let (disk_paths, ecstore, storage) = heal_env().await; + let manager = HealManager::new( + storage.clone(), + Some(HealConfig { + heal_interval: Duration::from_millis(1), + ..Default::default() + }), + ); + manager.start().await.expect("heal manager should start"); + + let bucket = "b5-stale-delete-marker-recursive"; + let object = "obj.bin"; + create_versioned_bucket(&ecstore, bucket).await; + put_unversioned(&ecstore, bucket, "control/object.bin", &versioned_test_data(42)).await; + let historical_data = versioned_test_data(43); + let historical = put_versioned(&ecstore, bucket, object, &historical_data).await; + let target = &disk_paths[1]; + let target_meta = xl_meta_path(&object_dir(target, bucket, object)); + let stale_meta = std::fs::read(&target_meta).expect("target xl.meta must exist before marker creation"); + let stale_digest = Sha256::digest(&stale_meta); + let marker = put_delete_marker(&ecstore, bucket, object).await; + std::fs::write(&target_meta, &stale_meta).expect("restore stale target xl.meta"); + + let request = HealRequest::new( + HealType::Bucket { + bucket: bucket.to_string(), + }, + HealOptions { + recursive: true, + scan_mode: HealScanMode::Deep, + pool_index: Some(0), + set_index: Some(0), + ..Default::default() + }, + HealPriority::Normal, + ); + let task_id = request.id.clone(); + assert!( + manager + .submit_heal_request(request) + .await + .expect("submit recursive heal") + .is_admitted() + ); + wait_for_task(&manager, &task_id, Duration::from_secs(60)).await; + + let report = manager.get_task_report(&task_id).await.expect("completed task report"); + let outcome = report.outcome.expect("recursive task must have an outcome"); + assert_eq!(outcome.counters.failed, 0); + assert_eq!(outcome.counters.unknown, 0); + assert_eq!(outcome.counters.processed, 3); + assert_eq!(outcome.counters.healed, 1, "delete marker repair must be counted as healed"); + assert_eq!(outcome.counters.unchanged, 2); + + let repaired_meta = std::fs::read(&target_meta).expect("repaired target xl.meta must exist"); + assert_ne!( + Sha256::digest(&repaired_meta), + stale_digest, + "deep heal must replace stale rejoined metadata" + ); + + let marker_info = physical_version(target, bucket, object, &marker); + assert!( + marker_info.deleted && marker_info.is_latest, + "target metadata must converge to delete-marker latest" + ); + let historical_info = physical_version(target, bucket, object, &historical); + assert!(!historical_info.is_latest, "historical version must no longer be marked latest"); + assert_eq!( + read_version(&ecstore, bucket, object, &historical).await, + historical_data, + "historical version must remain readable after marker convergence" + ); + let marker_receipt = outcome + .objects + .iter() + .find(|item| item.identity.version_id.as_deref() == Some(marker.as_str())) + .expect("deep heal must emit a receipt for the repaired delete marker"); + assert_eq!(marker_receipt.disposition, HealObjectDisposition::Repaired); + manager.stop().await.expect("heal manager should stop"); + } + /// A missing xl.meta must produce an exact marker repair receipt, while a /// healthy replay must prove metadata health without claiming payload integrity. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] From ff62c810b4e6e50ac7b7cf0352084abb6ea1f211 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 18 Sep 2026 21:54:03 +0800 Subject: [PATCH 02/30] fix(tier): gate opaque version recovery cleanup (#8006) * fix(tier): gate opaque version recovery cleanup * test(tier): authorize opaque recovery fixtures --------- Co-authored-by: Chris --- .../lifecycle/transition_transaction.rs | 32 +++++++++++++++++++ .../ecstore/src/services/notification_sys.rs | 14 ++++++++ crates/ecstore/src/set_disk/ops/object.rs | 2 +- crates/ecstore/src/store/init.rs | 26 ++++++++++++--- docs/operations/tier-ilm-debugging.md | 2 ++ 5 files changed, 71 insertions(+), 5 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs index e35444cf7..601b98b24 100644 --- a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs +++ b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs @@ -35,6 +35,7 @@ use crate::bucket::lifecycle::tier_sweeper::{ use crate::disk::RUSTFS_META_BUCKET; use crate::error::{Error, Result as EcstoreResult}; use crate::object_api::{ObjectInfo, ObjectOptions}; +use crate::services::notification_sys::acquire_remote_version_state_writer_fleet_proof; use crate::services::tier::{tier::TierConfigMgr, warm_backend::TransitionCandidateProbe}; use crate::storage_api_contracts::{ list::ListOperations as _, @@ -1237,6 +1238,12 @@ pub async fn delete_transition_candidate_for_operator( lease .validate_remote_version_id(remote_version_id) .map_err(TransitionOperatorError::Remote)?; + if remote_version_requires_fleet_proof(remote_version_id) && acquire_remote_version_state_writer_fleet_proof().is_none() { + return Err(TransitionOperatorError::Remote(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "opaque remote tier version cleanup requires the operator-attested live fleet capability gate", + ))); + } let before_delete_probe = lease .probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id) .await @@ -1845,6 +1852,11 @@ async fn recover_unknown_upload_outcome( )) } TransitionCandidateProbe::VersionedPresent(version_id) => { + if remote_version_requires_fleet_proof(&version_id) && acquire_remote_version_state_writer_fleet_proof().is_none() { + return Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous( + IlmRecoveryErrorCode::RemoteVersionUnknown, + )); + } cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await } TransitionCandidateProbe::Ambiguous => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous( @@ -1940,6 +1952,14 @@ fn transition_source_lookup_options(transaction: &TransitionTransaction) -> Obje async fn delete_transition_remote_candidate(api: Arc, transaction: &TransitionTransaction) -> EcstoreResult<()> { let version_id = transaction.remote_version.tier_delete_version_id().unwrap_or_default(); let version_id_exact = transaction.remote_version.kind == TransitionRemoteVersionKind::Versioned; + if version_id_exact + && remote_version_requires_fleet_proof(version_id) + && acquire_remote_version_state_writer_fleet_proof().is_none() + { + return Err(Error::other( + "opaque remote tier version cleanup requires the operator-attested live fleet capability gate", + )); + } delete_object_from_remote_tier_idempotent_with_manager_and_identity( &transaction.remote_object, version_id, @@ -1953,6 +1973,10 @@ async fn delete_transition_remote_candidate(api: Arc, transaction: &Tra .map_err(Error::other) } +fn remote_version_requires_fleet_proof(version_id: &str) -> bool { + !version_id.is_empty() && Uuid::parse_str(version_id).is_err() +} + pub async fn recover_transition_transaction_records( api: Arc, limit: usize, @@ -2439,6 +2463,14 @@ mod tests { } } + #[test] + fn opaque_remote_versions_require_the_fleet_proof_boundary() { + assert!(!remote_version_requires_fleet_proof("")); + assert!(!remote_version_requires_fleet_proof(&Uuid::new_v4().to_string())); + assert!(remote_version_requires_fleet_proof("null")); + assert!(remote_version_requires_fleet_proof("b2-opaque-version")); + } + #[test] fn remote_version_distinguishes_unknown_unversioned_and_versioned() { assert_eq!(TransitionRemoteVersion::known_from_put_response("").tier_delete_version_id(), None); diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index 23738641e..fcf07ee89 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -453,6 +453,20 @@ pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option Option { + let requested = rustfs_utils::get_env_bool( + rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, + rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE, + ); + let fleet_confirmed = rustfs_utils::get_env_bool( + rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, + rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, + ); + (requested && fleet_confirmed) + .then(acquire_remote_version_state_fleet_proof) + .flatten() +} + fn acquire_fleet_capability_proof_from( state: &FleetCapabilityProofState, expected_topology: &str, diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index f881457c5..3eecfce5d 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -6715,7 +6715,7 @@ fn remote_version_state_writer_enabled() -> bool { } fn remote_version_state_writer_fleet_proof() -> Option { - transaction_fencing_fleet_proof(remote_version_state_writer_requested()) + crate::services::notification_sys::acquire_remote_version_state_writer_fleet_proof() } fn remote_version_state_writer_requested() -> bool { diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 27f6f5dcd..e57d54e54 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -10052,9 +10052,18 @@ mod tests { recovered_transition_version.clone(), ))) .await; - let transaction_stats = recover_transition_transaction_records(store.clone(), 100, None) + let transaction_stats = { + let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test(); + temp_env::async_with_vars( + [ + (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")), + (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")), + ], + recover_transition_transaction_records(store.clone(), 100, None), + ) .await - .expect("transition recovery should advance and consume the migrated transaction before completion"); + .expect("transition recovery should advance and consume the migrated transaction before completion") + }; backend.set_transition_candidate_probe_override(None).await; assert_eq!( ( @@ -21938,9 +21947,18 @@ mod tests { expected_removes.push((transaction.remote_object, put_version)); } - let stats = recover_transition_transaction_records(store.clone(), 100, None) + let stats = { + let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test(); + temp_env::async_with_vars( + [ + (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")), + (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")), + ], + recover_transition_transaction_records(store.clone(), 100, None), + ) .await - .expect("transition transaction recovery should run"); + .expect("transition transaction recovery should run") + }; assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (3, 3, 0, 0)); assert_eq!(transition_transaction_record_count(store.clone()).await, 0); diff --git a/docs/operations/tier-ilm-debugging.md b/docs/operations/tier-ilm-debugging.md index 3cc99bc4e..6c91e391b 100644 --- a/docs/operations/tier-ilm-debugging.md +++ b/docs/operations/tier-ilm-debugging.md @@ -65,6 +65,8 @@ get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) `transition_version_id == None` means only that no usable legacy UUID projection exists; it does not prove the remote bucket's versioning model. Only an explicit `KnownDisabled` state authorizes ordinary GET/DELETE to omit `versionId`. A missing state with an absent or empty version key remains `Unknown` and requires the bounded compatibility probe or the approved reconcile workflow; it never directly authorizes cleanup. A nil UUID (`00000000-...`) sent as `?versionId=` causes `NoSuchVersion`. Do not use `Uuid::from_slice(..).unwrap_or_default()` here: it converts an empty metadata value into `Uuid::nil()`, which is exactly that failure. +Opaque remote version IDs, such as those returned by Backblaze B2, are treated as exact remote identifiers only after the operator-attested live fleet capability gate is active. The gate requires both `RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE=true` and `RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED=true`, plus a current fleet proof. Recovery and operator cleanup retain an ambiguous candidate while that proof is absent or stale; they never issue a guessed opaque-version delete. Existing persisted exact opaque IDs remain routable, but new persistence and recovery cleanup fail closed until the gate is current. + ## Inspect xl.meta directly ```bash From ddc5f97a0432fb72e1aa335ccddab22ed48365a7 Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 19 Sep 2026 03:22:44 +0800 Subject: [PATCH 03/30] fix(release): sort pre-release packages below their final release (#8016) Pre-release RPM versions replaced the SemVer `-` with `_`, which rpm treats as an ordinary segment separator, so `1.0.0_rc.5` compared as newer than `1.0.0` and a host with an rc RPM installed could not `dnf upgrade` to the GA package. DEB previews had the mirror-image problem: only the first `-` became `~`, so `1.0.0~rc.5-preview.2` carried `preview.2` as its Debian revision and sorted above `1.0.0~rc.5`. Both formats now spell every pre-release separator as `~`, which dpkg and rpm (>= 4.10) both treat as "sorts before anything". Verified with rpm 4.16 (AlmaLinux 9) and dpkg 1.21: `1.0.0~rc.5 < 1.0.0`, `1.0.0~rc.5~preview.2 < 1.0.0~rc.5 < 1.0.0~rc.6`, alpha < beta < rc, and `rc.9 < rc.10`; fpm 1.18 passes `~` through into both package headers, and `dnf upgrade` from a `~rc.5` RPM to the GA RPM succeeds where the `_rc.5` one refused. The test now pins the full ordering contract under both package managers and fails against the previous spelling. The release-checksum step already normalizes `~` to `.` for every uploaded asset, so RPM assets need no further handling. Hosts that already installed an `_rc`/`_beta` RPM need a one-time `dnf install rustfs-1.0.0` or `rpm -Uvh --oldpackage` to reach GA. Fixes #8012 --- scripts/release/package_versions.sh | 16 ++++++++--- scripts/test_package_versions.sh | 42 +++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/scripts/release/package_versions.sh b/scripts/release/package_versions.sh index 2852a04bb..e265cfbef 100755 --- a/scripts/release/package_versions.sh +++ b/scripts/release/package_versions.sh @@ -37,6 +37,14 @@ esac semver_core='(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' prerelease_id='(alpha|beta|rc)\.(0|[1-9][0-9]*)' +# Every SemVer pre-release separator becomes '~' in both package formats. +# dpkg and rpm (>= 4.10) both treat '~' as "sorts before anything, even the +# empty string", so 1.0.0~rc.5 < 1.0.0 and 1.0.0~rc.5~preview.2 < 1.0.0~rc.5. +# Neither '_' (an ordinary rpm segment separator, which makes 1.0.0_rc.5 sort +# above 1.0.0) nor a second '-' (which dpkg reads as the start of the Debian +# revision, so 1.0.0~rc.5-preview.2 sorts above 1.0.0~rc.5) preserves the +# SemVer ordering. GitHub stores '~' in asset names as '.'; package.yml +# accounts for that when it writes the release checksums. tilde='~' case "$build_type" in @@ -65,8 +73,8 @@ case "$build_type" in [[ "$source_version" =~ ^${semver_core}-${prerelease_id}$ ]] || fail "prerelease version must be strict alpha, beta, or rc SemVer" - deb_version=${source_version/-/$tilde} - rpm_version=${source_version//-/_} + deb_version=${source_version//-/$tilde} + rpm_version=${source_version//-/$tilde} rpm_release=1 ;; preview) @@ -74,8 +82,8 @@ case "$build_type" in [[ "$source_version" =~ ^${semver_core}-${prerelease_id}-preview\.(0|[1-9][0-9]*)$ ]] || fail "preview version must be strict prerelease-preview SemVer" - deb_version=${source_version/-/$tilde} - rpm_version=${source_version//-/_} + deb_version=${source_version//-/$tilde} + rpm_version=${source_version//-/$tilde} rpm_release=1 ;; *) fail "unsupported build type" ;; diff --git a/scripts/test_package_versions.sh b/scripts/test_package_versions.sh index ec44dd2fe..a847de815 100755 --- a/scripts/test_package_versions.sh +++ b/scripts/test_package_versions.sh @@ -74,16 +74,16 @@ run_success stable-amd64 \ $'deb_version=1.2.3\nrpm_version=1.2.3\nrpm_release=1\ndeb_file=rustfs_1.2.3_amd64.deb\nrpm_file=rustfs-1.2.3-1.x86_64.rpm' \ release 1.2.3 '' amd64 x86_64 run_success alpha-arm64 \ - $'deb_version=1.2.3~alpha.1\nrpm_version=1.2.3_alpha.1\nrpm_release=1\ndeb_file=rustfs_1.2.3~alpha.1_arm64.deb\nrpm_file=rustfs-1.2.3_alpha.1-1.aarch64.rpm' \ + $'deb_version=1.2.3~alpha.1\nrpm_version=1.2.3~alpha.1\nrpm_release=1\ndeb_file=rustfs_1.2.3~alpha.1_arm64.deb\nrpm_file=rustfs-1.2.3~alpha.1-1.aarch64.rpm' \ prerelease 1.2.3-alpha.1 '' arm64 aarch64 run_success beta-amd64 \ - $'deb_version=1.2.3~beta.2\nrpm_version=1.2.3_beta.2\nrpm_release=1\ndeb_file=rustfs_1.2.3~beta.2_amd64.deb\nrpm_file=rustfs-1.2.3_beta.2-1.x86_64.rpm' \ + $'deb_version=1.2.3~beta.2\nrpm_version=1.2.3~beta.2\nrpm_release=1\ndeb_file=rustfs_1.2.3~beta.2_amd64.deb\nrpm_file=rustfs-1.2.3~beta.2-1.x86_64.rpm' \ prerelease 1.2.3-beta.2 '' amd64 x86_64 run_success rc-amd64 \ - $'deb_version=1.2.3~rc.4\nrpm_version=1.2.3_rc.4\nrpm_release=1\ndeb_file=rustfs_1.2.3~rc.4_amd64.deb\nrpm_file=rustfs-1.2.3_rc.4-1.x86_64.rpm' \ + $'deb_version=1.2.3~rc.4\nrpm_version=1.2.3~rc.4\nrpm_release=1\ndeb_file=rustfs_1.2.3~rc.4_amd64.deb\nrpm_file=rustfs-1.2.3~rc.4-1.x86_64.rpm' \ prerelease 1.2.3-rc.4 '' amd64 x86_64 run_success preview-amd64 \ - $'deb_version=1.0.0~rc.5-preview.2\nrpm_version=1.0.0_rc.5_preview.2\nrpm_release=1\ndeb_file=rustfs_1.0.0~rc.5-preview.2_amd64.deb\nrpm_file=rustfs-1.0.0_rc.5_preview.2-1.x86_64.rpm' \ + $'deb_version=1.0.0~rc.5~preview.2\nrpm_version=1.0.0~rc.5~preview.2\nrpm_release=1\ndeb_file=rustfs_1.0.0~rc.5~preview.2_amd64.deb\nrpm_file=rustfs-1.0.0~rc.5~preview.2-1.x86_64.rpm' \ preview 1.0.0-rc.5-preview.2 '' amd64 x86_64 run_success development-amd64 \ "deb_version=0~dev.7463.${sha} @@ -126,23 +126,43 @@ run_failure newline release $'1.2.3\nforged=1' '' amd64 x86_64 run_failure unsupported-deb-arch release 1.2.3 '' x86_64 x86_64 run_failure mismatched-arch release 1.2.3 '' amd64 aarch64 +# Ordering contract shared by both package managers: every pre-release sorts +# below its final release, every preview sorts below the pre-release it +# previews, and pre-release kinds/numbers keep their SemVer order. if command -v dpkg >/dev/null 2>&1; then dpkg --compare-versions "0~dev.7462.${sha}" lt "0~dev.7463.${sha}" dpkg --compare-versions "0~dev.7463.${sha}" lt 0.1.0 dpkg --compare-versions 1.2.3~rc.4 lt 1.2.3 - passed=$((passed + 3)) + dpkg --compare-versions 1.2.3~alpha.1 lt 1.2.3~beta.2 + dpkg --compare-versions 1.2.3~beta.2 lt 1.2.3~rc.4 + dpkg --compare-versions 1.2.3~rc.9 lt 1.2.3~rc.10 + dpkg --compare-versions 1.0.0~rc.5~preview.2 lt 1.0.0~rc.5 + dpkg --compare-versions 1.0.0~rc.5~preview.1 lt 1.0.0~rc.5~preview.2 + dpkg --compare-versions 1.0.0~rc.5~preview.2 lt 1.0.0~rc.6 + dpkg --compare-versions 1.0.0~rc.5 lt 1.0.1 + passed=$((passed + 10)) elif [[ $require_package_managers == true ]]; then printf 'FAIL package ordering: dpkg is required\n' >&2 exit 1 fi if command -v rpm >/dev/null 2>&1; then - rpm_old="0-0.dev.7462.${sha}" - rpm_new="0-0.dev.7463.${sha}" - rpm_release=0.1.0-1 - [[ $(rpm --eval "%{lua: print(rpm.vercmp('${rpm_old}', '${rpm_new}'))}") == -1 ]] - [[ $(rpm --eval "%{lua: print(rpm.vercmp('${rpm_new}', '${rpm_release}'))}") == -1 ]] - passed=$((passed + 2)) + rpm_lt() { + [[ $(rpm --eval "%{lua: print(rpm.vercmp('$1', '$2'))}") == -1 ]] || + { printf 'FAIL rpm ordering: expected %s < %s\n' "$1" "$2" >&2; exit 1; } + passed=$((passed + 1)) + } + rpm_lt "0-0.dev.7462.${sha}" "0-0.dev.7463.${sha}" + rpm_lt "0-0.dev.7463.${sha}" 0.1.0-1 + # The former 1.2.3_rc.4 spelling compared as newer than 1.2.3 (issue #8012). + rpm_lt 1.2.3~rc.4-1 1.2.3-1 + rpm_lt 1.2.3~alpha.1-1 1.2.3~beta.2-1 + rpm_lt 1.2.3~beta.2-1 1.2.3~rc.4-1 + rpm_lt 1.2.3~rc.9-1 1.2.3~rc.10-1 + rpm_lt 1.0.0~rc.5~preview.2-1 1.0.0~rc.5-1 + rpm_lt 1.0.0~rc.5~preview.1-1 1.0.0~rc.5~preview.2-1 + rpm_lt 1.0.0~rc.5~preview.2-1 1.0.0~rc.6-1 + rpm_lt 1.0.0~rc.5-1 1.0.1-1 elif [[ $require_package_managers == true ]]; then printf 'FAIL package ordering: rpm is required\n' >&2 exit 1 From 0fcd34c331818de47828206395a5d1cda3e280c7 Mon Sep 17 00:00:00 2001 From: Chris Date: Sat, 19 Sep 2026 13:41:16 +0800 Subject: [PATCH 04/30] fix(ecstore): retry fleet capability probe while the notification system boots (#8017) Startup finalizes IAM before init_notification_runtime publishes the notification system, and IAM finalization is what starts the fleet capability probe. The first probe pass therefore always failed closed and then slept the full ten-second probe interval. On a single node that left every durable capability, including the durable hard quota fence, unavailable for about ten seconds after /health already reported ok, so SetBucketQuota answered 503 durable quota capability is not confirmed across the cluster during that window. Publishing the notification system now wakes the probe immediately through a Notify permit, with a 100ms bootstrap poll as the fallback for a wakeup that races the availability check. The probe keeps failing closed while the system is absent and logs the wait once. The four probe futures no longer carry an unreachable notification-system-unavailable arm, and the seven proof slots are revoked through one helper. Fixes #8014 --- .../ecstore/src/services/notification_sys.rs | 183 ++++++++++++------ 1 file changed, 125 insertions(+), 58 deletions(-) diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index fcf07ee89..35889c127 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -42,6 +42,7 @@ use std::sync::{ atomic::{AtomicBool, AtomicUsize, Ordering}, }; use std::time::{Duration, Instant, SystemTime}; +use tokio::sync::Notify; use tokio::time::{sleep, timeout}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -61,6 +62,14 @@ const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5); const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10); const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30); +/// Fallback poll cadence while startup has not yet published the notification +/// system. IAM finalization starts the probe before `init_notification_runtime` +/// runs, so the first pass always fails closed; waiting a full probe interval +/// there left a single node without its durable quota capability for ten +/// seconds after `/health` reported ok (rustfs/rustfs#8014). Publication wakes +/// the probe immediately through `NOTIFICATION_SYS_PUBLISHED`; this bound only +/// covers a wakeup that races the availability check. +const REMOTE_VERSION_STATE_PROBE_BOOTSTRAP_INTERVAL: Duration = Duration::from_millis(100); const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2; const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3; const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4; @@ -338,6 +347,9 @@ static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock> = OnceLock::new(); static TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF: OnceLock> = OnceLock::new(); static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock = OnceLock::new(); +/// Signalled once `GLOBAL_NOTIFICATION_SYS` is published so the fleet probe can +/// run its first real pass without waiting for the bootstrap poll. +static NOTIFICATION_SYS_PUBLISHED: Notify = Notify::const_new(); static ILM_RECOVERY_EXPORT_LOCAL_PROCESS_EPOCH: LazyLock = LazyLock::new(Uuid::new_v4); fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock { @@ -368,6 +380,24 @@ fn transition_transaction_compaction_fleet_proof_slot() -> &'static std::sync::R TRANSITION_TRANSACTION_COMPACTION_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) } +fn all_fleet_capability_proof_slots() -> [&'static std::sync::RwLock; 7] { + [ + remote_version_state_fleet_proof_slot(), + cross_pool_fence_fleet_proof_slot(), + tier_delete_journal_fleet_proof_slot(), + decommission_target_fence_fleet_proof_slot(), + legacy_transition_state_reconcile_fleet_proof_slot(), + ilm_recovery_export_fleet_proof_slot(), + transition_transaction_compaction_fleet_proof_slot(), + ] +} + +fn revoke_all_fleet_capability_proofs() { + for slot in all_fleet_capability_proof_slots() { + revoke_fleet_capability_proof(slot); + } +} + fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) { if let Some(proof) = state.proof.take() { proof.generation.revoke(); @@ -1226,15 +1256,7 @@ fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap, pe pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() { if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) { - for slot in [ - remote_version_state_fleet_proof_slot(), - cross_pool_fence_fleet_proof_slot(), - tier_delete_journal_fleet_proof_slot(), - decommission_target_fence_fleet_proof_slot(), - legacy_transition_state_reconcile_fleet_proof_slot(), - ilm_recovery_export_fleet_proof_slot(), - transition_transaction_compaction_fleet_proof_slot(), - ] { + for slot in all_fleet_capability_proof_slots() { mark_fleet_capability_topology_conflict(slot); } } @@ -1242,53 +1264,59 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { } tokio::spawn(async move { + let mut notification_sys_unavailable_logged = false; loop { - let notification_sys = get_global_notification_sys(); - let remote_version_state_probe = async { - match notification_sys.as_ref() { - Some(notification_sys) => timeout( - REMOTE_VERSION_STATE_PROBE_TIMEOUT, - notification_sys.probe_remote_version_state_fleet(&topology_fingerprint), - ) - .await - .unwrap_or_else(|_| Err(Error::other("remote version state fleet capability probe timed out"))), - None => Err(Error::other("remote version state fleet capability notification system is unavailable")), + let Some(notification_sys) = get_global_notification_sys() else { + // Startup publishes the notification system after this probe + // is started. Stay failed closed, but retry on the bootstrap + // cadence so a single node gains its capabilities as soon as + // the system appears instead of one full probe interval later. + revoke_all_fleet_capability_proofs(); + if !notification_sys_unavailable_logged { + notification_sys_unavailable_logged = true; + debug!( + event = EVENT_NOTIFICATION_CAPABILITY_PROBE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + state = "waiting_for_notification_system", + "notification capability probe" + ); } + let _ = timeout(REMOTE_VERSION_STATE_PROBE_BOOTSTRAP_INTERVAL, NOTIFICATION_SYS_PUBLISHED.notified()).await; + continue; + }; + notification_sys_unavailable_logged = false; + let remote_version_state_probe = async { + timeout( + REMOTE_VERSION_STATE_PROBE_TIMEOUT, + notification_sys.probe_remote_version_state_fleet(&topology_fingerprint), + ) + .await + .unwrap_or_else(|_| Err(Error::other("remote version state fleet capability probe timed out"))) }; let cross_pool_fence_probe = async { - match notification_sys.as_ref() { - Some(notification_sys) => timeout( - REMOTE_VERSION_STATE_PROBE_TIMEOUT, - notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint), - ) - .await - .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))), - None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")), - } + timeout( + REMOTE_VERSION_STATE_PROBE_TIMEOUT, + notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint), + ) + .await + .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))) }; let recovery_export_probe = async { - match notification_sys.as_ref() { - Some(notification_sys) => timeout( - REMOTE_VERSION_STATE_PROBE_TIMEOUT, - notification_sys.probe_ilm_recovery_export_fleet(&topology_fingerprint), - ) - .await - .unwrap_or_else(|_| Err(Error::other("ILM recovery export fleet capability probe timed out"))), - None => Err(Error::other("ILM recovery export fleet capability notification system is unavailable")), - } + timeout( + REMOTE_VERSION_STATE_PROBE_TIMEOUT, + notification_sys.probe_ilm_recovery_export_fleet(&topology_fingerprint), + ) + .await + .unwrap_or_else(|_| Err(Error::other("ILM recovery export fleet capability probe timed out"))) }; let transition_transaction_compaction_probe = async { - match notification_sys.as_ref() { - Some(notification_sys) => timeout( - REMOTE_VERSION_STATE_PROBE_TIMEOUT, - notification_sys.probe_transition_transaction_compaction_fleet(&topology_fingerprint), - ) - .await - .unwrap_or_else(|_| Err(Error::other("transition transaction compaction fleet capability probe timed out"))), - None => Err(Error::other( - "transition transaction compaction fleet capability notification system is unavailable", - )), - } + timeout( + REMOTE_VERSION_STATE_PROBE_TIMEOUT, + notification_sys.probe_transition_transaction_compaction_fleet(&topology_fingerprint), + ) + .await + .unwrap_or_else(|_| Err(Error::other("transition transaction compaction fleet capability probe timed out"))) }; let (result, fence_probe, recovery_export_result, transition_transaction_compaction_result) = tokio::join!( remote_version_state_probe, @@ -1313,13 +1341,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { .unwrap_or_else(std::sync::PoisonError::into_inner) .topology_conflict; if topology_conflict { - revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot()); - revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot()); - revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot()); - revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot()); - revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot()); - revoke_fleet_capability_proof(ilm_recovery_export_fleet_proof_slot()); - revoke_fleet_capability_proof(transition_transaction_compaction_fleet_proof_slot()); + revoke_all_fleet_capability_proofs(); } else if let Some(err) = publish_fleet_capability_probe_result( remote_version_state_fleet_proof_slot(), &topology_fingerprint, @@ -1442,9 +1464,11 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { } pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> { - let _ = GLOBAL_NOTIFICATION_SYS - .set(Arc::new(NotificationSys::new(eps).await)) - .map_err(|_| Error::other("init notification_sys fail")); + if GLOBAL_NOTIFICATION_SYS.set(Arc::new(NotificationSys::new(eps).await)).is_ok() { + // `notify_one` stores a permit, so a probe that checks availability + // just before this publication still wakes without losing the signal. + NOTIFICATION_SYS_PUBLISHED.notify_one(); + } Ok(()) } @@ -3919,6 +3943,49 @@ mod tests { stats } + /// rustfs/rustfs#8014: IAM finalization starts the fleet probe before the + /// notification system is published. The first probe therefore fails + /// closed, and a single node used to wait a full probe interval before the + /// durable quota capability appeared, even though `/health` was already + /// reporting ok. The probe must retry promptly while it waits for startup + /// to publish the notification system. + #[tokio::test] + async fn fleet_probe_publishes_promptly_once_the_notification_system_appears() { + if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get().is_some() || GLOBAL_NOTIFICATION_SYS.get().is_some() { + // Process-wide state was bound by another test in this binary. The + // startup ordering under test needs a fresh process; nextest + // (the authoritative runner) always provides one. + eprintln!("skipping: fleet probe globals already bound in this process"); + return; + } + + // Share the fingerprint every other proof-installing test in this crate + // binds, so the plain `cargo test` fallback cannot see two topologies. + start_remote_version_state_fleet_probe("object-transaction-fencing-test".to_string()); + sleep(Duration::from_millis(300)).await; + assert!( + acquire_cross_pool_fence_fleet_proof().is_none(), + "no capability proof may exist before the notification system is published" + ); + + let published_at = Instant::now(); + new_global_notification_sys(EndpointServerPools::default()) + .await + .expect("single-node notification system initializes"); + + let observed = timeout(Duration::from_secs(2), async { + while acquire_cross_pool_fence_fleet_proof().is_none() { + sleep(Duration::from_millis(20)).await; + } + }) + .await; + assert!( + observed.is_ok(), + "single-node cross-pool fence proof should publish within 2s of the notification system, waited {:?}", + published_at.elapsed() + ); + } + #[test] fn merging_peer_rings_sums_each_tier_without_double_counting() { let mut cluster = DailyAllTierStats::from([("WARM".to_string(), ring(10))]); From c3269ad4c2e1d1f0e2d693ea074372405cadde70 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 19 Sep 2026 13:41:30 +0800 Subject: [PATCH 05/30] fix(heal): close tail-truncation and stale current-marker gaps (#8019) * fix(heal): verify complete encoded shard tails * fix(heal): prove stale current marker deletion --- .../ecstore/src/io_support/shard_integrity.rs | 6 +- crates/ecstore/src/set_disk/mod.rs | 66 +++++ crates/ecstore/src/set_disk/ops/heal.rs | 230 +++++++++++++++++- crates/ecstore/src/set_disk/ops/object.rs | 5 +- crates/ecstore/src/store/object.rs | 94 +++++++ 5 files changed, 386 insertions(+), 15 deletions(-) diff --git a/crates/ecstore/src/io_support/shard_integrity.rs b/crates/ecstore/src/io_support/shard_integrity.rs index b47c5dcd7..9c8f9c417 100644 --- a/crates/ecstore/src/io_support/shard_integrity.rs +++ b/crates/ecstore/src/io_support/shard_integrity.rs @@ -587,7 +587,11 @@ pub(crate) async fn verify_deep_parts( return Err(DiskError::FileCorrupt); } let part_status = statuses.get_mut(&part_index).ok_or(DiskError::FileCorrupt)?; - let length = erasure.shard_file_offset(0, part.size, part.size); + // Deep verification reads the complete encoded shard. `shard_file_offset` + // is a range-read helper and can stop before the final encoded byte for + // a partial data block, which would let a one-byte tail truncation pass + // as healthy. Use the physical shard length for the integrity proof. + let length = usize::try_from(erasure.shard_file_size(part.size as i64)).map_err(|_| DiskError::FileCorrupt)?; let mut readers = Vec::with_capacity(disks.len()); for (index, disk) in disks.iter().enumerate() { if part_status[index] != CHECK_PART_UNKNOWN { diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 623485066..b9e766c2f 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -3915,6 +3915,72 @@ pub struct SetDisks { >, } +fn marker_purge_receipt_path( + bucket: &str, + object: &str, + version: Uuid, + purge: &rustfs_common::mrf_channel::MrfDeleteMarkerPurge, +) -> String { + let object_hash = rustfs_utils::crypto::hex(Sha256::digest(object.as_bytes())); + let marker_hash = rustfs_utils::crypto::hex(purge.marker_identity); + format!( + "marker-purge-receipts/{}/{}/{}-{}-{}.json", + bucket, purge.bucket_incarnation_id, version, object_hash, marker_hash + ) +} + +impl SetDisks { + /// Persist a quorum DELETE receipt outside the object marker itself. The + /// receipt is written to the metadata bucket so it survives a node restart + /// while an inaccessible member still holds the stale marker. + pub(crate) async fn persist_marker_purge_receipt( + &self, + bucket: &str, + object: &str, + version: Uuid, + purge: &rustfs_common::mrf_channel::MrfDeleteMarkerPurge, + ) -> bool { + let path = marker_purge_receipt_path(bucket, object, version, purge); + let api = Arc::new(self.clone()); + crate::config::com::save_config_with_opts( + api, + &path, + b"rustfs-marker-purge-receipt-v1".to_vec(), + &ObjectOptions { + max_parity: true, + no_lock: true, + ..Default::default() + }, + ) + .await + .is_ok() + } + + pub(crate) async fn has_marker_purge_receipt( + &self, + bucket: &str, + object: &str, + version: Uuid, + purge: &rustfs_common::mrf_channel::MrfDeleteMarkerPurge, + ) -> bool { + let path = marker_purge_receipt_path(bucket, object, version, purge); + crate::config::com::read_config_limited_preserve_empty(Arc::new(self.clone()), &path, 128) + .await + .is_ok() + } + + pub(crate) async fn consume_marker_purge_receipt( + &self, + bucket: &str, + object: &str, + version: Uuid, + purge: &rustfs_common::mrf_channel::MrfDeleteMarkerPurge, + ) { + let path = marker_purge_receipt_path(bucket, object, version, purge); + let _ = crate::config::com::delete_config_no_lock(Arc::new(self.clone()), &path).await; + } +} + /// Read every physical copy before selecting a version quorum. A minority /// legacy record is still evidence and must not disappear behind a majority /// not-found result. Only an explicit file/volume absence produces `None`; diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 6235cb7d1..bb80c1b92 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -38,6 +38,7 @@ use crate::disk::DiskAPI; use crate::disk::local::{DELETE_DATA_DIR_MARKER_PREFIX, metadata_less_part_file}; use crate::io_support::bitrot::object_mmap_read_enabled; use crate::storage_api_contracts::namespace::NamespaceLocking as _; +use rustfs_common::mrf_channel::MrfDeleteMarkerPurge; use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit}; use tracing::trace; @@ -1469,7 +1470,12 @@ impl SetDisks { }) .transpose() .map_err(DiskError::from)?; - let till_offset = erasure.shard_file_offset(0, part.size, part.size); + // This reader covers the whole encoded shard. A range + // offset can exclude the final encoded byte when the + // logical part ends inside a data block, allowing a + // one-byte tail truncation to evade deep Heal. + let till_offset = usize::try_from(erasure.shard_file_size(part.size as i64)) + .map_err(|_| DiskError::FileCorrupt)?; let use_mmap_read = object_mmap_read_enabled(); let mut readers = Vec::with_capacity(latest_disks.len()); @@ -1907,13 +1913,26 @@ impl SetDisks { .zip(&errs) .any(|(info, error)| error.is_none() && info.is_canonical_delete_marker()) { - let cleanup = match Self::retired_marker_candidate(&parts_metadata, &errs) { - Ok(marker) => { - self.remove_retired_marker(retirement, bucket, object, marker, opts, &disks) - .await - } - Err(error) => Err(error), - }; + // A current-generation delete marker can be left behind on + // an inaccessible member after an acknowledged S3 DELETE. + // That case has no bucket-retirement record, so evaluate its + // stricter same-incarnation proof before the retired-marker + // path. The proof requires an exact marker identity, at + // least one replica already absent, and every target online. + let cleanup = + match Self::same_incarnation_marker_candidate(&parts_metadata, &errs, retirement.current_incarnation) { + Ok(marker) => { + self.remove_current_marker(retirement, bucket, object, marker, opts, &disks) + .await + } + Err(_) => match Self::retired_marker_candidate(&parts_metadata, &errs) { + Ok(marker) => { + self.remove_retired_marker(retirement, bucket, object, marker, opts, &disks) + .await + } + Err(error) => Err(error), + }, + }; let absent = cleanup.is_ok(); let item = self .dangling_heal_result(FileInfo::default(), &errs, bucket, object, version_id, absent) @@ -2027,6 +2046,41 @@ impl SetDisks { candidate.ok_or_else(|| DiskError::retired_marker_deferred("no exact marker candidate")) } + fn same_incarnation_marker_candidate( + metadata: &[FileInfo], + errors: &[Option], + current_incarnation: Option, + ) -> disk::error::Result { + let defer = DiskError::retired_marker_deferred; + let Some(current) = current_incarnation.filter(|id| !id.is_nil()) else { + return Err(defer("current bucket incarnation is unavailable")); + }; + let mut candidate = None; + let mut missing_replica = false; + for (info, error) in metadata.iter().zip(errors) { + match error { + Some(DiskError::FileNotFound | DiskError::FileVersionNotFound) => { + missing_replica = true; + continue; + } + Some(_) => return Err(defer("not every replica is readable")), + None => {} + } + if !info.is_canonical_delete_marker() || info.delete_marker_incarnation() != Some(current) { + return Err(defer("marker is not a canonical current-generation marker")); + } + let marker = rustfs_filemeta::MetaDeleteMarker::from(info.clone()); + if candidate.as_ref().is_some_and(|previous| previous != &marker) { + return Err(defer("surviving marker identities conflict")); + } + candidate = Some(marker); + } + if !missing_replica { + return Err(defer("marker is present on every replica")); + } + candidate.ok_or_else(|| defer("no exact current-generation marker candidate")) + } + async fn remove_retired_marker( &self, context: &MarkerRetirementContext<'_>, @@ -2065,6 +2119,72 @@ impl SetDisks { if context.lifecycle_guard.is_lock_lost() { return Err(defer("bucket lifecycle fence was lost")); } + let result = self + .remove_marker_exact_under_fence(bucket, object, marker, version, disks) + .await; + if context.lifecycle_guard.is_lock_lost() { + return Err(defer("bucket lifecycle fence was lost")); + } + result + } + + async fn remove_current_marker( + &self, + context: &MarkerRetirementContext<'_>, + bucket: &str, + object: &str, + marker: rustfs_filemeta::MetaDeleteMarker, + opts: &HealOpts, + disks: &[Option], + ) -> disk::error::Result<()> { + let defer = DiskError::retired_marker_deferred; + if opts.dry_run || !opts.remove { + return Err(defer("cleanup requires remove=true and dry_run=false")); + } + if disks.len() != self.set_drive_count || disks.iter().any(Option::is_none) { + return Err(defer("every target disk must be online")); + } + let Some(current) = context.current_incarnation.filter(|id| !id.is_nil()) else { + return Err(defer("current bucket incarnation is unavailable")); + }; + if context.lifecycle_guard.is_lock_lost() { + return Err(defer("bucket lifecycle fence was lost")); + } + let incarnation = marker.into_fileinfo(bucket, object, false)?.delete_marker_incarnation(); + if incarnation != Some(current) { + return Err(defer("marker is not from the current bucket incarnation")); + } + let Some(version) = marker.version_id.filter(|id| !id.is_nil()) else { + return Err(defer("cleanup requires an explicit non-null version")); + }; + let marker_identity = marker.stable_identity(); + let marker_bytes = marker.marshal_msg().map_err(DiskError::other)?; + let Some(purge) = MrfDeleteMarkerPurge::new(current, current, marker_identity, marker_bytes) else { + return Err(defer("marker purge proof payload is invalid")); + }; + if !self.has_marker_purge_receipt(bucket, object, version, &purge).await { + return Err(defer("same-incarnation marker lacks an acknowledged delete receipt")); + } + let result = self + .remove_marker_exact_under_fence(bucket, object, marker, version, disks) + .await; + if context.lifecycle_guard.is_lock_lost() { + return Err(defer("bucket lifecycle fence was lost")); + } + if result.is_ok() { + self.consume_marker_purge_receipt(bucket, object, version, &purge).await; + } + result + } + + async fn remove_marker_exact_under_fence( + &self, + bucket: &str, + object: &str, + marker: rustfs_filemeta::MetaDeleteMarker, + version: Uuid, + disks: &[Option], + ) -> disk::error::Result<()> { let request = FileInfo { volume: bucket.to_owned(), name: object.to_owned(), @@ -2105,13 +2225,14 @@ impl SetDisks { let same_targets = selected.len() == disks.len() && selected.iter().zip(disks).all(|(current, original)| { matches!((current, original), (Some(current), Some(original)) if std::sync::Arc::ptr_eq(current, original)) }); - if context.lifecycle_guard.is_lock_lost() - || !same_targets + if !same_targets || !errors .iter() .all(|error| matches!(error, Some(DiskError::FileNotFound | DiskError::FileVersionNotFound))) { - return Err(defer("complete marker absence could not be verified under the original fence")); + return Err(DiskError::retired_marker_deferred( + "complete marker absence could not be verified under the original target set", + )); } self.invalidate_get_object_metadata_cache(bucket, object).await; Ok(()) @@ -3319,7 +3440,9 @@ mod heal_result_report_tests { use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, STORAGE_FORMAT_FILE, new_disk}; use crate::error::Error; use crate::object_api::{ObjectOptions, PutObjReader}; - use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated; + use crate::set_disk::ops::object::hermetic_set_disks_support::{ + hermetic_set_disks_for_pool_with_default_parity_isolated, hermetic_set_disks_isolated, + }; use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions}; use crate::storage_api_contracts::heal::HealOperations as _; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; @@ -4183,6 +4306,89 @@ mod heal_result_report_tests { ); } + #[tokio::test] + async fn deep_heal_rebuilds_h2_near_tail_truncated_part() { + let (temp_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(16, 0, 4).await; + let bucket = "deep-heal-h2-near-tail-truncation"; + let object = "object.bin"; + for disk in &disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let expected_payload = vec![0x6d; 5 * 1024 * 1024 + 123]; + set.put_object( + bucket, + object, + &mut PutObjReader::from_vec(expected_payload.clone()), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("source object should be written before shard truncation"); + let source = disks[2] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("source metadata should be readable"); + let data_dir = source.data_dir.expect("non-inline source should have a data directory"); + let truncated_part = temp_dirs[1] + .path() + .join(bucket) + .join(object) + .join(data_dir.to_string()) + .join("part.1"); + let original_len = tokio::fs::metadata(&truncated_part) + .await + .expect("target shard should exist") + .len(); + assert!(original_len > 1, "test shard must be large enough to truncate"); + let file = tokio::fs::OpenOptions::new() + .write(true) + .open(&truncated_part) + .await + .expect("target shard should be writable"); + file.set_len(original_len - 1) + .await + .expect("target shard should be truncated"); + + let mut reader = set + .get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default()) + .await + .expect("GET should remain readable after a one-byte shard truncation"); + let mut read_back = Vec::new(); + tokio::io::copy(&mut reader, &mut read_back) + .await + .expect("GET should reconstruct the truncated shard through EC"); + assert_eq!(read_back, expected_payload, "EC GET must preserve the object bytes"); + + let (result, error) = set + .heal_object( + bucket, + object, + "", + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("deep heal should finish after a one-byte H2 tail truncation"); + + assert!(error.is_none(), "deep heal should recover the H2 truncated shard: {error:?}"); + assert_eq!(result.drives_healed(), Some(1)); + assert_eq!(result.before.drives[1].state, DriveState::Corrupt.to_string()); + assert_eq!( + tokio::fs::metadata(&truncated_part) + .await + .expect("repaired H2 shard should exist") + .len(), + original_len, + "deep heal must restore the complete H2 shard length" + ); + } + #[tokio::test] async fn deep_heal_inline_bitrot_rebuilds_physical_data_and_parity_shards() { use crate::storage_api_contracts::range::HTTPRangeSpec; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 3eecfce5d..2c1e22ca2 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -7703,10 +7703,11 @@ impl SetDisks { drop(namespace_owner); if quorum_result.is_ok() && errs.iter().any(Option::is_some) - && let Some(purge) = delete_marker_purge + && let Some(purge) = delete_marker_purge.as_ref() && let Some(version) = fi.version_id.filter(|version| !version.is_nil()) { - let _ = self.persist_delete_marker_purge(bucket, object, version, purge).await; + let _ = self.persist_marker_purge_receipt(bucket, object, version, purge).await; + let _ = self.persist_delete_marker_purge(bucket, object, version, purge.clone()).await; } // An explicit purge can carry deleted=true for the existing marker. // It must not create a repair intent that could reintroduce that marker. diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index f138d06d0..f7c13e192 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -9124,6 +9124,100 @@ mod tests { assert_eq!(dirs.len(), 16); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn current_marker_delete_ack_removes_stale_rejoined_copy() { + let bucket = "current-marker-stale-rejoin"; + let object = "marker.bin"; + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original) = make_local_set_disks_with_ctx(16, 4, ctx.clone()).await; + let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], ctx).await); + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .handle_make_bucket( + bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create versioned bucket"); + let marker = store + .delete_object( + bucket, + object, + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("create current marker") + .version_id + .expect("marker has explicit version"); + let current = store + .bucket_incarnation_id_from_disk(bucket) + .await + .expect("bucket incarnation"); + let set = &store.pools[0].disk_set[0]; + let before = set.disks.read().await.clone(); + for slot in 12..16 { + set.disks.write().await[slot] = None; + } + store + .delete_object( + bucket, + object, + ObjectOptions { + versioned: true, + version_id: Some(marker.to_string()), + ..Default::default() + }, + ) + .await + .expect("acknowledged delete should remove the marker from the online quorum"); + *set.disks.write().await = before.clone(); + + for disk in before.iter().take(12).flatten() { + assert!(matches!( + disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default()) + .await, + Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound) + )); + } + for disk in before.iter().skip(12).flatten() { + assert!( + disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default()) + .await + .is_ok(), + "offline member must retain the stale current marker" + ); + } + let healed = store + .heal_object_at_incarnation( + bucket, + object, + &marker.to_string(), + current, + &rustfs_heal_contracts::heal_channel::HealOpts { + remove: true, + scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("stale current marker heal should complete"); + assert!(healed.error.is_none(), "stale current marker heal: {:?}", healed.error); + assert!(healed.absence.as_ref().is_some_and(|proof| proof.removed)); + for disk in before.iter().flatten() { + assert!(matches!( + disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default()) + .await, + Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound) + )); + } + } + #[tokio::test] async fn multipool_delete_marker_stays_with_existing_versions() { let bucket = "multipool-marker-routing"; From 0e03bcda49c6e8e8890f4a4b15ae1c389f4495ea Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 19 Sep 2026 13:41:39 +0800 Subject: [PATCH 06/30] fix: serialize notify runtime module switch reconciliation (#8020) --- rustfs/src/admin/handlers/module_switch.rs | 87 ++++++++++++---------- rustfs/src/server/event.rs | 61 ++++++++++++++- rustfs/src/server/mod.rs | 2 +- 3 files changed, 107 insertions(+), 43 deletions(-) diff --git a/rustfs/src/admin/handlers/module_switch.rs b/rustfs/src/admin/handlers/module_switch.rs index f560df991..47739bf9a 100644 --- a/rustfs/src/admin/handlers/module_switch.rs +++ b/rustfs/src/admin/handlers/module_switch.rs @@ -26,7 +26,7 @@ use crate::server::{ apply_audit_module_switch_for_context, current_module_switch_snapshot, mark_event_notifier_reconciled, mark_event_notifier_unreconciled, refresh_audit_module_enabled, refresh_notify_module_enabled, refresh_persisted_module_switches_from, refresh_persisted_module_switches_from_store, save_persisted_module_switches_to, - validate_module_switch_update, + validate_module_switch_update, with_notify_runtime_reconcile_lock, }; use http::{HeaderMap, StatusCode}; use hyper::Method; @@ -138,48 +138,53 @@ async fn refresh_module_switch_snapshot() -> S3Result { async fn apply_module_switch_update(context: Arc, switches: PersistedModuleSwitches) -> S3Result<()> { preflight_dynamic_config_reload_for_context(Some(context.as_ref()), MODULE_SWITCHES_SIGNAL_SUBSYSTEM).await?; let store = context.object_store(); - mark_event_notifier_unreconciled(); - let notification_system = rustfs_notify::ensure_live_events(); - if switches.notify_enabled { - notification_system - .reload_persisted_config_from_store(store.clone()) - .await - .map_err(|err| { - tracing::warn!(error = %err, "Failed to load notification config for module switch update"); - s3_error!(InternalError, "failed to load notification config") - })?; - } + let mut failures = with_notify_runtime_reconcile_lock(async { + mark_event_notifier_unreconciled(); + let notification_system = rustfs_notify::ensure_live_events(); + if switches.notify_enabled { + notification_system + .reload_persisted_config_from_store(store.clone()) + .await + .map_err(|err| { + tracing::warn!(error = %err, "Failed to load notification config for module switch update"); + s3_error!(InternalError, "failed to load notification config") + })?; + } - let transition_system = notification_system.clone(); - let notify_transition = save_persisted_module_switches_to(store.clone(), switches, move || { - let enabled = refresh_notify_module_enabled(); - transition_system.publish_targets_enabled(enabled, None) + let transition_system = notification_system.clone(); + let notify_transition = save_persisted_module_switches_to(store.clone(), switches, move || { + let enabled = refresh_notify_module_enabled(); + transition_system.publish_targets_enabled(enabled, None) + }) + .await + .map_err(|err| { + tracing::warn!(error = %err, "Failed to save module switches"); + s3_error!(InternalError, "failed to save module switches") + })?; + + let mut failures = Vec::new(); + let mut notify_converged = true; + if let Err(err) = notify_transition.wait().await { + tracing::warn!(error = %err, "Local notification runtime failed to apply module switch update"); + notify_converged = false; + failures.push("local notify"); + } + if !switches.notify_enabled + && let Err(err) = notification_system.reload_persisted_config_from_store(store).await + { + tracing::warn!(error = %err, "Local notification config cache failed to reload after module disable"); + notify_converged = false; + failures.push("local notify config cache"); + } + if notify_converged && notification_system.runtime_lifecycle_is_converged() { + mark_event_notifier_reconciled(); + } else if notify_converged { + failures.push("local notify convergence"); + } + + Ok::<_, s3s::S3Error>(failures) }) - .await - .map_err(|err| { - tracing::warn!(error = %err, "Failed to save module switches"); - s3_error!(InternalError, "failed to save module switches") - })?; - - let mut failures = Vec::new(); - let mut notify_converged = true; - if let Err(err) = notify_transition.wait().await { - tracing::warn!(error = %err, "Local notification runtime failed to apply module switch update"); - notify_converged = false; - failures.push("local notify"); - } - if !switches.notify_enabled - && let Err(err) = notification_system.reload_persisted_config_from_store(store).await - { - tracing::warn!(error = %err, "Local notification config cache failed to reload after module disable"); - notify_converged = false; - failures.push("local notify config cache"); - } - if notify_converged && notification_system.runtime_lifecycle_is_converged() { - mark_event_notifier_reconciled(); - } else if notify_converged { - failures.push("local notify convergence"); - } + .await?; if apply_audit_module_switch_for_context(Some(context.as_ref())).await.is_err() { tracing::warn!(reason = "apply_failed", "Local audit runtime failed to apply module switch update"); diff --git a/rustfs/src/server/event.rs b/rustfs/src/server/event.rs index 6f0769774..37b22bb71 100644 --- a/rustfs/src/server/event.rs +++ b/rustfs/src/server/event.rs @@ -29,6 +29,7 @@ use std::sync::OnceLock; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use tokio::spawn; +use tokio::sync::Mutex as AsyncMutex; use tokio::task::JoinHandle; use tokio::time::{Instant, MissedTickBehavior}; use tokio_util::sync::CancellationToken; @@ -38,6 +39,12 @@ static NOTIFY_RUNTIME_RECONCILED: AtomicBool = AtomicBool::new(false); static NOTIFY_BUCKET_RULES_RECONCILED: AtomicBool = AtomicBool::new(false); static ECSTORE_EVENT_DISPATCH_HOOK: OnceLock<()> = OnceLock::new(); +// Serialize local notification lifecycle publication with the periodic and +// peer-triggered reconciler. Lock order is this guard before the module-switch +// RMW/object locks and the server-config read lock; the module-switch handler's +// preliminary server-config read is released before it takes the RMW lock. +static NOTIFY_RUNTIME_RECONCILE_LOCK: AsyncMutex<()> = AsyncMutex::const_new(()); + const EVENT_NOTIFIER_RECONCILE_INTERVAL: Duration = Duration::from_secs(5); const EVENT_NOTIFIER_RECONCILE_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(120); const EVENT_NOTIFY_RUNTIME_RECONCILE: &str = "notify_runtime_reconcile"; @@ -55,6 +62,14 @@ pub(crate) fn mark_event_notifier_unreconciled() { NOTIFY_BUCKET_RULES_RECONCILED.store(false, Ordering::Release); } +pub(crate) async fn with_notify_runtime_reconcile_lock(operation: Fut) -> T +where + Fut: Future, +{ + let _guard = NOTIFY_RUNTIME_RECONCILE_LOCK.lock().await; + operation.await +} + fn are_bucket_notification_rules_reconciled() -> bool { NOTIFY_BUCKET_RULES_RECONCILED.load(Ordering::Acquire) } @@ -161,6 +176,12 @@ fn ensure_event_notifier_converged(system: &NotificationSystem) -> Result<(), No pub(crate) async fn reconcile_event_notifier_from_store( store: std::sync::Arc, +) -> Result<(), NotificationError> { + with_notify_runtime_reconcile_lock(reconcile_event_notifier_from_store_unlocked(store)).await +} + +async fn reconcile_event_notifier_from_store_unlocked( + store: std::sync::Arc, ) -> Result<(), NotificationError> { let result = async { validate_notify_module_env().map_err(NotificationError::Initialization)?; @@ -310,7 +331,7 @@ pub async fn shutdown_event_notifier() -> Result<(), NotificationError> { #[instrument] pub async fn init_event_notifier() -> Result<(), NotificationError> { - init_event_notifier_with_store(runtime_sources::current_object_store_handle).await + with_notify_runtime_reconcile_lock(init_event_notifier_with_store(runtime_sources::current_object_store_handle)).await } async fn init_event_notifier_with_store(current_store: CurrentStore) -> Result<(), NotificationError> @@ -408,6 +429,7 @@ mod tests { use super::{ convert_ecstore_object_info, init_event_notifier_with_store, mark_bucket_notification_rules_reconciled, parse_host_and_port, run_persisted_event_notifier_reconciler, should_reconcile_bucket_notification_rules, + with_notify_runtime_reconcile_lock, }; use crate::server::is_event_notifier_reconciled; use crate::storage_api::server::event::StorageObjectInfo; @@ -471,6 +493,43 @@ mod tests { .await; } + #[tokio::test] + #[serial] + async fn notify_runtime_reconcile_lock_serializes_overlapping_operations() { + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let second_attempted = Arc::new(Notify::new()); + let second_started = Arc::new(AtomicBool::new(false)); + + let first_entered = entered.clone(); + let first_release = release.clone(); + let first = tokio::spawn(with_notify_runtime_reconcile_lock(async move { + first_entered.notify_one(); + first_release.notified().await; + })); + entered.notified().await; + + let second_attempted_signal = second_attempted.clone(); + let second_started_flag = second_started.clone(); + let second = tokio::spawn(async move { + second_attempted_signal.notify_one(); + with_notify_runtime_reconcile_lock(async move { + second_started_flag.store(true, Ordering::Release); + }) + .await; + }); + second_attempted.notified().await; + assert!( + !second_started.load(Ordering::Acquire), + "overlapping reconcile work must wait for the first operation" + ); + + release.notify_one(); + first.await.expect("first reconcile operation should finish"); + second.await.expect("second reconcile operation should finish"); + assert!(second_started.load(Ordering::Acquire)); + } + #[test] fn parse_host_and_port_with_ipv4_and_port() { let (host, port) = parse_host_and_port("127.0.0.1:9000".to_string()); diff --git a/rustfs/src/server/mod.rs b/rustfs/src/server/mod.rs index 91b1ebc4d..7d79791ef 100644 --- a/rustfs/src/server/mod.rs +++ b/rustfs/src/server/mod.rs @@ -48,7 +48,7 @@ pub use service_state::wait_for_shutdown; // Items only used within the library crate (admin handlers, server/http.rs, etc.). pub(crate) use event::{ is_event_notifier_reconciled, mark_event_notifier_reconciled, mark_event_notifier_unreconciled, - reconcile_event_notifier_from_store, start_persisted_event_notifier_reconciler, + reconcile_event_notifier_from_store, start_persisted_event_notifier_reconciler, with_notify_runtime_reconcile_lock, }; #[cfg(test)] pub(crate) use health::{ From c35de00e8a0551b349668c403369c5a2406c854b Mon Sep 17 00:00:00 2001 From: Hauser Date: Sat, 19 Sep 2026 19:32:02 +0800 Subject: [PATCH 07/30] fix(scanner): protect single-disk foreground latency (#8022) Co-authored-by: zhi22915 --- crates/scanner/src/remote_scanner/stream.rs | 13 +++-- crates/scanner/src/scanner.rs | 4 +- crates/scanner/src/scanner/heal_info.rs | 15 ----- crates/scanner/src/scanner_folder.rs | 8 +++ crates/scanner/src/scanner_io.rs | 4 +- crates/scanner/src/scanner_io/cache.rs | 65 ++++++++++++++++++--- crates/scanner/src/scanner_io/io_cache.rs | 9 ++- 7 files changed, 82 insertions(+), 36 deletions(-) diff --git a/crates/scanner/src/remote_scanner/stream.rs b/crates/scanner/src/remote_scanner/stream.rs index f35af52dd..df8c1deba 100644 --- a/crates/scanner/src/remote_scanner/stream.rs +++ b/crates/scanner/src/remote_scanner/stream.rs @@ -16,8 +16,8 @@ use crate::RUSTFS_META_BUCKET; use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig}; use crate::scanner_io::{ - DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerCheckpointPersistResult, ScannerDiskScanOptions, - ScannerDiskScanOutcome, ScannerIODisk, acquire_scanner_cache_locks, cache_root_entry_info, + DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerCheckpointPersistContext, ScannerCheckpointPersistResult, + ScannerDiskScanOptions, ScannerDiskScanOutcome, ScannerIODisk, acquire_scanner_cache_locks, cache_root_entry_info, current_cache_root_or_prepare_with_generation, persist_scanner_checkpoint, scanner_set_disk_inventory, }; use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION; @@ -840,12 +840,15 @@ async fn scan_and_persist_local_bucket( } match persist_scanner_checkpoint( set.clone(), + ScannerCheckpointPersistContext { + ctx: &scan_ctx, + expected_publication_epoch, + cycle: next_cycle, + leader_epoch, + }, &cache_name, &checkpoint, &mut revisions, - expected_publication_epoch, - next_cycle, - leader_epoch, ) .await { diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index ea358cadb..fcd313ea9 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1778,9 +1778,7 @@ where mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; return ScannerCycleOutcome::Failed; } - BackgroundHealInfoReadStatus::ErasureSd - | BackgroundHealInfoReadStatus::Loaded - | BackgroundHealInfoReadStatus::Missing => {} + BackgroundHealInfoReadStatus::Loaded | BackgroundHealInfoReadStatus::Missing => {} } let mut background_heal_info = background_heal_read.info; let background_heal_epoch = background_heal_read.expected_epoch; diff --git a/crates/scanner/src/scanner/heal_info.rs b/crates/scanner/src/scanner/heal_info.rs index f7e3f61f9..5f898d44a 100644 --- a/crates/scanner/src/scanner/heal_info.rs +++ b/crates/scanner/src/scanner/heal_info.rs @@ -28,7 +28,6 @@ pub struct BackgroundHealInfo { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) enum BackgroundHealInfoReadStatus { - ErasureSd, Loaded, Missing, Blocked, @@ -66,15 +65,6 @@ pub(super) async fn read_background_heal_info_with_epoch(storeapi: Arc) -> where S: ScannerStorage, { - // Skip for ErasureSD setup - if storeapi.setup_is_erasure_sd().await { - return BackgroundHealInfoRead { - info: BackgroundHealInfo::default(), - expected_epoch: None, - status: BackgroundHealInfoReadStatus::ErasureSd, - }; - } - let expected_epoch = scanner_publication_epoch(storeapi.clone()).await; if expected_epoch.is_none() { return BackgroundHealInfoRead { @@ -146,11 +136,6 @@ pub(super) async fn save_background_heal_info_for_epoch( ) where S: ScannerStorage, { - // Skip for ErasureSD setup - if storeapi.setup_is_erasure_sd().await { - return; - } - // Serialize to JSON let data = match serde_json::to_vec(&info) { Ok(data) => data, diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index eedf06e4a..ac7c1abdb 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -1806,6 +1806,14 @@ impl FolderScanner { continue; } + // Do not start another metadata read while foreground work is + // active. The post-object timer protects the next request only + // after the read has already been dispatched; this admission + // point keeps the scanner from extending a single-disk I/O + // burst across foreground requests. + if crate::workload_admission::foreground_workload_activity() > 0 { + self.sleeper.sleep_folder().await; + } let timer = self.sleeper.timer(); let heal_enabled = this_hash.mod_alt( diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index e0374b4af..0ff8b2dfc 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -1609,8 +1609,8 @@ use dirty_usage::*; use guards::*; pub(crate) use cache::{ - DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerCheckpointPersistResult, acquire_scanner_cache_locks, - current_cache_root_or_prepare_with_generation, persist_scanner_checkpoint, + DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerCheckpointPersistContext, ScannerCheckpointPersistResult, + acquire_scanner_cache_locks, current_cache_root_or_prepare_with_generation, persist_scanner_checkpoint, }; pub use dirty_usage::{ ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageMutationObserver, diff --git a/crates/scanner/src/scanner_io/cache.rs b/crates/scanner/src/scanner_io/cache.rs index 9fc8a9004..fe4b6f36b 100644 --- a/crates/scanner/src/scanner_io/cache.rs +++ b/crates/scanner/src/scanner_io/cache.rs @@ -31,6 +31,41 @@ pub(crate) enum ScannerCheckpointPersistResult { Failed(StorageError), } +pub(crate) struct ScannerCheckpointPersistContext<'a> { + pub(crate) ctx: &'a CancellationToken, + pub(crate) expected_publication_epoch: u64, + pub(crate) cycle: u64, + pub(crate) leader_epoch: u64, +} + +const CHECKPOINT_FOREGROUND_QUIET_WAIT: Duration = Duration::from_secs(1); + +async fn wait_for_checkpoint_foreground_quiet(ctx: &CancellationToken) -> bool { + let deadline = tokio::time::Instant::now() + CHECKPOINT_FOREGROUND_QUIET_WAIT; + loop { + if crate::workload_admission::foreground_workload_activity() == 0 { + return true; + } + + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + if remaining.is_zero() { + return false; + } + + let backoff = Duration::from_millis( + crate::workload_admission::foreground_workload_activity() + .saturating_mul(10) + .min(250), + ) + .max(Duration::from_millis(10)) + .min(remaining); + tokio::select! { + _ = ctx.cancelled() => return false, + _ = tokio::time::sleep(backoff) => {} + } + } +} + /// Persist one bounded checkpoint and refresh its CAS revisions. /// /// Local and remote workers share the same publication/leader fencing and @@ -38,23 +73,37 @@ pub(crate) enum ScannerCheckpointPersistResult { /// at the caller because those guards have different concrete types. pub(crate) async fn persist_scanner_checkpoint( store: Arc, + context: ScannerCheckpointPersistContext<'_>, cache_name: &str, checkpoint: &DataUsageCache, revisions: &mut DataUsageCacheRevisions, - expected_publication_epoch: u64, - cycle: u64, - leader_epoch: u64, ) -> ScannerCheckpointPersistResult where S: ScannerObjectIO + ScannerConfigObjectDelete, { - if crate::remote_scanner::validate_remote_scanner_request_fence_with_store(cycle, leader_epoch, store.clone()) + let foreground_quiet = wait_for_checkpoint_foreground_quiet(context.ctx).await; + if !foreground_quiet && context.ctx.is_cancelled() { + return ScannerCheckpointPersistResult::FenceChanged; + } + if !foreground_quiet { + debug!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + cache_name, + state = "checkpoint_foreground_wait_expired", + "Scanner checkpoint foreground quiet wait expired; preserving bounded progress" + ); + } + + if crate::remote_scanner::validate_remote_scanner_request_fence_with_store(context.cycle, context.leader_epoch, store.clone()) .await .is_err() { return ScannerCheckpointPersistResult::FenceChanged; } - if scanner_publication_admission_for_epoch(store.clone(), expected_publication_epoch) + if scanner_publication_admission_for_epoch(store.clone(), context.expected_publication_epoch) .await .is_none() { @@ -62,16 +111,16 @@ where } if let Err(error) = checkpoint - .save_with_revisions_for_epoch(store.clone(), cache_name, revisions, expected_publication_epoch) + .save_with_revisions_for_epoch(store.clone(), cache_name, revisions, context.expected_publication_epoch) .await { return ScannerCheckpointPersistResult::Failed(error); } - if crate::remote_scanner::validate_remote_scanner_request_fence_with_store(cycle, leader_epoch, store.clone()) + if crate::remote_scanner::validate_remote_scanner_request_fence_with_store(context.cycle, context.leader_epoch, store.clone()) .await .is_err() - || scanner_publication_admission_for_epoch(store.clone(), expected_publication_epoch) + || scanner_publication_admission_for_epoch(store.clone(), context.expected_publication_epoch) .await .is_none() { diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index c3b190f7d..e64b1965e 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -1191,12 +1191,15 @@ impl ScannerIOCache for SetDisks { } match persist_scanner_checkpoint( store_clone_clone.clone(), + ScannerCheckpointPersistContext { + ctx: &ctx_clone, + expected_publication_epoch: expected_publication_epoch_clone, + cycle: want_cycle, + leader_epoch, + }, cache_name.as_str(), &checkpoint, &mut revisions, - expected_publication_epoch_clone, - want_cycle, - leader_epoch, ) .await { From 22243e791d37ca66ed5aa6d4c69b3e02bc384013 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sat, 19 Sep 2026 19:33:24 +0800 Subject: [PATCH 08/30] test(heal): cover stale current delete marker rejoin (#8021) --- crates/ecstore/src/set_disk/ops/heal.rs | 72 +++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index bb80c1b92..1f57c1142 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -4223,6 +4223,78 @@ mod heal_result_report_tests { ); } + #[tokio::test] + async fn deep_heal_rebuilds_a_stale_current_delete_marker_metadata_replica() { + let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "deep-heal-stale-current-marker"; + let object = "object.bin"; + for disk in &disks { + disk.make_volume(bucket).await.expect("test bucket should be created"); + } + + let versioned = ObjectOptions { + no_lock: true, + versioned: true, + ..Default::default() + }; + let original = set + .put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x41; 1024]), &versioned) + .await + .expect("versioned source object should be written"); + let stale_metadata = disks[0] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("pre-marker metadata should be readable"); + + let marker = set + .delete_object(bucket, object, versioned) + .await + .expect("versioned delete should create a current marker"); + let marker_version = marker.version_id.expect("delete marker should have a version id"); + + // Simulate a member that was offline while the acknowledged marker was + // committed on the other members: its physical root still contains the + // pre-delete metadata and the original data version. + let mut stale_metadata = stale_metadata; + stale_metadata.fresh = true; + disks[0] + .write_metadata("", bucket, object, stale_metadata) + .await + .expect("stale member metadata should be restored for the fixture"); + assert_eq!( + disks[0] + .read_version("", bucket, object, &marker_version.to_string(), &ReadOptions::default()) + .await + .expect_err("stale member must not contain the current marker") + .to_string(), + "file version not found" + ); + + let (result, error) = set + .heal_object( + bucket, + object, + &marker_version.to_string(), + &HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("deep heal should report the stale marker result"); + + assert!(error.is_none(), "deep heal should repair stale marker metadata: {error:?}"); + assert_eq!(result.drives_healed(), Some(1)); + let repaired = disks[0] + .read_version("", bucket, object, &marker_version.to_string(), &ReadOptions::default()) + .await + .expect("rejoined member should contain the marker"); + assert!(repaired.deleted, "repaired metadata must remain a delete marker"); + assert_eq!(repaired.version_id, Some(marker_version)); + assert!(original.version_id.is_some(), "source object should have a version id"); + } + #[tokio::test] async fn deep_heal_rebuilds_near_tail_truncated_part() { let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; From ed2c3eaf778c98b02106a7e1142a4ceddfc851f1 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Sun, 20 Sep 2026 11:27:41 +0800 Subject: [PATCH 09/30] fix(package): preserve service state across upgrades (#8024) * fix(package): preserve service state across upgrades * fix(package): match legacy DEB versions in tilde form Published prerelease packages carry ~ in the dpkg control version (package_versions.sh maps the SemVer prerelease - to ~), so the legacy fallback list written with dots never matched 1.0.0~rc.x and upgrades away from those DEBs still left the service stopped (#8011). Fix the legacy glob to the tilde form and update the contract test, which had enshrined the dot form. Verified on Ubuntu 24.04 systemd containers: DEB upgrade 1.0.0~rc.5 -> 1.0.1 now keeps the service running; 1.0.0 -> 1.0.1 and 1.0.1 -> 1.0.2 marker path still pass. * docs(package): expand /etc/default/rustfs example template Document the commonly used RUSTFS_* settings as commented examples in the packaged conffile and point to docs.rustfs.com. --- .config/make/tests.mak | 1 + .github/workflows/audit.yml | 7 ++ .github/workflows/nightly-gnu.yml | 88 +++++----------- .github/workflows/package.yml | 88 +++++----------- scripts/README.md | 1 + scripts/release/package_service_scripts.sh | 112 ++++++++++++++++++++ scripts/test_package_service_scripts.sh | 116 +++++++++++++++++++++ 7 files changed, 289 insertions(+), 124 deletions(-) create mode 100755 scripts/release/package_service_scripts.sh create mode 100755 scripts/test_package_service_scripts.sh diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 56d5d79df..ef288b904 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -29,6 +29,7 @@ script-tests: ## Run shell script tests ./scripts/test_entrypoint_credentials.sh ./scripts/test_internode_grpc_ab_bench.sh ./scripts/test_object_batch_bench_enhanced.sh + ./scripts/test_package_service_scripts.sh ./scripts/test_hotpath_warp_ab_gate.sh ./scripts/test_hotpath_warp_abba.sh ./scripts/test_scanner_validation_harness.sh diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 485ebea28..72a50a904 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -24,7 +24,9 @@ on: - '.github/actions/**' - '.github/workflows/**' - 'scripts/release/create_or_update_release.sh' + - 'scripts/release/package_service_scripts.sh' - 'scripts/release/package_versions.sh' + - 'scripts/test_package_service_scripts.sh' - 'scripts/test_package_versions.sh' - 'scripts/security/check_performance_ab_workflow.sh' - 'scripts/security/check_preview_release_workflow.sh' @@ -40,7 +42,9 @@ on: - '.github/actions/**' - '.github/workflows/**' - 'scripts/release/create_or_update_release.sh' + - 'scripts/release/package_service_scripts.sh' - 'scripts/release/package_versions.sh' + - 'scripts/test_package_service_scripts.sh' - 'scripts/test_package_versions.sh' - 'scripts/security/check_performance_ab_workflow.sh' - 'scripts/security/check_preview_release_workflow.sh' @@ -158,6 +162,9 @@ jobs: - name: Check package version contract run: ./scripts/test_package_versions.sh + - name: Check package service lifecycle contract + run: ./scripts/test_package_service_scripts.sh + dependency-review: name: Dependency Review runs-on: ubuntu-latest diff --git a/.github/workflows/nightly-gnu.yml b/.github/workflows/nightly-gnu.yml index d283c31f0..bc7b73a33 100644 --- a/.github/workflows/nightly-gnu.yml +++ b/.github/workflows/nightly-gnu.yml @@ -117,10 +117,15 @@ jobs: cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF' # RustFS Environment Configuration - # See https://rustfs.com/docs/ for more information + # See https://docs.rustfs.com for more information + # RUSTFS_ACCESS_KEY= + # RUSTFS_SECRET_KEY= # RUSTFS_VOLUMES="" - # RUSTFS_ROOT_USER="" - # RUSTFS_ROOT_PASSWORD="" + # RUSTFS_ADDRESS=":9000" + # RUSTFS_CONSOLE_ADDRESS=":9001" + # RUSTFS_CONSOLE_ENABLE=true + # RUSTFS_OBS_LOGGER_LEVEL=error + # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/" ENVEOF # dpkg versions must start with a digit and cannot contain hyphens; @@ -146,37 +151,13 @@ jobs: /etc/default/rustfs CONFFILES - cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST' - #!/bin/bash - set -e - if ! getent passwd rustfs > /dev/null 2>&1; then - useradd -r -s /bin/false -d /opt/rustfs rustfs - fi - mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs - chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs - if [ -d /run/systemd/system ]; then - systemctl daemon-reload - fi - echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs" - POSTINST + ./scripts/release/package_service_scripts.sh after-install > "${PKG_DIR}/DEBIAN/postinst" chmod 755 "${PKG_DIR}/DEBIAN/postinst" - cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM' - #!/bin/bash - set -e - if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then - systemctl stop rustfs - fi - PRERM + ./scripts/release/package_service_scripts.sh before-remove > "${PKG_DIR}/DEBIAN/prerm" chmod 755 "${PKG_DIR}/DEBIAN/prerm" - cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM' - #!/bin/bash - set -e - if [ -d /run/systemd/system ]; then - systemctl daemon-reload - fi - POSTRM + ./scripts/release/package_service_scripts.sh after-remove > "${PKG_DIR}/DEBIAN/postrm" chmod 755 "${PKG_DIR}/DEBIAN/postrm" cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/" @@ -214,10 +195,15 @@ jobs: mkdir -p ./tmp-pkg/etc/default cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF' # RustFS Environment Configuration - # See https://rustfs.com/docs/ for more information + # See https://docs.rustfs.com for more information + # RUSTFS_ACCESS_KEY= + # RUSTFS_SECRET_KEY= # RUSTFS_VOLUMES="" - # RUSTFS_ROOT_USER="" - # RUSTFS_ROOT_PASSWORD="" + # RUSTFS_ADDRESS=":9000" + # RUSTFS_CONSOLE_ADDRESS=":9001" + # RUSTFS_CONSOLE_ENABLE=true + # RUSTFS_OBS_LOGGER_LEVEL=error + # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/" ENVEOF fpm -s dir -t rpm \ @@ -231,35 +217,13 @@ jobs: --description "High-performance distributed object storage" \ --url "https://rustfs.com" \ --license "Apache-2.0" \ - --after-install <(cat << 'POSTINST' - #!/bin/bash - set -e - if ! getent passwd rustfs > /dev/null 2>&1; then - useradd -r -s /bin/false -d /opt/rustfs rustfs - fi - mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs - chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs - if [ -d /run/systemd/system ]; then - systemctl daemon-reload - fi - POSTINST - ) \ - --before-remove <(cat << 'PRERM' - #!/bin/bash - set -e - if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then - systemctl stop rustfs - fi - PRERM - ) \ - --after-remove <(cat << 'POSTRM' - #!/bin/bash - set -e - if [ -d /run/systemd/system ]; then - systemctl daemon-reload - fi - POSTRM - ) \ + --before-install <(./scripts/release/package_service_scripts.sh rpm-before-install) \ + --before-upgrade <(./scripts/release/package_service_scripts.sh rpm-before-upgrade) \ + --after-install <(./scripts/release/package_service_scripts.sh after-install) \ + --after-upgrade <(./scripts/release/package_service_scripts.sh after-install) \ + --before-remove <(./scripts/release/package_service_scripts.sh before-remove) \ + --after-remove <(./scripts/release/package_service_scripts.sh after-remove) \ + --rpm-posttrans <(./scripts/release/package_service_scripts.sh rpm-posttrans) \ --config-files /etc/default/rustfs \ "rustfs-nightly-${DEB_DATE}/usr/bin/rustfs=/usr/bin/rustfs" \ ./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \ diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 9c0b4eb9d..faa46ed64 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -328,10 +328,15 @@ jobs: cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF' # RustFS Environment Configuration - # See https://rustfs.com/docs/ for more information + # See https://docs.rustfs.com for more information + # RUSTFS_ACCESS_KEY= + # RUSTFS_SECRET_KEY= # RUSTFS_VOLUMES="" - # RUSTFS_ROOT_USER="" - # RUSTFS_ROOT_PASSWORD="" + # RUSTFS_ADDRESS=":9000" + # RUSTFS_CONSOLE_ADDRESS=":9001" + # RUSTFS_CONSOLE_ENABLE=true + # RUSTFS_OBS_LOGGER_LEVEL=error + # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/" ENVEOF cat > "${PKG_DIR}/DEBIAN/control" << EOF @@ -354,37 +359,13 @@ jobs: /etc/default/rustfs CONFFILES - cat > "${PKG_DIR}/DEBIAN/postinst" << 'POSTINST' - #!/bin/bash - set -e - if ! getent passwd rustfs > /dev/null 2>&1; then - useradd -r -s /bin/false -d /opt/rustfs rustfs - fi - mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs - chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs - if [ -d /run/systemd/system ]; then - systemctl daemon-reload - fi - echo "RustFS installed. Configure /etc/default/rustfs then: systemctl start rustfs" - POSTINST + ./scripts/release/package_service_scripts.sh after-install > "${PKG_DIR}/DEBIAN/postinst" chmod 755 "${PKG_DIR}/DEBIAN/postinst" - cat > "${PKG_DIR}/DEBIAN/prerm" << 'PRERM' - #!/bin/bash - set -e - if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then - systemctl stop rustfs - fi - PRERM + ./scripts/release/package_service_scripts.sh before-remove > "${PKG_DIR}/DEBIAN/prerm" chmod 755 "${PKG_DIR}/DEBIAN/prerm" - cat > "${PKG_DIR}/DEBIAN/postrm" << 'POSTRM' - #!/bin/bash - set -e - if [ -d /run/systemd/system ]; then - systemctl daemon-reload - fi - POSTRM + ./scripts/release/package_service_scripts.sh after-remove > "${PKG_DIR}/DEBIAN/postrm" chmod 755 "${PKG_DIR}/DEBIAN/postrm" cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/" @@ -422,10 +403,15 @@ jobs: mkdir -p ./tmp-pkg/etc/default cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF' # RustFS Environment Configuration - # See https://rustfs.com/docs/ for more information + # See https://docs.rustfs.com for more information + # RUSTFS_ACCESS_KEY= + # RUSTFS_SECRET_KEY= # RUSTFS_VOLUMES="" - # RUSTFS_ROOT_USER="" - # RUSTFS_ROOT_PASSWORD="" + # RUSTFS_ADDRESS=":9000" + # RUSTFS_CONSOLE_ADDRESS=":9001" + # RUSTFS_CONSOLE_ENABLE=true + # RUSTFS_OBS_LOGGER_LEVEL=error + # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/" ENVEOF fpm -s dir -t rpm \ @@ -439,35 +425,13 @@ jobs: --description "High-performance distributed object storage" \ --url "https://rustfs.com" \ --license "Apache-2.0" \ - --after-install <(cat <<'POSTINST' - #!/bin/bash - set -e - if ! getent passwd rustfs > /dev/null 2>&1; then - useradd -r -s /bin/false -d /opt/rustfs rustfs - fi - mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs - chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs - if [ -d /run/systemd/system ]; then - systemctl daemon-reload - fi - POSTINST - ) \ - --before-remove <(cat <<'PRERM' - #!/bin/bash - set -e - if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then - systemctl stop rustfs - fi - PRERM - ) \ - --after-remove <(cat <<'POSTRM' - #!/bin/bash - set -e - if [ -d /run/systemd/system ]; then - systemctl daemon-reload - fi - POSTRM - ) \ + --before-install <(./scripts/release/package_service_scripts.sh rpm-before-install) \ + --before-upgrade <(./scripts/release/package_service_scripts.sh rpm-before-upgrade) \ + --after-install <(./scripts/release/package_service_scripts.sh after-install) \ + --after-upgrade <(./scripts/release/package_service_scripts.sh after-install) \ + --before-remove <(./scripts/release/package_service_scripts.sh before-remove) \ + --after-remove <(./scripts/release/package_service_scripts.sh after-remove) \ + --rpm-posttrans <(./scripts/release/package_service_scripts.sh rpm-posttrans) \ --config-files /etc/default/rustfs \ ./bin/rustfs=/usr/bin/rustfs \ ./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \ diff --git a/scripts/README.md b/scripts/README.md index a802401a9..922f9dfe7 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -90,6 +90,7 @@ their issue closes. | `test_build_rustfs_options.sh` | dev-tool | Shell test for rustfs build-option wiring | `make test` (script-tests) | | `test_entrypoint_credentials.sh` | dev-tool | Container entrypoint credential-handling test | `make test` (script-tests) | | `test_helm_chart_version.sh` | dev-tool | Test for `helm_chart_version.sh` | — | +| `test_package_service_scripts.sh` | ci-gate | Verifies DEB/RPM install, upgrade, removal, and service restart scriptlets | audit.yml `workflow-pin-report` | | `test_package_versions.sh` | ci-gate | Exact-output and fail-closed tests for DEB/RPM package version normalization | audit.yml `workflow-pin-report`; package.yml RPM build | | `windows-sftp-listener-smoke.sh` | dev-tool | Confirms `rustfs.exe --features sftp` binds an SFTP listener on Windows | — | diff --git a/scripts/release/package_service_scripts.sh b/scripts/release/package_service_scripts.sh new file mode 100755 index 000000000..7dcf2fb8f --- /dev/null +++ b/scripts/release/package_service_scripts.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +set -euo pipefail + +case ${1:-} in + after-install) + cat <<'SCRIPT' +#!/bin/sh +set -e +if ! getent passwd rustfs > /dev/null 2>&1; then + useradd -r -s /bin/false -d /opt/rustfs rustfs +fi +mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs +chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs +if [ -d /run/systemd/system ]; then + systemctl daemon-reload + if [ "${1:-}" = configure ]; then + marker=/run/rustfs-package-upgrade-was-active + if [ -z "${2:-}" ]; then + rm -f "$marker" + else + # Versions published before this fix had no upgrade-aware prerm: + # the old prerm stopped the service and left no marker, so the + # marker check below cannot see it. Fall back to restarting for + # upgrades from those versions. NOTE: these are dpkg control-file + # versions - package_versions.sh maps the SemVer prerelease "-" + # to "~", so 1.0.0-rc.5 was published as 1.0.0~rc.5. Drop this list once + # upgrades from <= 1.0.0 no longer need support. + case $2 in + 1.0.0|1.0.0~rc.[1-6]|1.0.0~rc.[1-6]~preview.*) legacy_upgrade=true ;; + *) legacy_upgrade=false ;; + esac + if [ -f "$marker" ] || [ "$legacy_upgrade" = true ]; then + rm -f "$marker" + if command -v deb-systemd-invoke > /dev/null 2>&1; then + deb-systemd-invoke restart rustfs.service + else + systemctl restart rustfs.service + fi + fi + fi + fi +fi +SCRIPT + ;; + before-remove) + cat <<'SCRIPT' +#!/bin/sh +set -e +case ${1:-} in + upgrade) + marker=/run/rustfs-package-upgrade-was-active + rm -f "$marker" + if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then + touch "$marker" + fi + exit 0 + ;; + 0|remove|deconfigure) ;; + *) exit 0 ;; +esac +if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then + systemctl stop rustfs +fi +SCRIPT + ;; + after-remove) + cat <<'SCRIPT' +#!/bin/sh +set -e +if [ -d /run/systemd/system ]; then + systemctl daemon-reload +fi +if [ "${1:-}" != upgrade ]; then + rm -f /run/rustfs-package-upgrade-was-active +fi +SCRIPT + ;; + rpm-before-install) + cat <<'SCRIPT' +#!/bin/sh +set -e +rm -f /run/rustfs-package-upgrade-was-active +SCRIPT + ;; + rpm-before-upgrade) + cat <<'SCRIPT' +#!/bin/sh +set -e +marker=/run/rustfs-package-upgrade-was-active +rm -f "$marker" +if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then + touch "$marker" +fi +SCRIPT + ;; + rpm-posttrans) + cat <<'SCRIPT' +#!/bin/sh +set -e +marker=/run/rustfs-package-upgrade-was-active +if [ -f "$marker" ]; then + rm -f "$marker" + systemctl restart rustfs +fi +SCRIPT + ;; + *) + echo "usage: $0 {after-install|before-remove|after-remove|rpm-before-install|rpm-before-upgrade|rpm-posttrans}" >&2 + exit 2 + ;; +esac diff --git a/scripts/test_package_service_scripts.sh b/scripts/test_package_service_scripts.sh new file mode 100755 index 000000000..3ae599271 --- /dev/null +++ b/scripts/test_package_service_scripts.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +generator="${repo_root}/scripts/release/package_service_scripts.sh" +test_tmp=$(mktemp -d) +trap 'rm -rf "$test_tmp"' EXIT + +mkdir -p "${test_tmp}/bin" +mkdir -p "${test_tmp}/run/systemd/system" +cat > "${test_tmp}/bin/getent" <<'SCRIPT' +#!/bin/sh +exit 0 +SCRIPT +cat > "${test_tmp}/bin/mkdir" <<'SCRIPT' +#!/bin/sh +exit 0 +SCRIPT +cat > "${test_tmp}/bin/chown" <<'SCRIPT' +#!/bin/sh +exit 0 +SCRIPT +cat > "${test_tmp}/bin/systemctl" <<'SCRIPT' +#!/bin/sh +printf 'systemctl %s\n' "$*" >> "$PACKAGE_SCRIPT_LOG" +case "$1" in + is-active) exit "${SYSTEMCTL_ACTIVE_EXIT:-0}" ;; +esac +SCRIPT +cat > "${test_tmp}/bin/deb-systemd-invoke" <<'SCRIPT' +#!/bin/sh +printf 'deb-systemd-invoke %s\n' "$*" >> "$PACKAGE_SCRIPT_LOG" +SCRIPT +chmod +x "${test_tmp}/bin/"* + +export PATH="${test_tmp}/bin:${PATH}" +export PACKAGE_SCRIPT_LOG="${test_tmp}/calls" + +run_script() { + local name=$1 + shift + "$generator" "$name" | + sed \ + -e "s#/run/systemd/system#${test_tmp}/run/systemd/system#g" \ + -e "s#/run/rustfs-package-upgrade-was-active#${test_tmp}/run/rustfs-package-upgrade-was-active#g" \ + > "${test_tmp}/${name}" + chmod +x "${test_tmp}/${name}" + "${test_tmp}/${name}" "$@" +} + +assert_log() { + local expected=$1 + printf '%s\n' "$expected" > "${test_tmp}/expected" + diff -u "${test_tmp}/expected" "$PACKAGE_SCRIPT_LOG" +} + +: > "$PACKAGE_SCRIPT_LOG" +run_script before-remove upgrade 1.0.0 +assert_log 'systemctl is-active --quiet rustfs' +marker="${test_tmp}/run/rustfs-package-upgrade-was-active" +[[ -f $marker ]] + +: > "$PACKAGE_SCRIPT_LOG" +run_script after-remove upgrade 1.0.1 +[[ -f $marker ]] +run_script after-install configure 1.0.1 +[[ ! -e $marker ]] +assert_log $'systemctl daemon-reload\nsystemctl daemon-reload\ndeb-systemd-invoke restart rustfs.service' + +: > "$PACKAGE_SCRIPT_LOG" +touch "$marker" +SYSTEMCTL_ACTIVE_EXIT=1 run_script before-remove upgrade 1.0.1 +[[ ! -e $marker ]] +run_script after-remove upgrade 1.0.2 +run_script after-install configure 1.0.1 +assert_log $'systemctl is-active --quiet rustfs\nsystemctl daemon-reload\nsystemctl daemon-reload' + +: > "$PACKAGE_SCRIPT_LOG" +run_script after-install configure 1.0.0~rc.5 +assert_log $'systemctl daemon-reload\ndeb-systemd-invoke restart rustfs.service' + +: > "$PACKAGE_SCRIPT_LOG" +run_script after-install configure 0~dev.9000.0123456789abcdef +assert_log 'systemctl daemon-reload' + +: > "$PACKAGE_SCRIPT_LOG" +run_script before-remove remove +assert_log $'systemctl is-active --quiet rustfs\nsystemctl stop rustfs' + +: > "$PACKAGE_SCRIPT_LOG" +touch "$marker" +run_script after-remove remove +[[ ! -e $marker ]] +assert_log 'systemctl daemon-reload' + +: > "$PACKAGE_SCRIPT_LOG" +touch "$marker" +run_script after-install configure +[[ ! -e $marker ]] +assert_log 'systemctl daemon-reload' + +: > "$PACKAGE_SCRIPT_LOG" +run_script rpm-before-upgrade +[[ -f $marker ]] +run_script rpm-posttrans +[[ ! -e $marker ]] +assert_log $'systemctl is-active --quiet rustfs\nsystemctl restart rustfs' + +: > "$PACKAGE_SCRIPT_LOG" +SYSTEMCTL_ACTIVE_EXIT=1 run_script rpm-before-upgrade +[[ ! -e $marker ]] +run_script rpm-posttrans +assert_log 'systemctl is-active --quiet rustfs' + +echo "PASS package service script contract" From cbf6b02905711328ebb8b52db5065468a54cda62 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Sun, 20 Sep 2026 11:27:56 +0800 Subject: [PATCH 10/30] ci(chain): fall back to auto-testing main HEAD when the pinned harness goes stale (#8026) The functional chain pins the test harness to the revision recorded in .config/functional-script-revision.txt, which is only refreshed by release syncs - it went stale on Sep 16 and the nightly chain has been replaying a Sep-17-era harness ever since. Last night that meant: the performance lane died on a dpkg conffile prompt (fix merged as #88 but never reached the chain), the FT lane died on 'unknown option: --results-file' (harness/workflow version mismatch), and the STS-105 signing fix (#94) sat untested. When the pinned revision's commit is older than 24 hours, fall back to auto-testing main HEAD so the chain always runs the current harness. Any staleness-probe failure keeps the pinned revision (fail-safe). The revision file stays in place as the audit trail and the release sync continues to manage it. --- scripts/resolve_functional_candidate.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/resolve_functional_candidate.py b/scripts/resolve_functional_candidate.py index 08f9eb9e4..7acdcd863 100644 --- a/scripts/resolve_functional_candidate.py +++ b/scripts/resolve_functional_candidate.py @@ -5,6 +5,7 @@ from __future__ import annotations import hashlib import io import json +from datetime import datetime, timezone import os from pathlib import Path import re @@ -112,6 +113,25 @@ def prepare(): candidate = resolve(run_id, attempt) revision = (ROOT / ".config/functional-script-revision.txt").read_text().strip() require(sha(revision), "private test script revision must be pinned") + # The revision file is refreshed only by release syncs and goes stale + # within days, so the nightly chain keeps replaying an old harness while + # fixes (conffile prompts, FT --results-file, tier retries, STS-105 + # signing) sit untested on auto-testing main. When the pinned revision is + # older than a day, fall back to auto-testing main HEAD. Any probe + # failure keeps the pinned revision - the chain still runs, just on the + # old harness. + try: + head = api("repos/rustfs/auto-testing/git/ref/heads/main")["object"]["sha"] + pinned_date = api(f"repos/rustfs/auto-testing/commits/{revision}")["commit"]["committer"]["date"] + pinned_at = datetime.fromisoformat(pinned_date.replace("Z", "+00:00")) + age_hours = (datetime.now(timezone.utc) - pinned_at).total_seconds() / 3600 + if age_hours > 24: + print(f"functional-script revision {revision[:8]} is {age_hours:.0f}h old; falling back to auto-testing main HEAD {head[:8]}") + revision = head + else: + print(f"functional-script revision {revision[:8]} is {age_hours:.0f}h old; keeping pin") + except Exception as error: + print(f"staleness probe failed ({error}); keeping pinned revision {revision[:8]}") chain = {"schema": 1, "run_id": int(os.environ["GITHUB_RUN_ID"]), "attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]), "workflow_sha": os.environ["GITHUB_SHA"], "testing_sha": revision, "candidate": candidate} encoded = json.dumps(chain, sort_keys=True, separators=(",", ":")) From 0632a64f914f76794e2ebdaa913a050149eb7d10 Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 20 Sep 2026 12:36:53 +0800 Subject: [PATCH 11/30] fix(ecstore): allow server-side copy at normal read quorum (#8029) --- crates/ecstore/src/erasure/coding/decode.rs | 154 +++++++++++++++++++- 1 file changed, 151 insertions(+), 3 deletions(-) diff --git a/crates/ecstore/src/erasure/coding/decode.rs b/crates/ecstore/src/erasure/coding/decode.rs index e97e79b43..233219c11 100644 --- a/crates/ecstore/src/erasure/coding/decode.rs +++ b/crates/ecstore/src/erasure/coding/decode.rs @@ -2573,6 +2573,11 @@ impl Erasure { .with_deferred_parity_reopeners(deferred_reopeners); let integrity = reader.integrity.clone(); + // Copy sources use demand-bound scheduling for backpressure, but retain + // the ordinary GET read quorum. Only the opt-in GET rollout requires a + // surplus source; available surplus shards and integrity proofs are + // still verified before any reconstructed copy bytes are emitted. + let require_surplus_source = reader.demand_bound_lockstep && matches!(decode_read_policy(), DecodeReadPolicy::Default); let start = offset / self.block_size; let end = end_offset.saturating_sub(1) / self.block_size; @@ -2667,7 +2672,6 @@ impl Erasure { // `shards` are borrowed again below. In the `Stop` case that // drop is what cancels the still-in-flight prefetch read. let (flow, next): (Option, Option) = { - let require_surplus_source = reader.demand_bound_lockstep; let read_fut = read_stripe_timed(&mut reader, stage_metrics_enabled); let emit_fut = self.emit_decoded_stripe( writer, @@ -2728,7 +2732,7 @@ impl Erasure { &mut written, &mut ret_err, stage_metrics_enabled, - reader.demand_bound_lockstep, + require_surplus_source, integrity.as_ref(), offset, ) @@ -2771,7 +2775,7 @@ impl Erasure { &mut written, &mut ret_err, stage_metrics_enabled, - reader.demand_bound_lockstep, + require_surplus_source, integrity.as_ref(), offset, ) @@ -3449,6 +3453,150 @@ mod tests { } } + async fn copy_source_bitrot_shards(erasure: &Erasure, payload: &[u8], corrupt_parity: bool) -> Vec> { + let mut writers: Vec<_> = (0..erasure.data_shards + erasure.parity_shards) + .map(|_| BitrotWriter::new(Cursor::new(Vec::new()), erasure.shard_size(), HashAlgorithm::HighwayHash256)) + .collect(); + for block in payload.chunks(erasure.block_size) { + let shards = erasure.encode_data(block).expect("copy source should encode"); + for (index, shard) in shards.iter().enumerate() { + let mut bytes = shard.to_vec(); + if corrupt_parity && index == erasure.data_shards { + // A stale source can have a valid per-shard bitrot checksum. + // Reframe the changed parity to exercise reconstruction verification. + bytes[0] ^= 0x80; + } + writers[index].write(&bytes).await.expect("copy source should frame"); + } + } + writers.into_iter().map(|writer| writer.into_inner().into_inner()).collect() + } + + #[tokio::test] + #[serial_test::serial] + async fn copy_source_reads_exact_quorum_across_stripes_and_ranges() { + const BLOCK_SIZE: usize = 768; + let payload: Vec<_> = (0..BLOCK_SIZE * 3 + 37) + .map(|index| (index.wrapping_mul(31) % 251) as u8) + .collect(); + + for uses_legacy in [false, true] { + let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy); + let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await; + for offline in [[0, 1, 2, 3], [0, 5, 12, 15]] { + for (offset, length) in [(0, payload.len()), (BLOCK_SIZE + 17, payload.len() - BLOCK_SIZE - 17)] { + let frame_offset = offset / BLOCK_SIZE * (erasure.shard_size() + HashAlgorithm::HighwayHash256.size()); + let positioned: Vec<_> = buffers.iter().map(|bytes| bytes[frame_offset..].to_vec()).collect(); + let (mut readers, handles) = + readers_with_deferred_parity(&positioned, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]); + for index in offline { + readers[index] = None; + } + let mut output = Vec::new(); + let (written, error) = crate::set_disk::with_get_object_read_policy( + crate::set_disk::GetObjectReadPolicy::CopySource, + erasure.decode_with_stripe_handles(&mut output, readers, offset, length, payload.len(), None, handles), + ) + .await; + + assert!(error.is_none(), "legacy={uses_legacy}, offline={offline:?}, offset={offset}: {error:?}"); + assert_eq!(written, length); + assert_eq!(output, payload[offset..offset + length]); + } + } + } + } + + #[tokio::test] + #[serial_test::serial] + async fn copy_source_recovers_exact_quorum_after_first_stripe() { + const BLOCK_SIZE: usize = 768; + let payload: Vec<_> = (0..BLOCK_SIZE * 3 + 37) + .map(|index| (index.wrapping_mul(31) % 251) as u8) + .collect(); + for uses_legacy in [false, true] { + let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy); + let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await; + let first_frame = erasure.shard_size() + HashAlgorithm::HighwayHash256.size(); + let (readers, handles) = readers_with_deferred_parity( + &buffers, + 12, + erasure.shard_size(), + &HashAlgorithm::HighwayHash256, + &[(0, first_frame), (1, first_frame), (2, first_frame), (3, first_frame)], + ); + let mut output = Vec::new(); + let (written, error) = crate::set_disk::with_get_object_read_policy( + crate::set_disk::GetObjectReadPolicy::CopySource, + erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles), + ) + .await; + + assert!(error.is_none(), "legacy={uses_legacy}: late shard loss should recover: {error:?}"); + assert_eq!(written, payload.len()); + assert_eq!(output, payload); + } + } + + #[tokio::test] + #[serial_test::serial] + async fn copy_source_below_read_quorum_fails_before_output() { + const BLOCK_SIZE: usize = 768; + let payload = vec![0x5a; BLOCK_SIZE * 2 + 37]; + for uses_legacy in [false, true] { + let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy); + let buffers = copy_source_bitrot_shards(&erasure, &payload, false).await; + let (mut readers, handles) = + readers_with_deferred_parity(&buffers, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]); + for reader in readers.iter_mut().take(5) { + *reader = None; + } + let mut output = Vec::new(); + let (written, error) = crate::set_disk::with_get_object_read_policy( + crate::set_disk::GetObjectReadPolicy::CopySource, + erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles), + ) + .await; + + assert_eq!(written, 0); + assert!(output.is_empty()); + assert_eq!( + error + .expect("eleven shards cannot reconstruct twelve data shards") + .to_string(), + Error::ErasureReadQuorum.to_string() + ); + } + } + + #[tokio::test] + #[serial_test::serial] + async fn copy_source_rejects_corrupt_surplus_parity_before_output() { + const BLOCK_SIZE: usize = 768; + let payload = vec![0x5a; BLOCK_SIZE * 2 + 37]; + for uses_legacy in [false, true] { + let erasure = Erasure::new_with_options(12, 4, BLOCK_SIZE, uses_legacy); + let buffers = copy_source_bitrot_shards(&erasure, &payload, true).await; + let (mut readers, handles) = + readers_with_deferred_parity(&buffers, 12, erasure.shard_size(), &HashAlgorithm::HighwayHash256, &[]); + for reader in readers.iter_mut().take(3) { + *reader = None; + } + let mut output = Vec::new(); + let (written, error) = crate::set_disk::with_get_object_read_policy( + crate::set_disk::GetObjectReadPolicy::CopySource, + erasure.decode_with_stripe_handles(&mut output, readers, 0, payload.len(), payload.len(), None, handles), + ) + .await; + + assert_eq!(written, 0); + assert!(output.is_empty()); + let error = error.expect("copy must reject inconsistent surplus parity despite valid bitrot framing"); + assert_eq!(error.kind(), ErrorKind::InvalidData); + assert!(error.to_string().contains("inconsistent read source shards")); + } + } + /// Guards the shard-buffer reuse in `ParallelReader`: a multi-stripe /// `Erasure::decode` that reconstructs missing data shards on every stripe /// must still return byte-exact output. Reconstructed and parity buffers are From d997b0750526ce819d387e859a8c7f06266cc959 Mon Sep 17 00:00:00 2001 From: Hauser Date: Sun, 20 Sep 2026 13:25:08 +0800 Subject: [PATCH 12/30] fix(get): skip futile resume for single-disk objects (#8031) * chore: pin s3s to upstream git revision Use the upstream s3s git repository at 50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6 for both s3s and s3s-sigv4. Allow the official s3s git source in cargo-deny so source checks continue to pass. Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(get): skip futile resume for single-disk objects Co-Authored-By: heihutu Co-Authored-By: zhi22915 --------- Co-authored-by: zhi22915 --- Cargo.lock | 43 +++++++++++++++----------- Cargo.toml | 4 +-- deny.toml | 2 +- rustfs/src/app/object/get.rs | 58 +++++++++++++++++++++++++++++------- 4 files changed, 77 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f28d4ce8..82d57995a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1984,9 +1984,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.6" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" dependencies = [ "find-msvc-tools", "jobserver", @@ -4410,9 +4410,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" [[package]] name = "findshlibs" @@ -11258,9 +11258,8 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87d22507a8ebd0ea64972c76dfedbc477418dc70afd98bfce928263894067d2f" +version = "0.16.0" +source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6" dependencies = [ "arc-swap", "arrayvec", @@ -11278,7 +11277,6 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "httparse", "hyper", "itoa", "jiff", @@ -11291,6 +11289,7 @@ dependencies = [ "pin-project-lite", "quick-xml", "regex", + "s3s-multipart", "s3s-rfc2047", "s3s-sigv2", "s3s-sigv4", @@ -11315,11 +11314,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "s3s-multipart" +version = "0.17.0-alpha.1" +source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6" +dependencies = [ + "arrayvec", + "bytes", + "futures-core", + "httparse", + "memchr", + "thiserror 2.0.20", +] + [[package]] name = "s3s-rfc2047" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d3e6b36be7cca3da377b4cabcd668ffe5b16c3b73a83f7a8f7b63ff6a178ee" +version = "0.16.0" +source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6" dependencies = [ "base64-simd", "thiserror 2.0.20", @@ -11327,9 +11338,8 @@ dependencies = [ [[package]] name = "s3s-sigv2" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96880b5ad8d639c9224155126741708aeb62ad3682e0476d2fd14290af808ef" +version = "0.16.0" +source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6" dependencies = [ "base64-simd", "hmac 0.13.0", @@ -11341,9 +11351,8 @@ dependencies = [ [[package]] name = "s3s-sigv4" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed605cbaaa1cf4d82e8ee15098fb3e42d49c92bb6215eb1359f63bfc0c6b020" +version = "0.16.0" +source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6" dependencies = [ "arrayvec", "base64-simd", diff --git a/Cargo.toml b/Cargo.toml index 848a273c2..29353f472 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -313,8 +313,8 @@ rustify = { version = "0.7", default-features = false } rustix = { version = "1.1.5" } rust-embed = { version = "8.12.0" } rustc-hash = { version = "2.1.3" } -s3s = { version = "0.16.1", features = ["minio"] } -s3s-sigv4 = { version = "0.16.1" } +s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6", features = ["minio"] } +s3s-sigv4 = { git = "https://github.com/s3s-project/s3s.git", rev = "50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6" } serial_test = "4.0.1" shadow-rs = { default-features = false, version = "2.0.0" } siphasher = "1.0.3" diff --git a/deny.toml b/deny.toml index cd645812d..ad657571b 100644 --- a/deny.toml +++ b/deny.toml @@ -39,7 +39,7 @@ allow-git = [ # Official s3s repository. Temporarily pinned to the merged generic REST # SigV4 payload-checksum fix until it is available in a crates.io release. # owner: marshawcoco review: 2026-10 - # "https://github.com/s3s-project/s3s.git", + "https://github.com/s3s-project/s3s.git", # RustFS fork carrying presigned expiry and constant-time authentication fixes. # owner: rustfs-maintainers review: 2026-10 # "https://github.com/rustfs/s3s.git", diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index 441626d38..e24b20c9e 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -499,6 +499,14 @@ fn get_object_stream_failure_reason(error_class: &'static str) -> &'static str { } } +/// A mid-stream reopen can recover only when another representation of the +/// committed object may exist. A single-disk, no-parity object has no alternate +/// shard after a bitrot/read-quorum failure, so retrying the same read only +/// repeats the failure and adds latency. +fn should_attach_get_object_resume(info: &ObjectInfo) -> bool { + info.parity_blocks > 0 || info.data_blocks > 1 || !info.transitioned_object.tier.is_empty() +} + fn record_get_object_reader_stream_failure( reason: &'static str, error_class: &'static str, @@ -4188,16 +4196,18 @@ impl DefaultObjectUsecase { part_number, lifecycle, |info| { - Some(get_object_resume_control(GetObjectResumeContext::new( - store, - &bucket, - &key, - opts, - &req.headers, - info, - resume_range_start, - resume_range_end, - ))) + should_attach_get_object_resume(info).then(|| { + get_object_resume_control(GetObjectResumeContext::new( + store, + &bucket, + &key, + opts, + &req.headers, + info, + resume_range_start, + resume_range_end, + )) + }) }, ) .await; @@ -8365,6 +8375,34 @@ mod tests { assert_eq!(reopen_count.load(Ordering::Relaxed), 0); } + #[test] + fn get_object_resume_only_attaches_for_recoverable_layouts() { + let single_disk = ObjectInfo { + data_blocks: 1, + parity_blocks: 0, + ..Default::default() + }; + assert!(!should_attach_get_object_resume(&single_disk)); + + let erasure = ObjectInfo { + data_blocks: 2, + parity_blocks: 1, + ..Default::default() + }; + assert!(should_attach_get_object_resume(&erasure)); + + let transitioned = ObjectInfo { + data_blocks: 1, + parity_blocks: 0, + transitioned_object: crate::storage_api::server::event::contract::lifecycle::TransitionedObject { + tier: "remote-tier".to_string(), + ..Default::default() + }, + ..Default::default() + }; + assert!(should_attach_get_object_resume(&transitioned)); + } + #[test] fn get_object_resume_identity_requires_same_version() { let version_id = Uuid::from_u128(0x1234); From 3738b32df14ab42021cd0ca2f282d67430c5ec87 Mon Sep 17 00:00:00 2001 From: Hauser Date: Sun, 20 Sep 2026 14:54:45 +0800 Subject: [PATCH 13/30] chore(deps): update flake.lock (#8033) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/aff8a0b' (2026-09-10) → 'github:NixOS/nixpkgs/a32edd7' (2026-09-17) • Updated input 'rust-overlay': 'github:oxalica/rust-overlay/228ecef' (2026-09-12) → 'github:oxalica/rust-overlay/26a71e6' (2026-09-19) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 12ca3e340..f3d119b57 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1789073787, - "narHash": "sha256-xfX/toC2QV707s06GbP4II/TxYF0fNQj7s5/LClNDKc=", + "lastModified": 1789632929, + "narHash": "sha256-RjR8AyvGlWuw16XRj7C1YDEw4E27ciuyPeB2nDMzgTU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "aff8a0b28396750446e5537a96461bc4facdb287", + "rev": "a32edd7654519351e48e80372a928df336394670", "type": "github" }, "original": { @@ -29,11 +29,11 @@ ] }, "locked": { - "lastModified": 1789196581, - "narHash": "sha256-yJr1Bt4fKkKIpPYbsKGqJ0VFdoDnURgRwuffmBQ2WzY=", + "lastModified": 1789802016, + "narHash": "sha256-QVFfArbYaQMGK4dFV+5RUL72Ci+hNHVhaF/R9qJfDhs=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "228ecefb6329d5a531b77b46b581a2f0c26ee056", + "rev": "26a71e661c47bd21a05d06fec749f3f7c75e9d12", "type": "github" }, "original": { From 9aeecad4a92bbd34f61f90cde8a2b16c82c1e4b9 Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 20 Sep 2026 23:52:47 +0800 Subject: [PATCH 14/30] docs(security): add encrypted ETag advisory lesson (#8040) --- .../references/advisory-patterns.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.agents/skills/security-advisory-lessons/references/advisory-patterns.md b/.agents/skills/security-advisory-lessons/references/advisory-patterns.md index ca7117347..339ff4f6d 100644 --- a/.agents/skills/security-advisory-lessons/references/advisory-patterns.md +++ b/.agents/skills/security-advisory-lessons/references/advisory-patterns.md @@ -101,6 +101,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect ### SSE and on-disk storage invariants - `GHSA-xrrf-67jm-3c2r`: SSE metadata reported encryption while reader composition bypassed `EncryptReader` and stored plaintext. Lesson: test actual bytes on disk and wrapper order, not only API metadata. +- `GHSA-wqmc-vjgv-jrpw`: SSE-C persisted and listed the plaintext MD5 as the ETag, allowing list-only users to confirm or brute-force low-entropy content and correlate equal plaintext across keys. Lesson: encrypted-object ETags must not expose deterministic plaintext fingerprints; protect them with per-object key material and keep listing, conditional, copy, and multipart semantics consistent. ### Object Lock and retention invariants @@ -126,7 +127,7 @@ rg -n "normalize_extract_entry_key|Snowball|auto-extract|PathBuf::join|canonical rg -n "DEFAULT_SECRET|DEFAULT_ACCESS|TEST_PRIVATE_KEY|rustfs rpc|RUSTFS_RPC_SECRET" rustfs crates rg -n "TONIC_RPC_PREFIX|verify_rpc_signature|check_auth|NodeServiceServer|x-rustfs-signature" rustfs crates rg -n "debug!|trace!|info!|error!|\\?resp|\\?merged_config|session_token|secret_key" rustfs crates -rg -n "HashReader|EncryptReader|SSE|server-side encryption|Access-Control-Allow-Credentials|Origin" rustfs crates +rg -n "HashReader|EncryptReader|try_resolve_etag|ETag|SSE|server-side encryption|Access-Control-Allow-Credentials|Origin" rustfs crates rg -n "ObjectLock|object_lock|retention|COMPLIANCE|GOVERNANCE|delete_prefix|lifecycle|scanner" rustfs crates rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates ``` @@ -145,5 +146,5 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates - IAM export fixes: assert exported archives omit plaintext user and service-account secrets unless the format deliberately encrypts or seals them. - RPC auth fixes: include captured metadata replay across two concrete methods, stale timestamps, wrong path, wrong method surrogate, wrong secret, and valid same-method calls. - Browser/CORS fixes: assert no credentials on reflected/default origins, correct behavior for explicit allowlists, and no same-origin script execution for previewed object content. -- SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together. +- SSE fixes: inspect stored bytes and verify API metadata, read-back behavior, and on-disk ciphertext together; for ETags, upload equal plaintext under different keys and assert list-only callers cannot derive or correlate the plaintext digest, including multipart objects. - Object Lock fixes: include unreadable metadata, fabricated metadata defaults, unparsable config, confirmed absent config, COMPLIANCE/GOVERNANCE retention, lifecycle expiry, scanner sweeps, and force-delete paths. From 42dd76d3b256fb50cb7f92dddca7ed1458d1dc8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=9E=E3=83=AB=E3=82=B3=E3=83=A1?= Date: Mon, 21 Sep 2026 08:04:47 +0900 Subject: [PATCH 15/30] ci(docker): sync the Docker Hub overview from README.md on release (#8034) Docker Hub's overview is a separate `full_description` field that `docker push` never touches, so it had drifted into an 8 KB snapshot of an old README.md that still linked to https://docs.rustfs.com/introduction.html (now 404). Add a `sync-dockerhub-description` job to docker.yml that runs after the images are pushed and republishes README.md from the same commit via peter-evans/dockerhub-description (pinned to v5.0.0). It reuses the existing DOCKERHUB_USERNAME / DOCKERHUB_TOKEN credentials, so no new secrets are needed. Relative links (docs/, CONTRIBUTING.md) are rewritten to github.com URLs so they resolve on Docker Hub. Docker Hub caps the field at 25,000 bytes and the action truncates to fit with only a warning; README.md is at 22,879 bytes today. Read the published overview back after the sync and fail the job if it hit the cap, so a truncated overview cannot be published silently. Fixes #7995 Co-authored-by: Claude Opus 5 Co-authored-by: Hauser --- .github/workflows/docker.yml | 48 ++++++++++++++++++++++++++++++++++++ docs/testing/ci-gates.md | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c74efa7e4..eb134bc78 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -542,6 +542,54 @@ jobs: if-no-files-found: error retention-days: 30 + # Docker Hub's overview is a separate `full_description` field that + # `docker push` never touches, so it drifted from README.md and kept + # linking to pages that no longer exist (#7995). Republish it from the + # commit the images were built from, only when images were pushed. + sync-dockerhub-description: + name: Sync Docker Hub Description + needs: [ build-check, build-docker ] + if: needs.build-check.outputs.should_build == 'true' && needs.build-check.outputs.should_push == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + ref: ${{ needs.build-check.outputs.source_ref }} + sparse-checkout: README.md + sparse-checkout-cone-mode: false + + - name: Sync Docker Hub description + uses: peter-evans/dockerhub-description@1b9a80c056b620d92cedb9d9b5a223409c68ddfa # v5.0.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: ${{ env.REGISTRY_DOCKERHUB }} + readme-filepath: ./README.md + # README.md links to docs/ and CONTRIBUTING.md by relative path; + # rewrite those to github.com URLs so they resolve on Docker Hub. + enable-url-completion: true + + # Docker Hub caps `full_description` at 25,000 bytes and the action + # truncates to fit with only a warning, which would publish a README + # cut off mid-sentence. README.md is already near the cap, so read the + # published overview back and fail if it hit the limit. + - name: Check published description was not truncated + env: + REPOSITORY: ${{ env.REGISTRY_DOCKERHUB }} + run: | + limit=25000 + published=$(curl -sSf "https://hub.docker.com/v2/repositories/${REPOSITORY}/" \ + | jq -j '.full_description' | wc -c | tr -d ' ') + echo "Published overview: ${published} bytes (Docker Hub limit: ${limit})" + # Truncation lands within one UTF-8 character of the cap. + if [ "$published" -ge $((limit - 4)) ]; then + echo "::error file=README.md::Docker Hub overview was truncated at ${limit} bytes; trim README.md before the next release." + exit 1 + fi + # Docker build summary docker-summary: name: Docker Build Summary diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 73a49ad5c..171faf10c 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -108,7 +108,7 @@ Post-merge and tag-driven; not a substitute for a PR gate. |---|---|---| | Push to `main`, weekly schedule, dispatch | `build.yml` `build-rustfs` (a development build on a main push restricts the matrix to the Linux targets) | build artifacts; no release publication | | Valid release or preview tag | `build.yml` `build-rustfs`, `create-release`, `upload-release-assets`, `publish-release` | draft release, checksummed assets, publish | -| Successful non-preview release-tag build (`workflow_run`) | `docker.yml` `build-docker`, `scan-docker-image` | multi-architecture images and vulnerability report | +| Successful non-preview release-tag build (`workflow_run`) | `docker.yml` `build-docker`, `scan-docker-image`, `sync-dockerhub-description` | multi-architecture images, vulnerability report, and the Docker Hub overview republished from `README.md` | | Successful release-tag build (`workflow_run`) | `package.yml` `package` | DEB/RPM packages and checksums uploaded to the release | | Successful non-preview release-tag build (`workflow_run`) | `helm-package.yml` `build-helm-package`, `publish-helm-package` | versioned chart and repository index | | Final tag's release published | `build.yml` `cleanup-preview-releases` | deletes every `-preview.` Release for that target; the tags are kept | From a7c875b48fcd67b941b2dca5155a499d5847e05a Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 21 Sep 2026 10:33:03 +0800 Subject: [PATCH 16/30] fix(heal): preserve remote corruption and reject incomplete deep scans (#8032) * fix(heal): reject truncated encoded shards * test(heal): cover truncated shards across integrity modes (#8035) Co-authored-by: zhi22915 * fix(heal): preserve remote corruption and reject incomplete deep scans * test(admin): tolerate config lock timeout while polling * fix(test): use admin storage error facade --------- Co-authored-by: Hauser Co-authored-by: zhi22915 Co-authored-by: overtrue --- crates/ecstore/src/disk/error.rs | 35 +- .../ecstore/src/io_support/shard_integrity.rs | 9 +- crates/ecstore/src/set_disk/mod.rs | 70 +++ crates/ecstore/src/set_disk/ops/heal.rs | 498 ++++++++++++++++-- crates/rio/src/http_reader.rs | 112 ++++ .../admin/handlers/audit_runtime_config.rs | 17 +- rustfs/src/storage/rpc/http_service.rs | 11 +- 7 files changed, 667 insertions(+), 85 deletions(-) diff --git a/crates/ecstore/src/disk/error.rs b/crates/ecstore/src/disk/error.rs index 881f93778..b987056a9 100644 --- a/crates/ecstore/src/disk/error.rs +++ b/crates/ecstore/src/disk/error.rs @@ -244,13 +244,16 @@ impl StdError for ConditionalFileNotCommittedError { } } -fn classify_internode_missing_error(error: &InternodeHttpError) -> Option { +fn classify_internode_disk_error(error: &InternodeHttpError) -> Option { if error.is_remote_file_not_found() { return Some(DiskError::FileNotFound); } if error.is_remote_volume_not_found() { return Some(DiskError::VolumeNotFound); } + if error.is_remote_file_corrupt() { + return Some(DiskError::FileCorrupt); + } None } @@ -526,13 +529,10 @@ fn io_error_chain_contains_kind(io_error: &std::io::Error, kind: std::io::ErrorK impl From for DiskError { fn from(e: std::io::Error) -> Self { - if let Some(error) = e.get_ref().and_then(|source| source.downcast_ref::()) { - if error.is_remote_file_not_found() { - return DiskError::FileNotFound; - } - if error.is_remote_volume_not_found() { - return DiskError::VolumeNotFound; - } + if let Some(error) = e.get_ref().and_then(|source| source.downcast_ref::()) + && let Some(classified) = classify_internode_disk_error(error) + { + return classified; } let e = match e.downcast::() { Ok(terminal_error) => { @@ -541,7 +541,7 @@ impl From for DiskError { && let Some(internode_error) = io_error .get_ref() .and_then(|source| source.downcast_ref::()) - && let Some(classified) = classify_internode_missing_error(internode_error) + && let Some(classified) = classify_internode_disk_error(internode_error) { return classified; } @@ -963,6 +963,7 @@ mod tests { for (remote_error, expected) in [ (rustfs_rio::new_test_remote_file_not_found_http_io_error(), DiskError::FileNotFound), (rustfs_rio::new_test_remote_volume_not_found_http_io_error(), DiskError::VolumeNotFound), + (rustfs_rio::new_test_remote_file_corrupt_http_io_error(), DiskError::FileCorrupt), ] { let wrapped = terminal_read_error_to_io(DiskError::Io(remote_error)); assert_eq!(DiskError::from(wrapped), expected); @@ -1533,25 +1534,23 @@ mod tests { } #[test] - fn test_internode_missing_errors_preserve_disk_error_types() { + fn test_internode_disk_errors_preserve_disk_error_types() { let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error()); let volume_missing = DiskError::from(rustfs_rio::new_test_remote_volume_not_found_http_io_error()); + let file_corrupt = DiskError::from(rustfs_rio::new_test_remote_file_corrupt_http_io_error()); let unmarked_server_error = DiskError::from(rustfs_rio::new_test_internode_http_io_error( rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR), )); assert_eq!(file_missing, DiskError::FileNotFound); assert_eq!(volume_missing, DiskError::VolumeNotFound); + assert_eq!(file_corrupt, DiskError::FileCorrupt); assert!(matches!(unmarked_server_error, DiskError::Io(_))); - for missing in [file_missing, volume_missing] { - assert_eq!(missing.clone(), missing); + for error in [file_missing, volume_missing, file_corrupt] { + assert_eq!(error.clone(), error); assert_eq!( - crate::disk::error_reduce::reduce_write_quorum_errs( - &[Some(missing.clone()), Some(missing.clone()), None], - &[], - 2 - ), - Some(missing) + crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(error.clone()), Some(error.clone()), None], &[], 2), + Some(error) ); } } diff --git a/crates/ecstore/src/io_support/shard_integrity.rs b/crates/ecstore/src/io_support/shard_integrity.rs index 9c8f9c417..deb41f382 100644 --- a/crates/ecstore/src/io_support/shard_integrity.rs +++ b/crates/ecstore/src/io_support/shard_integrity.rs @@ -587,10 +587,8 @@ pub(crate) async fn verify_deep_parts( return Err(DiskError::FileCorrupt); } let part_status = statuses.get_mut(&part_index).ok_or(DiskError::FileCorrupt)?; - // Deep verification reads the complete encoded shard. `shard_file_offset` - // is a range-read helper and can stop before the final encoded byte for - // a partial data block, which would let a one-byte tail truncation pass - // as healthy. Use the physical shard length for the integrity proof. + // Deep verification must consume every encoded stripe, including a + // partial final stripe, before certifying the shard's integrity. let length = usize::try_from(erasure.shard_file_size(part.size as i64)).map_err(|_| DiskError::FileCorrupt)?; let mut readers = Vec::with_capacity(disks.len()); for (index, disk) in disks.iter().enumerate() { @@ -635,6 +633,9 @@ pub(crate) async fn verify_deep_parts( part_status[index] = CHECK_PART_FILE_CORRUPT; } Err(error) => { + if !matches!(error, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::FileCorrupt) { + return Err(error); + } readers.push(None); part_status[index] = conv_part_err_to_int(&Some(error)); } diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index b9e766c2f..68cf4d582 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -6473,6 +6473,28 @@ async fn verify_inline_part_bitrot(meta: &FileInfo) -> disk::error::Result<()> { .map_err(|_| DiskError::FileCorrupt) } +fn validate_deep_scan_results(results: &[usize], expected_parts: usize) -> disk::error::Result<()> { + if results.len() != expected_parts { + return Err(DiskError::other(format!( + "incomplete deep scan result: expected {expected_parts} parts, received {}", + results.len() + ))); + } + for (part, status) in results.iter().enumerate() { + match *status { + CHECK_PART_SUCCESS | CHECK_PART_FILE_NOT_FOUND | CHECK_PART_FILE_CORRUPT => {} + CHECK_PART_DISK_NOT_FOUND => return Err(DiskError::DiskNotFound), + crate::disk::CHECK_PART_VOLUME_NOT_FOUND => return Err(DiskError::VolumeNotFound), + _ => { + return Err(DiskError::other(format!( + "incomplete deep scan result: part {part} has unverified status {status}" + ))); + } + } + } + Ok(()) +} + /// disks_with_all_partsv2 is a corrected version based on Go implementation. /// It sets partsMetadata and onlineDisks when xl.meta is inexistant/corrupted or outdated. /// It also checks if the status of each part (corrupted, missing, ok) in each drive. @@ -6693,9 +6715,13 @@ async fn disks_with_all_parts( // it needs healing too. match disk.verify_file(bucket, object, meta).await { Ok(v) => { + validate_deep_scan_results(&v.results, latest_meta.parts.len())?; verify_resp = v; } Err(err) => { + if !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::FileCorrupt) { + return Err(err); + } debug!( event = EVENT_SET_DISK_HEAL, component = LOG_COMPONENT_ECSTORE, @@ -11109,6 +11135,50 @@ mod tests { assert_eq!(conv_part_err_to_int(&Some(other_err)), CHECK_PART_UNKNOWN); // Other errors should return UNKNOWN, not SUCCESS } + #[test] + fn deep_scan_results_accept_only_complete_verified_or_repairable_parts() { + for (results, expected_parts) in [ + (vec![], 0), + (vec![CHECK_PART_SUCCESS], 1), + (vec![CHECK_PART_FILE_NOT_FOUND], 1), + (vec![CHECK_PART_FILE_CORRUPT], 1), + (vec![CHECK_PART_SUCCESS, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_FILE_CORRUPT], 3), + ] { + validate_deep_scan_results(&results, expected_parts) + .expect("complete results must allow healthy or repairable parts"); + } + } + + #[test] + fn deep_scan_results_reject_unknown_and_incomplete_observations() { + for (results, expected_parts) in [ + (vec![CHECK_PART_UNKNOWN], 1), + (vec![usize::MAX], 1), + (vec![CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN], 2), + (vec![], 1), + (vec![CHECK_PART_SUCCESS], 2), + (vec![CHECK_PART_SUCCESS, CHECK_PART_SUCCESS], 1), + (vec![CHECK_PART_SUCCESS], 0), + ] { + let error = + validate_deep_scan_results(&results, expected_parts).expect_err("unverified parts must not become healthy"); + assert!(matches!(error, DiskError::Io(_)), "an incomplete observation is not proven corruption"); + assert!(error.to_string().contains("incomplete deep scan result")); + } + } + + #[test] + fn deep_scan_results_preserve_disk_and_volume_failures() { + for (status, expected_error) in [ + (CHECK_PART_DISK_NOT_FOUND, DiskError::DiskNotFound), + (CHECK_PART_VOLUME_NOT_FOUND, DiskError::VolumeNotFound), + ] { + let error = validate_deep_scan_results(&[CHECK_PART_SUCCESS, status], 2) + .expect_err("an unavailable part cannot certify a healthy disk"); + assert_eq!(error, expected_error); + } + } + #[test] fn test_has_part_err() { // Test checking for part errors diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 1f57c1142..906ff1eb3 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -4378,63 +4378,243 @@ mod heal_result_report_tests { ); } - #[tokio::test] - async fn deep_heal_rebuilds_h2_near_tail_truncated_part() { - let (temp_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(16, 0, 4).await; - let bucket = "deep-heal-h2-near-tail-truncation"; - let object = "object.bin"; - for disk in &disks { - disk.make_volume(bucket).await.expect("bucket volume should be created"); + #[derive(Debug)] + struct FailedIntegrityReadTransport { + error: DiskError, + reads: std::sync::atomic::AtomicUsize, + } + + #[async_trait::async_trait] + impl crate::cluster::rpc::internode_data_transport::InternodeDataTransport for FailedIntegrityReadTransport { + async fn open_read( + &self, + _request: crate::cluster::rpc::internode_data_transport::ReadStreamRequest, + ) -> crate::disk::error::Result { + self.reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let error = match &self.error { + DiskError::FileCorrupt => DiskError::from(rustfs_rio::new_test_remote_file_corrupt_http_io_error()), + DiskError::FileNotFound => DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error()), + DiskError::VolumeNotFound => DiskError::from(rustfs_rio::new_test_remote_volume_not_found_http_io_error()), + error => error.clone(), + }; + Err(error) } - let expected_payload = vec![0x6d; 5 * 1024 * 1024 + 123]; + async fn open_write( + &self, + _request: crate::cluster::rpc::internode_data_transport::WriteStreamRequest, + ) -> crate::disk::error::Result { + panic!("an integrity scan must not write remote data"); + } + + async fn open_walk_dir( + &self, + _request: crate::cluster::rpc::internode_data_transport::WalkDirStreamRequest, + ) -> crate::disk::error::Result { + panic!("an object integrity scan must not walk remote directories"); + } + + fn name(&self) -> &'static str { + "failed-integrity-read" + } + + fn capabilities(&self) -> crate::cluster::rpc::internode_data_transport::InternodeDataTransportCapabilities { + crate::cluster::rpc::internode_data_transport::InternodeDataTransportCapabilities::tcp_http() + } + } + + async fn assert_remote_integrity_read_failure(expected_error: DiskError) { + use crate::disk::{CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN}; + use crate::object_api::ShardIntegrityWriteMode; + + let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(4, 0, 2).await; + let bucket = "remote-integrity-read-failure"; + let object = "object.bin"; + for disk in &disks { + disk.make_volume(bucket).await.expect("create fixture bucket"); + } set.put_object( bucket, object, - &mut PutObjReader::from_vec(expected_payload.clone()), + &mut PutObjReader::from_vec(vec![0x6d; 1024 * 1024 + 123]), &ObjectOptions { + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected), no_lock: true, ..Default::default() }, ) .await - .expect("source object should be written before shard truncation"); - let source = disks[2] - .read_version("", bucket, object, "", &ReadOptions::default()) - .await - .expect("source metadata should be readable"); - let data_dir = source.data_dir.expect("non-inline source should have a data directory"); - let truncated_part = temp_dirs[1] - .path() - .join(bucket) - .join(object) - .join(data_dir.to_string()) - .join("part.1"); - let original_len = tokio::fs::metadata(&truncated_part) - .await - .expect("target shard should exist") - .len(); - assert!(original_len > 1, "test shard must be large enough to truncate"); - let file = tokio::fs::OpenOptions::new() - .write(true) - .open(&truncated_part) - .await - .expect("target shard should be writable"); - file.set_len(original_len - 1) - .await - .expect("target shard should be truncated"); + .expect("write protected external fixture"); - let mut reader = set - .get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default()) - .await - .expect("GET should remain readable after a one-byte shard truncation"); - let mut read_back = Vec::new(); - tokio::io::copy(&mut reader, &mut read_back) - .await - .expect("GET should reconstruct the truncated shard through EC"); - assert_eq!(read_back, expected_payload, "EC GET must preserve the object bytes"); + let mut files = Vec::new(); + let mut snapshots = Vec::new(); + for (dir, disk) in dirs.iter().zip(&disks) { + let file = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("read fixture metadata"); + assert!(file.data.is_none(), "the remote test must open an external shard"); + let object_dir = dir.path().join(bucket).join(object); + let part_path = object_dir + .join(file.data_dir.expect("external shard directory").to_string()) + .join("part.1"); + for path in [part_path, object_dir.join("xl.meta")] { + let bytes = tokio::fs::read(&path).await.expect("snapshot fixture before scan"); + snapshots.push((path, bytes)); + } + files.push(file); + } + let transport = Arc::new(FailedIntegrityReadTransport { + error: expected_error.clone(), + reads: std::sync::atomic::AtomicUsize::new(0), + }); + let endpoint = Endpoint { + url: url::Url::parse("http://remote-integrity.invalid:9000/data/disk0").expect("fixture endpoint"), + is_local: false, + pool_idx: 0, + set_idx: 0, + disk_idx: 0, + }; + let remote = crate::cluster::rpc::RemoteDisk::new( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + transport.clone(), + ) + .await + .expect("construct remote disk with failing data transport"); + let mut verification_disks: Vec<_> = disks.iter().cloned().map(Some).collect(); + verification_disks[0] = Some(Arc::new(crate::disk::Disk::Remote(Box::new(remote)))); + let mut statuses = [(0, vec![CHECK_PART_UNKNOWN; disks.len()])].into_iter().collect(); + let result = crate::io_support::shard_integrity::verify_deep_parts( + &files, + &verification_disks, + &files[0], + bucket, + object, + &mut statuses, + ) + .await; + assert!(transport.reads.load(std::sync::atomic::Ordering::Relaxed) > 0); + if matches!(expected_error, DiskError::FileCorrupt | DiskError::FileNotFound) { + result.expect("typed remote payload damage must identify a repairable shard"); + let expected_status = if expected_error == DiskError::FileCorrupt { + CHECK_PART_FILE_CORRUPT + } else { + CHECK_PART_FILE_NOT_FOUND + }; + assert_eq!(statuses[&0][0], expected_status); + assert!(statuses[&0][1..].iter().all(|status| *status == CHECK_PART_SUCCESS)); + let (heal, metadata, reason) = super::should_heal_object_on_disk(&None, &[statuses[&0][0]], &files[0], &files[0]); + assert!(heal, "remote payload damage must schedule repair: {expected_error:?}"); + assert!(!metadata); + let expected_reason = if expected_error == DiskError::FileCorrupt { + DiskError::FileCorrupt + } else { + DiskError::PartMissingOrCorrupt + }; + assert_eq!(reason, Some(expected_reason)); + } else { + let error = result.expect_err("unavailable payload storage must abort health verification"); + if expected_error.is_internode_http_status(500) { + assert!(error.is_internode_http_status(500), "preserve the original transport error: {error:?}"); + } else { + assert_eq!(error, expected_error, "preserve the original storage failure"); + } + assert_eq!(statuses[&0][0], CHECK_PART_UNKNOWN, "failed verification cannot certify the shard"); + } + for (path, before) in snapshots { + assert_eq!(tokio::fs::read(path).await.expect("read fixture after scan"), before); + } + } - let (result, error) = set + #[tokio::test] + async fn deep_scan_remote_payload_damage_schedules_repair() { + for error in [DiskError::FileCorrupt, DiskError::FileNotFound] { + assert_remote_integrity_read_failure(error).await; + } + } + + #[tokio::test] + async fn deep_scan_remote_storage_failure_cannot_certify_healthy() { + for error in [ + DiskError::from(rustfs_rio::new_test_internode_http_io_error( + rustfs_rio::InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR), + )), + DiskError::DiskNotFound, + DiskError::VolumeNotFound, + DiskError::FileAccessDenied, + ] { + assert_remote_integrity_read_failure(error).await; + } + } + + #[cfg(unix)] + #[tokio::test] + async fn deep_heal_legacy_verification_error_cannot_certify_healthy() { + use crate::object_api::ShardIntegrityWriteMode; + + let (dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "legacy-deep-scan-verification-error"; + let object = "object.bin"; + for disk in &disks { + disk.make_volume(bucket).await.expect("create legacy fixture bucket"); + } + set.put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![0x6d; 1024 * 1024 + 123]), + &ObjectOptions { + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy), + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("write external legacy fixture"); + + let mut files = Vec::new(); + let mut part_paths = Vec::new(); + let mut preserved = Vec::new(); + for (index, (dir, disk)) in dirs.iter().zip(&disks).enumerate() { + let file = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("read legacy fixture metadata"); + assert!(file.data.is_none(), "legacy verification must open the physical shard"); + assert!(file.parts.iter().all(|part| part.integrity.is_none())); + let object_dir = dir.path().join(bucket).join(object); + let part = object_dir + .join(file.data_dir.expect("external legacy shard directory").to_string()) + .join("part.1"); + let metadata = object_dir.join("xl.meta"); + let bytes = tokio::fs::read(&metadata).await.expect("snapshot legacy metadata"); + preserved.push((metadata, bytes)); + if index != 0 { + let bytes = tokio::fs::read(&part).await.expect("snapshot healthy legacy shard"); + preserved.push((part.clone(), bytes)); + } + part_paths.push(part); + files.push(file); + } + let broken_part = &part_paths[0]; + tokio::fs::remove_file(broken_part) + .await + .expect("remove fixture shard before symlink fault"); + std::os::unix::fs::symlink("part.1", broken_part).expect("create self-referential shard symlink"); + let open_error = tokio::fs::File::open(broken_part) + .await + .expect_err("self-referential symlink must fail to open"); + assert_eq!(open_error.raw_os_error(), Some(libc::ELOOP)); + + let verification_error = disks[0] + .verify_file(bucket, object, &files[0]) + .await + .expect_err("legacy disk verifier must reject the symlink path"); + assert_eq!(verification_error, DiskError::InvalidPath); + let error = set .heal_object( bucket, object, @@ -4446,19 +4626,229 @@ mod heal_result_report_tests { }, ) .await - .expect("deep heal should finish after a one-byte H2 tail truncation"); - - assert!(error.is_none(), "deep heal should recover the H2 truncated shard: {error:?}"); - assert_eq!(result.drives_healed(), Some(1)); - assert_eq!(result.before.drives[1].state, DriveState::Corrupt.to_string()); + .expect_err("an unverified legacy part must fail the complete heal operation"); + assert_eq!(error, DiskError::InvalidPath, "complete heal must preserve the verifier's path error"); assert_eq!( - tokio::fs::metadata(&truncated_part) + tokio::fs::read_link(broken_part) .await - .expect("repaired H2 shard should exist") - .len(), - original_len, - "deep heal must restore the complete H2 shard length" + .expect("failed heal must preserve the symlink fault"), + std::path::Path::new("part.1") ); + for (path, before) in preserved { + assert_eq!(tokio::fs::read(path).await.expect("read unchanged legacy fixture"), before); + } + } + + async fn assert_h2_truncated_shards( + mode: crate::object_api::ShardIntegrityWriteMode, + coding_indexes: &[usize], + retained_len: fn(usize, usize) -> usize, + ) { + use crate::object_api::ShardIntegrityWriteMode; + + let (temp_dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(16, 0, 4).await; + let bucket = "deep-heal-h2-truncation"; + let object = "object.bin"; + for disk in &disks { + disk.make_volume(bucket).await.expect("create bucket volume"); + } + let expected_payload: Vec<_> = (0..5 * 1024 * 1024 + 123) + .map(|index| u8::try_from((index / 4096 + index) % 251).expect("payload byte fits")) + .collect(); + set.put_object( + bucket, + object, + &mut PutObjReader::from_vec(expected_payload.clone()), + &ObjectOptions { + shard_integrity_write_mode: Some(mode), + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("commit all source shards"); + + let mut files = Vec::new(); + let mut paths = Vec::new(); + let mut original_shards = Vec::new(); + let mut original_metadata = Vec::new(); + let mut damaged = Vec::new(); + for (slot, disk) in disks.iter().enumerate() { + let source = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("read source metadata"); + assert_eq!((source.erasure.data_blocks, source.erasure.parity_blocks), (12, 4)); + assert_eq!(source.parts[0].integrity.is_some(), mode == ShardIntegrityWriteMode::Protected); + let directory = temp_dirs[slot].path().join(bucket).join(object); + let path = directory + .join(source.data_dir.expect("external shard data directory").to_string()) + .join("part.1"); + let original = tokio::fs::read(&path).await.expect("read original shard"); + original_metadata.push(tokio::fs::read(directory.join("xl.meta")).await.expect("read xl.meta")); + if coding_indexes.contains(&source.erasure.index) { + let erasure = crate::erasure::coding::Erasure::try_new_with_options( + 12, + 4, + source.erasure.block_size, + source.uses_legacy_checksum, + ) + .expect("valid erasure layout"); + let stripe_size = erasure.shard_size() + source.erasure.get_checksum_info(1).algorithm.size(); + let keep = retained_len(original.len(), stripe_size); + assert!(keep < original.len(), "fixture must truncate the shard"); + tokio::fs::OpenOptions::new() + .write(true) + .open(&path) + .await + .expect("open target shard") + .set_len(u64::try_from(keep).expect("shard length fits")) + .await + .expect("truncate target shard"); + damaged.push((slot, keep)); + } + paths.push(path); + original_shards.push(original); + files.push(source); + } + assert_eq!(damaged.len(), coding_indexes.len()); + if mode == ShardIntegrityWriteMode::Protected { + let mut statuses = [(0, vec![crate::disk::CHECK_PART_UNKNOWN; disks.len()])] + .into_iter() + .collect(); + let online_disks: Vec<_> = disks.iter().cloned().map(Some).collect(); + crate::io_support::shard_integrity::verify_deep_parts( + &files, + &online_disks, + &files[0], + bucket, + object, + &mut statuses, + ) + .await + .expect("verify protected shards directly"); + for (slot, status) in statuses[&0].iter().enumerate() { + let expected = if damaged.iter().any(|(index, _)| *index == slot) { + crate::disk::CHECK_PART_FILE_CORRUPT + } else { + crate::disk::CHECK_PART_SUCCESS + }; + assert_eq!(*status, expected, "protected shard {slot}"); + } + } + + let opts = HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }; + let (dry_run, error) = set + .heal_object(bucket, object, "", &HealOpts { dry_run: true, ..opts }) + .await + .expect("dry-run should report corruption"); + assert!(error.is_none(), "dry-run error: {error:?}"); + assert_eq!(dry_run.drives_healed(), Some(0)); + for (slot, keep) in &damaged { + assert_eq!(dry_run.after.drives[*slot].state, DriveState::Corrupt.to_string()); + assert_eq!( + tokio::fs::read(&paths[*slot]).await.expect("dry-run preserved shard"), + original_shards[*slot][..*keep] + ); + } + + let recoverable = damaged.len() <= 4; + if recoverable { + let mut reader = set + .get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default()) + .await + .expect("degraded GET should remain readable"); + let mut read_back = Vec::new(); + tokio::io::copy(&mut reader, &mut read_back) + .await + .expect("read complete degraded GET"); + assert_eq!(read_back, expected_payload); + } + let (result, error) = set + .heal_object(bucket, object, "", &opts) + .await + .expect("heal should report outcome"); + if recoverable { + assert!(error.is_none(), "heal error: {error:?}"); + assert_eq!(result.drives_healed(), Some(damaged.len())); + for (slot, _) in &damaged { + assert_eq!(result.before.drives[*slot].state, DriveState::Corrupt.to_string()); + } + } else { + assert_eq!(error, Some(DiskError::ErasureReadQuorum)); + assert_eq!(result.drives_healed(), Some(0)); + } + for (slot, path) in paths.iter().enumerate() { + let expected = if !recoverable && let Some((_, keep)) = damaged.iter().find(|(index, _)| *index == slot) { + &original_shards[slot][..*keep] + } else { + original_shards[slot].as_slice() + }; + assert_eq!( + tokio::fs::read(path).await.expect("read shard after heal"), + expected, + "physical shard {slot}" + ); + if !recoverable { + let metadata_path = temp_dirs[slot].path().join(bucket).join(object).join("xl.meta"); + assert_eq!(tokio::fs::read(metadata_path).await.expect("preserved metadata"), original_metadata[slot]); + } + } + if recoverable { + let mut reader = set + .get_object_reader(bucket, object, None, Default::default(), &ObjectOptions::default()) + .await + .expect("GET after repair"); + let mut read_back = Vec::new(); + tokio::io::copy(&mut reader, &mut read_back) + .await + .expect("read repaired object"); + assert_eq!(read_back, expected_payload); + let (healthy, error) = set.heal_object(bucket, object, "", &opts).await.expect("second deep scan"); + assert!(error.is_none()); + assert_eq!(healthy.drives_healed(), Some(0), "repair must converge"); + assert_eq!(healthy.integrity_verified, mode == ShardIntegrityWriteMode::Protected); + } + } + + #[tokio::test] + async fn deep_heal_rebuilds_h2_near_tail_truncated_part() { + use crate::object_api::ShardIntegrityWriteMode::{Legacy, Protected}; + for mode in [Legacy, Protected] { + for coding_index in [1, 13] { + assert_h2_truncated_shards(mode, &[coding_index], |length, _| length - 1).await; + } + } + } + + #[tokio::test] + async fn deep_heal_rebuilds_h2_half_truncated_part() { + use crate::object_api::ShardIntegrityWriteMode::{Legacy, Protected}; + for mode in [Legacy, Protected] { + for coding_index in [1, 13] { + assert_h2_truncated_shards(mode, &[coding_index], |length, _| length / 2).await; + } + } + } + + #[tokio::test] + async fn deep_heal_rebuilds_h2_stripe_boundary_truncated_parts_at_read_quorum() { + use crate::object_api::ShardIntegrityWriteMode::{Legacy, Protected}; + for mode in [Legacy, Protected] { + assert_h2_truncated_shards(mode, &[1, 2, 13, 16], |_, stripe| stripe).await; + } + } + + #[tokio::test] + async fn deep_heal_rejects_h2_truncated_parts_below_read_quorum() { + use crate::object_api::ShardIntegrityWriteMode::{Legacy, Protected}; + for mode in [Legacy, Protected] { + assert_h2_truncated_shards(mode, &[1, 2, 3, 13, 16], |length, _| length / 2).await; + } } #[tokio::test] diff --git a/crates/rio/src/http_reader.rs b/crates/rio/src/http_reader.rs index 17c5fa2a9..c814aa114 100644 --- a/crates/rio/src/http_reader.rs +++ b/crates/rio/src/http_reader.rs @@ -57,6 +57,7 @@ const EXCESSIVE_EMPTY_CHUNKS_ERROR: &str = "HTTP body returned too many empty ch pub const INTERNODE_DISK_ERROR_HEADER: &str = "x-rustfs-disk-error"; pub const INTERNODE_FILE_NOT_FOUND: &str = "file-not-found"; pub const INTERNODE_VOLUME_NOT_FOUND: &str = "volume-not-found"; +pub const INTERNODE_FILE_CORRUPT: &str = "file-corrupt"; #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum InternodeHttpErrorKind { @@ -185,6 +186,7 @@ pub struct InternodeHttpError { enum RemoteDiskErrorKind { FileNotFound, VolumeNotFound, + FileCorrupt, } impl std::fmt::Debug for InternodeHttpError { @@ -215,6 +217,10 @@ impl InternodeHttpError { self.remote_disk_error == Some(RemoteDiskErrorKind::VolumeNotFound) } + pub fn is_remote_file_corrupt(&self) -> bool { + self.remote_disk_error == Some(RemoteDiskErrorKind::FileCorrupt) + } + fn new(kind: InternodeHttpErrorKind, context: InternodeHttpRequestContext) -> Self { Self { kind, @@ -328,6 +334,20 @@ pub fn new_test_remote_volume_not_found_http_io_error() -> io::Error { .into_io_error() } +#[doc(hidden)] +pub fn new_test_remote_file_corrupt_http_io_error() -> io::Error { + InternodeHttpError::with_remote_disk_error( + InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR), + InternodeHttpRequestContext { + method: "GET".to_string(), + target: READ_FILE_STREAM_PATH.to_string(), + operation: Some(INTERNODE_OPERATION_READ_FILE_STREAM), + }, + RemoteDiskErrorKind::FileCorrupt, + ) + .into_io_error() +} + fn add_root_certificates_from_der(builder: reqwest::ClientBuilder, certs_der: &[Vec]) -> reqwest::ClientBuilder { let mut b = builder; for der in certs_der { @@ -850,6 +870,7 @@ fn classify_http_response( let remote_disk_error = match headers.get(INTERNODE_DISK_ERROR_HEADER).and_then(|value| value.to_str().ok()) { Some(INTERNODE_FILE_NOT_FOUND) => Some(RemoteDiskErrorKind::FileNotFound), Some(INTERNODE_VOLUME_NOT_FOUND) => Some(RemoteDiskErrorKind::VolumeNotFound), + Some(INTERNODE_FILE_CORRUPT) => Some(RemoteDiskErrorKind::FileCorrupt), _ => None, }; ClassifiedHttpResponse { kind, remote_disk_error } @@ -2047,6 +2068,7 @@ mod tests { assert_eq!(INTERNODE_DISK_ERROR_HEADER, "x-rustfs-disk-error"); assert_eq!(INTERNODE_FILE_NOT_FOUND, "file-not-found"); assert_eq!(INTERNODE_VOLUME_NOT_FOUND, "volume-not-found"); + assert_eq!(INTERNODE_FILE_CORRUPT, "file-corrupt"); } #[test] @@ -2090,6 +2112,48 @@ mod tests { assert!(wrong_operation.remote_disk_error.is_none()); } + #[test] + fn classify_http_response_scopes_corrupt_disk_error_to_read_failures() { + let mut headers = HeaderMap::new(); + headers.insert(INTERNODE_DISK_ERROR_HEADER, INTERNODE_FILE_CORRUPT.parse().expect("valid error token")); + let classified = classify_http_response( + reqwest::StatusCode::INTERNAL_SERVER_ERROR, + &headers, + Some(INTERNODE_OPERATION_READ_FILE_STREAM), + ); + assert_eq!(classified.remote_disk_error, Some(RemoteDiskErrorKind::FileCorrupt)); + for status in [reqwest::StatusCode::NOT_FOUND, reqwest::StatusCode::SERVICE_UNAVAILABLE] { + assert!( + classify_http_response(status, &headers, Some(INTERNODE_OPERATION_READ_FILE_STREAM)) + .remote_disk_error + .is_none() + ); + } + for operation in [ + None, + Some(INTERNODE_OPERATION_WALK_DIR), + Some(INTERNODE_OPERATION_PUT_FILE_STREAM), + ] { + assert!( + classify_http_response(reqwest::StatusCode::INTERNAL_SERVER_ERROR, &headers, operation) + .remote_disk_error + .is_none() + ); + } + headers.insert(INTERNODE_DISK_ERROR_HEADER, "unknown-disk-error".parse().expect("valid unknown token")); + for headers in [&headers, &HeaderMap::new()] { + assert!( + classify_http_response( + reqwest::StatusCode::INTERNAL_SERVER_ERROR, + headers, + Some(INTERNODE_OPERATION_READ_FILE_STREAM), + ) + .remote_disk_error + .is_none() + ); + } + } + #[derive(Clone, Default)] struct TestState { head_count: Arc, @@ -2158,6 +2222,16 @@ mod tests { let addr = listener.local_addr().expect("listener local address should be available"); let app = Router::new() .route("/stream", get(get_stream).head(reject_head).put(accept_put)) + .route( + READ_FILE_STREAM_PATH, + get(|| async { + ( + StatusCode::INTERNAL_SERVER_ERROR, + [(INTERNODE_DISK_ERROR_HEADER, INTERNODE_FILE_CORRUPT)], + "read file err file corrupt", + ) + }), + ) .route(WALK_DIR_PATH, get(get_stream)) .route("/reject-put", get(get_stream).put(reject_put)) .route("/stall", get(get_stalling_stream)) @@ -2172,6 +2246,44 @@ mod tests { Some((format!("http://{addr}/stream"), handle)) } + #[tokio::test] + async fn http_readers_preserve_remote_file_corruption() { + let (url, server) = start_test_server(TestState::default()) + .await + .expect("corruption regression server must bind"); + let url = format!("{}{READ_FILE_STREAM_PATH}", url.strip_suffix("/stream").expect("test server URL suffix")); + let byte_error = HttpReader::new(url.clone(), Method::GET, HeaderMap::new(), None) + .await + .err() + .expect("byte reader must reject corrupt shard response"); + let chunk_error = HttpChunkReader::new_with_stall_timeout(url, Method::GET, HeaderMap::new(), None, None) + .await + .err() + .expect("chunk reader must reject corrupt shard response"); + for error in [byte_error, chunk_error] { + let source = error + .get_ref() + .and_then(|source| source.downcast_ref::()) + .expect("HTTP error must retain typed internode source"); + assert!(source.is_remote_file_corrupt()); + assert!(!source.is_remote_file_not_found()); + assert!(!source.is_remote_volume_not_found()); + assert_eq!( + source.kind(), + InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR) + ); + let cloned = clone_internode_http_io_error(&error).expect("typed transport error must be cloneable"); + assert!( + cloned + .get_ref() + .and_then(|source| source.downcast_ref::()) + .expect("clone must retain typed internode source") + .is_remote_file_corrupt() + ); + } + server.abort(); + } + struct BlockedH2Server { url: String, shutdown: tokio::sync::oneshot::Sender<()>, diff --git a/rustfs/src/admin/handlers/audit_runtime_config.rs b/rustfs/src/admin/handlers/audit_runtime_config.rs index 1055d3bf3..ad58b726a 100644 --- a/rustfs/src/admin/handlers/audit_runtime_config.rs +++ b/rustfs/src/admin/handlers/audit_runtime_config.rs @@ -189,6 +189,7 @@ mod tests { use crate::admin::handlers::target_descriptor::admin_target_spec_from_builtin; use crate::admin::runtime_sources::{IamInterface, KmsInterface}; use crate::admin::storage_api::config::save_admin_server_config; + use crate::admin::storage_api::error::StorageError; use rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS; use rustfs_config::server_config::KVS; use rustfs_config::{ENABLE_KEY, EnableState, SCANNER_CYCLE, SCANNER_SUB_SYS, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR}; @@ -231,11 +232,17 @@ mod tests { let mut poll = tokio::time::interval(Duration::from_millis(10)); loop { poll.tick().await; - let config = read_admin_config_without_migrate(store.clone()) - .await - .expect("read persisted server config"); - if config.0.get(subsystem).is_some_and(|targets| targets.contains_key(target)) { - return; + match read_admin_config_without_migrate(store.clone()).await { + Ok(config) => { + if config.0.get(subsystem).is_some_and(|targets| targets.contains_key(target)) { + return; + } + } + Err(StorageError::Lock(rustfs_lock::LockError::Timeout { .. })) => { + // The writer may still be committing the config snapshot. Retry after + // the object lock is released instead of failing the polling helper. + } + Err(error) => panic!("read persisted server config: {error}"), } } }) diff --git a/rustfs/src/storage/rpc/http_service.rs b/rustfs/src/storage/rpc/http_service.rs index 2b0bd9a68..a96f992db 100644 --- a/rustfs/src/storage/rpc/http_service.rs +++ b/rustfs/src/storage/rpc/http_service.rs @@ -1712,16 +1712,17 @@ fn response_with_status(status: StatusCode, message: impl Into) -> Respo } fn response_with_disk_error(error: &DiskError, message: impl Into) -> Response { - let missing = match error { + let disk_error = match error { DiskError::FileNotFound => Some(rustfs_rio::INTERNODE_FILE_NOT_FOUND), DiskError::VolumeNotFound => Some(rustfs_rio::INTERNODE_VOLUME_NOT_FOUND), + DiskError::FileCorrupt => Some(rustfs_rio::INTERNODE_FILE_CORRUPT), _ => None, }; let mut response = response_with_status(StatusCode::INTERNAL_SERVER_ERROR, message); - if let Some(missing) = missing { + if let Some(disk_error) = disk_error { response .headers_mut() - .insert(rustfs_rio::INTERNODE_DISK_ERROR_HEADER, HeaderValue::from_static(missing)); + .insert(rustfs_rio::INTERNODE_DISK_ERROR_HEADER, HeaderValue::from_static(disk_error)); } response } @@ -3087,12 +3088,14 @@ mod tests { } #[test] - fn read_file_error_response_marks_only_missing_disk_errors() { + fn read_file_error_response_preserves_typed_disk_errors() { for (error, expected) in [ (DiskError::FileNotFound, rustfs_rio::INTERNODE_FILE_NOT_FOUND), (DiskError::VolumeNotFound, rustfs_rio::INTERNODE_VOLUME_NOT_FOUND), + (DiskError::FileCorrupt, rustfs_rio::INTERNODE_FILE_CORRUPT), ] { let response = response_with_disk_error(&error, error.to_string()); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); assert_eq!( response.headers().get(rustfs_rio::INTERNODE_DISK_ERROR_HEADER), Some(&HeaderValue::from_static(expected)) From ae6bbaef27ce778de211a8b008233e6fa5ca473e Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:08:41 +0800 Subject: [PATCH 17/30] fix(ci): give the chain staleness probe a token that can read auto-testing (#8041) resolve_functional_candidate.py probes rustfs/auto-testing (private) to age the pinned functional-script revision and fall back to main HEAD after 24h. The prepare step passed github.token, which cannot see the private repo, so every nightly chain logged staleness probe failed (...exit status 1.); keeping pinned revision and replayed the 09-14 harness. On the 09-20 nightly that harness died on the dpkg conffile prompt in all 12 lanes (see rustfs/auto-testing#97) because the --force-confold and other fixes never reached the chain. Use PF_TESTING_GH_TOKEN - already required by the other steps in this workflow - so the probe can actually run and the >24h fallback works. --- .github/workflows/rustfs-functional-chain.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml index f6a45e4e7..945d4bc51 100644 --- a/.github/workflows/rustfs-functional-chain.yml +++ b/.github/workflows/rustfs-functional-chain.yml @@ -54,7 +54,10 @@ jobs: - name: Resolve published candidate id: candidate env: - GH_TOKEN: ${{ github.token }} + # The staleness probe in resolve_functional_candidate.py reads the + # private rustfs/auto-testing repo; github.token cannot, and a probe + # failure silently keeps a stale testing_sha pinned for the chain. + GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} BUILD_RUN_ID: ${{ inputs.build_run_id }} BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }} CHAIN_OUTPUT: ${{ runner.temp }}/chain-candidate.json From 4047ed6c8d3cfc5c8c36657cc1259dcb414a91a3 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 21 Sep 2026 12:18:06 +0800 Subject: [PATCH 18/30] fix(heal): restore bucket metadata before replacement completion (#8042) --- crates/ecstore/src/cluster/rpc/remote_disk.rs | 2 +- crates/ecstore/src/disk/mod.rs | 2 +- crates/ecstore/src/set_disk/ops/heal.rs | 2 +- crates/ecstore/src/store/heal.rs | 309 ++++++++++++++++++ crates/ecstore/src/store/mod.rs | 2 +- crates/ecstore/src/store/peer.rs | 42 ++- crates/heal/src/heal/erasure_healer.rs | 41 +++ crates/heal/src/heal/storage.rs | 12 + .../tests/heal_pool_metadata_scope_test.rs | 46 ++- 9 files changed, 451 insertions(+), 7 deletions(-) diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index d5e41c03e..729c16cf0 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -2141,7 +2141,7 @@ impl RemoteDisk { let file_info_bin = encode_file_info_msgpack(fi)?; let mut client = self.get_client().await?; let mut request = Request::new(RenameDataRequest { - bucket_incarnation_id: crate::store::bucket_heal_scope(dst_volume) + bucket_incarnation_id: crate::store::bucket_heal_scope_for_object(dst_volume, dst_path) .map(|scope| scope.incarnation.as_bytes().to_vec().into()) .unwrap_or_default(), disk: self.endpoint.to_string(), diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index dfaae98bb..7a3f0b174 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -825,7 +825,7 @@ impl Disk { dst_volume: &str, dst_path: &str, ) -> Result { - let Some(scope) = crate::store::bucket_heal_scope(dst_volume) else { + let Some(scope) = crate::store::bucket_heal_scope_for_object(dst_volume, dst_path) else { return self .rename_data_borrowed_with_fence(src_volume, src_path, fi, dst_volume, dst_path, None) .await; diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 906ff1eb3..e469826d5 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -854,7 +854,7 @@ impl SetDisks { ); let disks = self.get_disks_internal().await; - let bucket_heal_scope = crate::store::bucket_heal_scope(bucket); + let bucket_heal_scope = crate::store::bucket_heal_scope_for_object(bucket, object); if let Some(scope) = &bucket_heal_scope { scope.check()?; } diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index 0e7f86168..216fc7778 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -58,6 +58,26 @@ pub(crate) fn bucket_heal_scope(bucket: &str) -> Option> { .flatten() } +/// Resolve only the two canonical bucket records; other internal paths carry no bucket authority. +pub(super) fn bucket_metadata_owner(object: &str) -> Option<&str> { + use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE}; + let (prefix, rest) = object.split_once('/')?; + let (bucket, file) = rest.split_once('/')?; + (prefix == disk::BUCKET_META_PREFIX + && matches!(file, BUCKET_METADATA_FILE | BUCKET_INCARNATION_FILE) + && check_valid_bucket_name_strict(bucket).is_ok()) + .then_some(bucket) +} + +pub(crate) fn bucket_heal_scope_for_object(bucket: &str, object: &str) -> Option> { + let owner = if bucket == RUSTFS_META_BUCKET { + bucket_metadata_owner(object)? + } else { + bucket + }; + bucket_heal_scope(owner) +} + /// Storage-owned proof for the exact version and every selected erasure location. /// This is an in-process result, never reconstructed from admin drive telemetry. #[derive(Debug)] @@ -422,6 +442,97 @@ impl ECStore { Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME))) } + /// Restore the bucket configuration and incarnation before retiring a replacement marker. + pub async fn heal_replacement_bucket_metadata( + self: &Arc, + bucket: &str, + opts: &HealOpts, + targets: &[String], + ) -> Result<()> { + use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE}; + use crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in; + + let (Some(pool_index), Some(set_index)) = (opts.pool, opts.set) else { + return Err(Error::PreconditionFailed); + }; + if targets.is_empty() || opts.dry_run || opts.no_lock || check_valid_bucket_name_strict(bucket).is_err() { + return Err(Error::PreconditionFailed); + } + let pool = self + .pools + .get(pool_index) + .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?; + let target_set = pool.get_disks_for_heal_object(BUCKET_METADATA_FILE, opts)?; + if targets.iter().any(|target| { + target_set + .set_endpoints + .iter() + .filter(|endpoint| endpoint.to_string() == *target) + .count() + != 1 + }) { + return Err(Error::PreconditionFailed); + } + let objects: Vec<_> = [BUCKET_METADATA_FILE, BUCKET_INCARNATION_FILE] + .into_iter() + .map(|file| format!("{}/{bucket}/{file}", disk::BUCKET_META_PREFIX)) + .filter(|object| Arc::ptr_eq(&target_set, &pool.get_disks_by_key(object))) + .collect(); + if objects.is_empty() { + return Ok(()); + } + + // Read the persisted identity without lazy migration: a replacement + // must never turn lost configuration into newly fabricated defaults. + let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?; + let targets = targets.to_vec(); + self.run_bucket_heal_at_incarnation(bucket, incarnation, opts, move |store, bucket, opts| async move { + // Match config writers: lifecycle, metadata transaction, object locks. + // Keep the transaction stable through target readback, including when + // the caller is cancelled and the storage owner finishes its write. + let transaction = acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, &bucket).await?; + for object in objects { + let scope = bucket_heal_scope(&bucket).ok_or(Error::PreconditionFailed)?; + scope.check()?; + if transaction.is_lock_lost() { + return Err(Error::PreconditionFailed); + } + // Config objects occupy one pool, unlike pool.bin. Require a + // successful all-pool lookup before accepting another pool's ownership. + if !store.single_pool() { + let (owner, _) = store + .get_pool_info_for_delete_marker(RUSTFS_META_BUCKET, &object, &ObjectOptions::default()) + .await?; + if owner.index != pool_index { + continue; + } + } + let metadata_opts = HealOpts { remove: false, ..opts }; + let (result, error) = store.heal_object(RUSTFS_META_BUCKET, &object, "", &metadata_opts).await?; + if let Some(error) = error { + return Err(error); + } + for target in &targets { + let mut drives = result.after.drives.iter().filter(|drive| &drive.endpoint == target); + if !drives.next().is_some_and(|drive| drive.state == "ok") || drives.next().is_some() { + return Err(Error::PreconditionFailed); + } + } + if !store + .replacement_targets_have_version(RUSTFS_META_BUCKET, &object, "", pool_index, set_index, &targets) + .await? + { + return Err(Error::PreconditionFailed); + } + } + if transaction.is_lock_lost() { + return Err(Error::PreconditionFailed); + } + Ok(()) + }) + .await + } + /// Return every live erasure set selected by an object-heal scope. pub async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result> { let pools = self.get_pools_for_heal_object(opts)?; @@ -1345,6 +1456,16 @@ mod tests { .await .is_err() ); + for file in [".metadata.bin", ".bucket-incarnation"] { + let metadata = format!("buckets/{bucket}/{file}"); + assert!( + store + .rename_local_data_at_incarnation(&disk_ref, (&bucket, object), &fi, (RUSTFS_META_BUCKET, &metadata), old) + .await + .is_err(), + "stale remote heal must not replace the successor's {file}" + ); + } assert!(store.heal_bucket_at_incarnation(&bucket, old, &opts).await.is_err()); assert!( store @@ -1743,6 +1864,194 @@ mod tests { (temp_dir, store, shutdown) } + #[test] + fn bucket_metadata_heal_owner_rejects_path_aliases_and_unrelated_records() { + for file in [".metadata.bin", ".bucket-incarnation"] { + assert_eq!(bucket_metadata_owner(&format!("buckets/example/{file}")), Some("example")); + } + for path in [ + "/buckets/example/.metadata.bin", + "buckets/../.metadata.bin", + "buckets/example/../.metadata.bin", + "buckets/example/.metadata.bin/extra", + "buckets/example//.metadata.bin", + "buckets/example/.metadata.bin.old", + "config/iam/example/.metadata.bin", + "buckets/example/usage.json", + ] { + assert_eq!(bucket_metadata_owner(path), None, "unexpected bucket authority for {path}"); + } + } + + #[tokio::test] + async fn replacement_bucket_metadata_respects_pool_ownership_and_required_records() { + let (root, store, shutdown) = multi_pool_heal_store().await; + let bucket = "replacement-bucket-metadata"; + for invalid_bucket in ["MixedCase", "../example", RUSTFS_META_BUCKET] { + assert!(matches!( + store + .heal_replacement_bucket_metadata( + invalid_bucket, + &HealOpts { + pool: Some(0), + set: Some(0), + ..Default::default() + }, + &[root.path().join("pool0-disk0").to_string_lossy().into_owned()], + ) + .await, + Err(Error::PreconditionFailed) + )); + } + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket and its persisted records"); + let mut owned = 0; + for pool in 0..2 { + let target = root.path().join(format!("pool{pool}-disk0")); + let records: Vec<_> = [".metadata.bin", ".bucket-incarnation"] + .into_iter() + .map(|file| { + let path = target.join(RUSTFS_META_BUCKET).join("buckets").join(bucket).join(file); + let exists = path.join("xl.meta").exists(); + if exists { + std::fs::remove_dir_all(&path).expect("remove replacement metadata shard"); + owned += 1; + } + (path, exists) + }) + .collect(); + store + .heal_replacement_bucket_metadata( + bucket, + &HealOpts { + pool: Some(pool), + set: Some(0), + recreate: true, + ..Default::default() + }, + &[target.to_string_lossy().into_owned()], + ) + .await + .expect("repair owned records and accept authoritative ownership in the other pool"); + for (path, existed) in records { + assert_eq!(path.join("xl.meta").exists(), existed, "wrong placement for {}", path.display()); + } + } + assert_eq!(owned, 2, "each required record must have exactly one owning pool"); + let object = format!("buckets/{bucket}/.metadata.bin"); + let owner = store + .get_pool_idx_existing_with_opts(RUSTFS_META_BUCKET, &object, &ObjectOptions::default()) + .await + .expect("persisted metadata owner"); + let disk = store.pools[owner].disk_set[0].disks.read().await[0] + .clone() + .expect("metadata target disk"); + let fi = disk + .read_version( + "", + RUSTFS_META_BUCKET, + &object, + "", + &disk::ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("inline metadata to repair through target-side admission"); + assert!(fi.data.is_some(), "fixture must contain an inline metadata shard"); + let target_path = root.path().join(format!("pool{owner}-disk0")); + std::fs::remove_dir_all(target_path.join(RUSTFS_META_BUCKET).join(&object)).expect("remove remote target shard"); + let incarnation = store + .bucket_incarnation_id_from_disk(bucket) + .await + .expect("current incarnation"); + store + .rename_local_data_at_incarnation( + &disk.endpoint().to_string(), + (disk::RUSTFS_META_TMP_BUCKET, "metadata-heal-control"), + &fi, + (RUSTFS_META_BUCKET, &object), + incarnation, + ) + .await + .expect("the target-side RPC admission must accept current bucket metadata repair"); + assert!(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta").exists()); + let (configuration, _) = read_config_no_lock_preserve_empty_with_metadata(store.clone(), &object) + .await + .expect("read the current bucket configuration"); + save_config(store.clone(), &object, configuration) + .await + .expect("publish a newer config revision in the same bucket incarnation"); + let committed = + std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("new target configuration"); + assert!( + store + .rename_local_data_at_incarnation( + &disk.endpoint().to_string(), + (disk::RUSTFS_META_TMP_BUCKET, "delayed-metadata-heal"), + &fi, + (RUSTFS_META_BUCKET, &object), + incarnation, + ) + .await + .is_err(), + "a delayed repair must not overwrite a newer config in the same incarnation" + ); + assert_eq!( + std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("retained configuration"), + committed, + "rejected repair must leave the newer configuration unchanged" + ); + for pool in 0..2 { + for disk in 0..4 { + let path = root + .path() + .join(format!("pool{pool}-disk{disk}")) + .join(RUSTFS_META_BUCKET) + .join("buckets") + .join(bucket) + .join(".metadata.bin"); + if path.exists() { + std::fs::remove_dir_all(path).expect("remove every copy of required bucket configuration"); + } + } + } + metadata_sys::remove_bucket_metadata_in(&store.ctx, bucket) + .await + .expect("remove the cached configuration so repair cannot rely on it"); + assert!( + store + .heal_replacement_bucket_metadata( + bucket, + &HealOpts { + pool: Some(0), + set: Some(0), + recreate: true, + ..Default::default() + }, + &[root.path().join("pool0-disk0").to_string_lossy().into_owned()], + ) + .await + .is_err(), + "total metadata loss must not be accepted as ownership in another pool" + ); + for pool in 0..2 { + assert!( + !root + .path() + .join(format!("pool{pool}-disk0")) + .join(RUSTFS_META_BUCKET) + .join(&object) + .exists(), + "missing required configuration must not be recreated with defaults" + ); + } + shutdown.cancel(); + } + #[tokio::test] #[serial_test::serial] async fn absence_proof_requires_every_selected_pool() { diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 625cf32a2..2e6338993 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -426,8 +426,8 @@ mod bucket_fence; pub(crate) use bucket::await_bucket_namespace_operation; pub use bucket_fence::BucketIncarnationFenceGuard; mod heal; -pub(crate) use heal::bucket_heal_scope; pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult}; +pub(crate) use heal::{bucket_heal_scope, bucket_heal_scope_for_object}; mod heal_walk; pub use heal_walk::HealWalkVersion; mod init; diff --git a/crates/ecstore/src/store/peer.rs b/crates/ecstore/src/store/peer.rs index eb79964db..51aea5dce 100644 --- a/crates/ecstore/src/store/peer.rs +++ b/crates/ecstore/src/store/peer.rs @@ -161,14 +161,52 @@ impl ECStore { "incarnation-bound rename requires namespace locking and a non-nil identity", )); } + let bucket = if destination.0 == RUSTFS_META_BUCKET { + super::heal::bucket_metadata_owner(destination.1).ok_or(DiskError::FileAccessDenied)? + } else { + destination.0 + }; let guard = Arc::new( - self.acquire_bucket_incarnation_fence(destination.0, expected) + self.acquire_bucket_incarnation_fence(bucket, expected) .await .map_err(|error| DiskError::other(error.to_string()))?, ); if guard.is_lock_lost() { return Err(DiskError::other("bucket heal incarnation fence was lost")); } + let transaction = if destination.0 == RUSTFS_META_BUCKET { + // The receiver must retain the config fence after an RPC timeout. + // A delayed request must also reject a newer config in the same + // bucket incarnation before publishing its older repair payload. + let transaction = crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket) + .await + .map_err(|error| DiskError::other(error.to_string()))?; + let (current, _) = self + .get_pool_info_for_delete_marker( + destination.0, + destination.1, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .map_err(|error| DiskError::other(error.to_string()))?; + let current = current.object_info; + if transaction.is_lock_lost() + || guard.is_lock_lost() + || fi.mod_time.is_none() + || current.mod_time != fi.mod_time + || current.data_dir != fi.data_dir + || current.size != fi.size + || current.etag != fi.get_etag() + { + return Err(DiskError::FileAccessDenied); + } + Some(transaction) + } else { + None + }; rename_local_data_with_ctx( &self.ctx, disk_ref, @@ -176,7 +214,7 @@ impl ECStore { fi, destination, RenameDataGuards { - external_guard: Some(guard), + external_guard: Some(Arc::new((guard, transaction))), ..Default::default() }, ) diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 2e3759884..803b661b1 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -907,6 +907,21 @@ impl ErasureSetHealer { } if failed_objects == 0 && skipped_objects == 0 && failed_buckets == 0 { + let targets = if self.pool_metadata_target_endpoints.is_empty() { + self.target_endpoints.as_ref() + } else { + self.pool_metadata_target_endpoints.as_ref() + }; + if self.replacement_task_id.is_some() || (!self.heal_opts.dry_run && self.heal_opts.recreate && !targets.is_empty()) { + self.verify_replacement_identity_fence("bucket metadata").await?; + // Recheck even resumed buckets: their user-object cursor does not + // prove that the replacement holds the internal bucket records. + for bucket in buckets { + self.storage + .heal_replacement_bucket_metadata(bucket, &self.heal_opts, targets) + .await?; + } + } self.heal_replacement_pool_metadata( set_disk_id, &mut ErasureSetPassCounters { @@ -2267,6 +2282,8 @@ mod resume_loop_tests { list_include_lifecycle_object_info: Mutex>, replacement_target_identity_sequences: Mutex>>, pool_metadata_placement: Mutex>, + bucket_metadata_calls: Mutex>, + bucket_metadata_failure: AtomicBool, fail_listing: AtomicBool, fail_listing_buckets: Mutex>, } @@ -2462,6 +2479,13 @@ mod resume_loop_tests { None => Ok(true), } } + async fn heal_replacement_bucket_metadata(&self, bucket: &str, _opts: &HealOpts, _targets: &[String]) -> Result<()> { + self.bucket_metadata_calls.lock().unwrap().push(bucket.to_owned()); + if self.bucket_metadata_failure.load(Ordering::SeqCst) { + return Err(Error::Storage(EcstoreError::PreconditionFailed)); + } + Ok(()) + } async fn replacement_targets_have_version( &self, _bucket: &str, @@ -3058,6 +3082,18 @@ mod resume_loop_tests { ) .with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone())); + env.storage.bucket_metadata_failure.store(true, Ordering::SeqCst); + assert!(healer.heal_erasure_set(&["b".to_string()], "pool_0_set_0").await.is_err()); + let incomplete = ResumeManager::load_replacement_intent(env.healer.disk.clone(), &replacement_task_id) + .await + .expect("failed metadata repair must retain replacement intent") + .get_state() + .await; + assert!(!incomplete.completed); + assert_ne!(incomplete.replacement_phase, crate::heal::resume::ReplacementPhase::Verified); + assert!(env.storage.calls().is_empty(), "metadata failure must stop completion before pool.bin"); + env.storage.bucket_metadata_failure.store(false, Ordering::SeqCst); + let error = healer .heal_erasure_set(&["b".to_string()], "pool_0_set_0") .await @@ -3073,6 +3109,11 @@ mod resume_loop_tests { assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Intent); assert_eq!(state.retry_count, 1); assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]); + assert_eq!( + *env.storage.bucket_metadata_calls.lock().unwrap(), + vec!["b", "b"], + "resuming a completed user scan must retry bucket metadata" + ); } #[tokio::test] diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index a82a6452c..33b260e3f 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -591,6 +591,11 @@ pub trait HealStorageAPI: Send + Sync { Err(Error::other("replacement pool metadata placement is unsupported")) } + /// Heal and physically verify bucket configuration on its replacement targets. + async fn heal_replacement_bucket_metadata(&self, _bucket: &str, _opts: &HealOpts, _targets: &[String]) -> Result<()> { + Err(Error::Storage(StorageError::PreconditionFailed)) + } + /// Read target-specific physical evidence for one replacement version. /// /// This is only used by automatic replacement healing after the normal @@ -1595,6 +1600,13 @@ impl HealStorageAPI for ECStoreHealStorage { .map_err(Error::Storage) } + async fn heal_replacement_bucket_metadata(&self, bucket: &str, opts: &HealOpts, targets: &[String]) -> Result<()> { + self.ecstore + .heal_replacement_bucket_metadata(bucket, opts, targets) + .await + .map_err(Error::Storage) + } + async fn replacement_targets_have_version( &self, bucket: &str, diff --git a/crates/heal/tests/heal_pool_metadata_scope_test.rs b/crates/heal/tests/heal_pool_metadata_scope_test.rs index 0b16fed29..0fa819dee 100644 --- a/crates/heal/tests/heal_pool_metadata_scope_test.rs +++ b/crates/heal/tests/heal_pool_metadata_scope_test.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Regression for replacement healing requiring pool.bin on a non-owning set. +//! Replacement healing must restore internal records only on their owning sets. #![recursion_limit = "256"] use http::HeaderMap; @@ -101,6 +101,31 @@ async fn replacement_pool_metadata_follows_real_two_set_placement() { storage.replacement_pool_metadata_required(&opts).is_err(), "invalid scope must fail closed" ); + assert!( + storage + .heal_replacement_bucket_metadata(bucket, &opts, &[paths[0].to_string_lossy().into_owned()]) + .await + .is_err(), + "bucket metadata repair must reject an incomplete or invalid replacement scope" + ); + } + + for targets in [vec![], vec![paths[4].to_string_lossy().into_owned()]] { + assert!( + storage + .heal_replacement_bucket_metadata( + bucket, + &HealOpts { + pool: Some(0), + set: Some(0), + ..Default::default() + }, + &targets, + ) + .await + .is_err(), + "empty or wrong-set targets must be rejected before repairing metadata" + ); } let mut owning_set = None; @@ -120,6 +145,17 @@ async fn replacement_pool_metadata_follows_real_two_set_placement() { assert!(owning_set.replace(set).is_none(), "only one set owns pool.bin"); std::fs::remove_dir_all(&metadata_path).unwrap(); } + let bucket_records: Vec<_> = [".metadata.bin", ".bucket-incarnation"] + .into_iter() + .map(|file| { + let path = target_path.join(RUSTFS_META_BUCKET).join("buckets").join(bucket).join(file); + let existed = path.join("xl.meta").exists(); + if existed { + std::fs::remove_dir_all(&path).expect("remove bucket metadata shard from replacement"); + } + (path, existed) + }) + .collect(); // Select a user key using the real placement algorithm, independently // of the metadata-scope decision under test. let key = (0..1000) @@ -191,6 +227,14 @@ async fn replacement_pool_metadata_follows_real_two_set_placement() { "completed repair must clear its healing marker" ); assert_eq!(metadata_path.join("xl.meta").exists(), owns_metadata); + for (path, owned) in bucket_records { + assert_eq!( + path.join("xl.meta").exists(), + owned, + "replacement must restore each owned bucket record without creating a wrong-set copy: {}", + path.display() + ); + } for (relative, bytes) in original_parts { assert_eq!(std::fs::read(object_path.join(relative)).unwrap(), bytes, "reconstructed shard differs"); } From bb43ef861a09b57cab1d2e0b9a2cb37a8bc567bb Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 21 Sep 2026 15:29:30 +0800 Subject: [PATCH 19/30] fix(decommission): use owned data for source capacity (#8038) --- crates/ecstore/src/core/pools.rs | 354 ++++++++++++++++++++++++++++--- 1 file changed, 323 insertions(+), 31 deletions(-) diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index ad39321c6..84a1f3c62 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -41,7 +41,7 @@ use crate::config::com::{ }; use crate::data_movement; use crate::data_movement::backpressure::{self, DataMovementOperation}; -use crate::data_usage::DATA_USAGE_CACHE_NAME; +use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_cache}; use crate::disk::error::DiskError; use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET}; use crate::error::{Error, Result}; @@ -925,6 +925,29 @@ fn capacity_target_physical_bytes(data_bytes: usize, layout: DecommissionErasure Ok(capacity_mul_div_ceil(data_bytes, layout.width(), layout.data)) } +async fn decommission_owned_physical_usage(sets: &Sets, layout: DecommissionErasureLayout) -> HashMap { + let mut usage = HashMap::with_capacity(sets.disk_set.len()); + for (set_index, set) in sets.disk_set.iter().enumerate() { + let Ok(cache) = load_data_usage_cache(set.as_ref(), DATA_USAGE_CACHE_NAME).await else { + continue; + }; + let data_usage = cache.dui(DATA_USAGE_ROOT, &[]); + if !data_usage.is_complete_bucket_usage_snapshot() { + continue; + } + let Ok(logical_bytes) = usize::try_from(data_usage.objects_total_size) else { + continue; + }; + let Ok(set_index) = i32::try_from(set_index) else { + continue; + }; + // The data-usage snapshot excludes unrelated files on a shared mount. + // Convert logical object bytes to the source set's physical EC footprint. + usage.insert(set_index, capacity_mul_div_ceil(logical_bytes, layout.width(), layout.data)); + } + usage +} + fn worst_decommission_target_layout(targets: &[DecommissionPoolCapacityInfo]) -> Result { targets .iter() @@ -1861,6 +1884,132 @@ fn ensure_external_decommission_target_admission( Err(Error::SlowDown) } +fn grow_decommission_target_reservation( + meta: &mut PoolMeta, + capacity_infos: &[DecommissionPoolCapacityInfo], + source_pool_index: usize, + target_pool_index: usize, + expected_data_bytes: usize, + mutation_id: uuid::Uuid, + now: OffsetDateTime, +) -> Result { + let original = meta.clone(); + let result = (|| { + let target_capacity = capacity_infos + .iter() + .find(|capacity| capacity.pool_index == target_pool_index) + .ok_or_else(|| { + decommission_capacity_blocked_error("target capacity snapshot is missing during reservation growth") + })?; + let pool_count = meta.pools.len(); + let pool = meta + .pools + .get_mut(source_pool_index) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?; + let reservation = pool + .decommission + .as_mut() + .and_then(|info| info.capacity_reservation.as_mut()) + .filter(|reservation| reservation.active()) + .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during reservation growth"))?; + let target_layout = reservation + .targets + .iter() + .find(|target| target.pool_index == target_pool_index) + .map(|target| target.layout) + .unwrap_or(target_capacity.layout); + let data_bytes = expected_data_bytes.max(1); + let required_target_bytes = capacity_target_physical_bytes(data_bytes, target_layout)?; + let required_peak = required_target_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies)); + let existing_target = reservation + .targets + .iter() + .find(|target| target.pool_index == target_pool_index); + if let Some(target) = existing_target + && target.pending_physical_bytes > 0 + && target.pending_mutation_id != Some(mutation_id) + { + return Err(decommission_capacity_blocked_error(format!( + "source pool {source_pool_index} target pool {target_pool_index} has an unresolved target capacity intent" + ))); + } + let existing_remaining = existing_target + .map(|target| target.remaining_reserved_physical_bytes(reservation.temporary_copies)) + .unwrap_or_default(); + if required_peak <= existing_remaining { + return Ok(false); + } + if reservation.source_data_equivalent_bytes > 0 { + return Err(decommission_capacity_blocked_error(format!( + "source pool {source_pool_index} reservation estimate is exhausted" + ))); + } + if let Some(target) = existing_target + && target_capacity.physical_free <= target.physical_free_at_reservation + { + return Err(decommission_capacity_blocked_error(format!( + "source pool {source_pool_index} target pool {target_pool_index} reservation is exhausted" + ))); + } + let current_peak = reservation.remaining_peak_physical_bytes(); + let additional_peak = required_peak.saturating_sub(existing_remaining); + let additional_predicted = additional_peak.div_ceil(1usize.saturating_add(reservation.temporary_copies)); + let additional_source_data = capacity_source_data_equivalent(additional_predicted, target_layout)?; + reservation.source_data_equivalent_bytes = reservation + .source_data_equivalent_bytes + .saturating_add(additional_source_data); + reservation.source_physical_bytes = reservation + .source_physical_bytes + .saturating_add(capacity_target_physical_bytes(additional_source_data, reservation.source_layout)?); + reservation.predicted_physical_bytes = reservation + .predicted_physical_bytes + .saturating_add(additional_predicted) + .max(capacity_target_physical_bytes(reservation.source_data_equivalent_bytes, target_layout)?); + reservation.temporary_physical_bytes = reservation + .predicted_physical_bytes + .saturating_mul(reservation.temporary_copies); + reservation.peak_physical_bytes = reservation + .predicted_physical_bytes + .saturating_add(reservation.temporary_physical_bytes); + let allocation_growth = reservation.remaining_peak_physical_bytes().saturating_sub(current_peak); + if let Some(target) = reservation + .targets + .iter_mut() + .find(|target| target.pool_index == target_pool_index) + { + target.reserved_physical_bytes = target.reserved_physical_bytes.saturating_add(allocation_growth); + target.physical_total_at_reservation = target.physical_total_at_reservation.max(target_capacity.physical_total); + target.physical_free_at_reservation = target.physical_free_at_reservation.max(target_capacity.physical_free); + } else { + reservation.targets.push(DecommissionCapacityTarget { + pool_index: target_pool_index, + layout: target_layout, + physical_total_at_reservation: target_capacity.physical_total, + physical_free_at_reservation: target_capacity.physical_free, + reserved_physical_bytes: allocation_growth, + consumed_physical_bytes: 0, + observed_physical_bytes: 0, + inflight_physical_bytes: 0, + pending_physical_bytes: 0, + pending_mutation_id: None, + temporary_mutations: Vec::new(), + }); + } + renew_decommission_capacity_reservation(reservation, now, true); + pool.last_update = pool.last_update.max(now); + Ok(true) + })(); + if result.is_ok() { + if let Err(err) = ensure_decommission_capacity_reservations_available(meta, capacity_infos, "reservation growth") { + *meta = original; + return Err(err); + } + } else { + *meta = original; + } + result +} + fn ensure_decommission_target_owner_admission( meta: &PoolMeta, owner: DecommissionCapacityOwner, @@ -11402,10 +11551,11 @@ impl ECStore { } }) .filter(|reservation| { - reservation - .targets - .iter() - .any(|target| target.pool_index == target_pool_index) + !temporary_release + || reservation + .targets + .iter() + .any(|target| target.pool_index == target_pool_index) }) .map(|reservation| (owner, reservation.model_version)) }); @@ -11464,10 +11614,11 @@ impl ECStore { } else { reservation.admits_owner(owner, OffsetDateTime::now_utc()) }) && reservation.model_version == model_version - && reservation - .targets - .iter() - .any(|target| target.pool_index == target_pool_index) + && (!temporary_release + || reservation + .targets + .iter() + .any(|target| target.pool_index == target_pool_index)) }) .is_some(); if !owner_current { @@ -11476,9 +11627,22 @@ impl ECStore { )); } let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; - if !temporary_release { + let reservation_grew = if temporary_release { + false + } else { + let data_bytes = expected_data_bytes.unwrap_or(1).max(1); + let grew = grow_decommission_target_reservation( + &mut snapshot, + &capacity_infos, + source_pool_index, + target_pool_index, + data_bytes, + mutation_id, + OffsetDateTime::now_utc(), + )?; ensure_decommission_capacity_reservations_available(&snapshot, &capacity_infos, "mutation")?; - } + grew + }; let target_layout = snapshot .pools .get(source_pool_index) @@ -11544,7 +11708,7 @@ impl ECStore { OffsetDateTime::now_utc(), )? }; - if pending_added > 0 { + if reservation_grew || pending_added > 0 { let outcome = snapshot .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &[source_pool_index]) .await?; @@ -12386,7 +12550,11 @@ impl ECStore { Ok(()) } - async fn get_decommission_pool_capacity_info(&self, idx: usize) -> Result { + async fn get_decommission_pool_capacity_info( + &self, + idx: usize, + use_owned_usage: bool, + ) -> Result { if let Some(sets) = self.pools.get(idx) { let mut info = sets.storage_info_snapshot().await; info.backend = StorageAdminApi::backend_info(self).await; @@ -12418,8 +12586,13 @@ impl ECStore { layout.data, layout.parity ))); } + let owned_physical_used = if use_owned_usage { + Some(decommission_owned_physical_usage(sets, layout).await) + } else { + None + }; let (physical_total, physical_free, physical_used) = - decommission_physical_pool_capacity(&info.disks, idx, layout, space); + decommission_physical_pool_capacity(&info.disks, idx, layout, space, owned_physical_used.as_ref()); Ok(DecommissionPoolCapacityInfo { pool_index: idx, @@ -12442,7 +12615,20 @@ impl ECStore { let mut capacity_infos = Vec::with_capacity(self.pools.len()); for idx in 0..self.pools.len() { - capacity_infos.push(self.get_decommission_pool_capacity_info(idx).await?); + capacity_infos.push(self.get_decommission_pool_capacity_info(idx, false).await?); + } + Ok(capacity_infos) + } + + async fn get_decommission_all_pool_capacity_infos_with_owned_usage(&self) -> Result> { + #[cfg(any(test, feature = "test-util"))] + if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id) { + return Ok(capacity_infos); + } + + let mut capacity_infos = Vec::with_capacity(self.pools.len()); + for idx in 0..self.pools.len() { + capacity_infos.push(self.get_decommission_pool_capacity_info(idx, true).await?); } Ok(capacity_infos) } @@ -12544,15 +12730,37 @@ impl ECStore { // different target so one mutation never holds two target gates. discard_decommission_capacity_target_permit_except(self.id, owner, None); } - reservation + if let Some((pool_index, _)) = reservation .targets .iter() .filter_map(candidate) .max_by_key(|(_, remaining)| *remaining) - .map(|(pool_index, _)| pool_index) + { + return Ok(pool_index); + } + if reservation.source_data_equivalent_bytes > 0 { + return Err(decommission_capacity_blocked_error(format!( + "source pool {} has no target allocation for {expected_data_bytes} data bytes", + owner.source_pool_index + ))); + } + let capacities = self.get_decommission_all_pool_capacity_infos().await?; + let active_sources = active_decommission_source_indices(&pool_meta); + capacities + .into_iter() + .filter(|capacity| { + capacity.pool_index != owner.source_pool_index + && !active_sources.contains(&capacity.pool_index) + && pool_meta + .pools + .get(capacity.pool_index) + .is_some_and(is_decommission_start_active_pool) + }) + .max_by_key(|capacity| capacity.physical_free) + .map(|capacity| capacity.pool_index) .ok_or_else(|| { decommission_capacity_blocked_error(format!( - "source pool {} has no target allocation for {expected_data_bytes} data bytes", + "source pool {} has no target pool for {expected_data_bytes} data bytes", owner.source_pool_index )) }) @@ -13048,7 +13256,7 @@ impl ECStore { let capacity_infos = if self.pools.is_empty() { Vec::new() } else { - self.get_decommission_all_pool_capacity_infos().await? + self.get_decommission_all_pool_capacity_infos_with_owned_usage().await? }; let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof(); let mut pool_meta = self.pool_meta.write().await; @@ -15355,7 +15563,7 @@ impl ECStore { .is_some_and(|reservation| !reservation.active()) }; if needs_capacity_reservation { - let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; + let capacity_infos = self.get_decommission_all_pool_capacity_infos_with_owned_usage().await?; let mut pool_meta = self.pool_meta.write().await; let version = pool_meta.version; pool_meta.version = POOL_META_VERSION; @@ -16219,7 +16427,7 @@ impl ECStore { self.ensure_decommission_rebalance_idle_after_refresh_under_start_gate() .await?; - let all_capacity_infos = self.get_decommission_all_pool_capacity_infos().await?; + let all_capacity_infos = self.get_decommission_all_pool_capacity_infos_with_owned_usage().await?; let target_fence_proof_available = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof().is_some(); // Signal cancellation before waiting for the movement writer so active @@ -22653,6 +22861,7 @@ fn decommission_physical_pool_capacity( pool_index: usize, layout: DecommissionErasureLayout, logical: PoolSpaceInfo, + owned_physical_used: Option<&HashMap>, ) -> (usize, usize, usize) { let width = layout.width(); let mut sets: HashMap> = HashMap::new(); @@ -22692,13 +22901,20 @@ fn decommission_physical_pool_capacity( .map(|disk| disk.available_space as usize) .min() .unwrap_or_default(); - let max_used = set_disks - .iter() - .map(|disk| (disk.used_space as usize).max((disk.total_space as usize).saturating_sub(disk.available_space as usize))) - .max() - .unwrap_or_default(); + let physical_used_for_set = owned_physical_used + .and_then(|owned| owned.get(&set_disks[0].set_index).copied()) + .unwrap_or_else(|| { + set_disks + .iter() + .map(|disk| { + (disk.used_space as usize).max((disk.total_space as usize).saturating_sub(disk.available_space as usize)) + }) + .max() + .unwrap_or_default() + .saturating_mul(width) + }); physical_total = physical_total.saturating_add(min_total.saturating_mul(width)); - physical_used = physical_used.saturating_add(max_used.saturating_mul(width)); + physical_used = physical_used.saturating_add(physical_used_for_set); if set_disks.len() >= width { physical_free = physical_free.saturating_add(min_free.saturating_mul(width)); } @@ -22881,6 +23097,8 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi #[cfg(test)] mod pools_tests { + use std::collections::HashMap; + use super::DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF; use super::persist_v3_pool_meta_for_test; use super::record_decommission_entry_error; @@ -22914,10 +23132,11 @@ mod pools_tests { ensure_decommission_start_rebalance_meta_allowed, ensure_decommission_start_target_capacity, ensure_decommission_terminal_operation_supported, ensure_decommission_unresolved_verification_disk_count, ensure_local_decommission_pool_leaders, ensure_pool_meta_write_fence, ensure_valid_decommission_pool_index, get_by_index, - guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested, - load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done, - merge_decommission_durable_ilm_receipts, merge_pool_meta_updates_for_save, merge_pool_status_refresh, - missing_decommission_worker_prefix, next_decommission_capacity_generation, observe_decommission_terminal_reload_result, + grow_decommission_target_reservation, guard_decommission_cancelers, has_active_decommission_canceler, + is_decommission_active, is_decommission_cancel_requested, load_decommission_entry_versions, + local_decommission_queue_prefix, mark_decommission_bucket_done, merge_decommission_durable_ilm_receipts, + merge_pool_meta_updates_for_save, merge_pool_status_refresh, missing_decommission_worker_prefix, + next_decommission_capacity_generation, observe_decommission_terminal_reload_result, parse_decommission_durable_ilm_receipt_path, pool_meta_has_active_decommission, publish_pool_meta_updates, read_pool_meta_replica, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion, record_decommission_unresolved_entry, recover_decommission_capacity_reservations, @@ -28472,6 +28691,44 @@ mod pools_tests { } } + #[test] + fn decommission_capacity_reservation_grows_after_stale_empty_snapshot() { + let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(1); + let layout = DecommissionErasureLayout { data: 3, parity: 1 }; + let capacity_infos = vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 0, 0), + DecommissionPoolCapacityInfo::for_test(1, layout, 64, 64, 0), + ]; + let mut meta = PoolMeta { + version: POOL_META_VERSION, + pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)], + ..Default::default() + }; + meta.decommission(0, capacity_infos[0].space).unwrap(); + reserve_decommission_start_target_capacity( + &mut meta, + &[0], + &capacity_infos, + uuid::Uuid::new_v4(), + 1, + now, + DECOMMISSION_CAPACITY_MODEL_VERSION, + ) + .expect("an empty scanner snapshot should still start decommission"); + let grew = grow_decommission_target_reservation(&mut meta, &capacity_infos, 0, 1, 1, uuid::Uuid::new_v4(), now) + .expect("the reservation should grow against current target capacity"); + assert!(grew); + let reservation = meta.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .unwrap(); + assert_eq!(reservation.source_data_equivalent_bytes, 2); + assert_eq!(reservation.predicted_physical_bytes, 3); + assert_eq!(reservation.targets.len(), 1); + assert_eq!(reservation.targets[0].reserved_physical_bytes, 6); + } + #[test] fn decommission_capacity_model_reserves_versions_delete_markers_and_temporary_copies_from_physical_usage() { let reservation = build_decommission_capacity_reservation( @@ -29028,11 +29285,46 @@ mod pools_tests { total: 0, used: 0, }, + None, ); assert_eq!(capacity, (400, 40, 360)); } + #[test] + fn decommission_physical_capacity_prefers_rustfs_owned_usage_over_filesystem_usage() { + let disks = [10_u64, 20, 30, 40] + .into_iter() + .enumerate() + .map(|(disk_index, available_space)| rustfs_madmin::Disk { + endpoint: format!("http://node-{disk_index}"), + drive_path: format!("/disk-{disk_index}"), + state: "ok".to_string(), + total_space: 100, + used_space: 100 - available_space, + available_space, + pool_index: 0, + set_index: 0, + disk_index: disk_index as i32, + ..Default::default() + }) + .collect::>(); + let owned = HashMap::from([(0, 12)]); + let capacity = decommission_physical_pool_capacity( + &disks, + 0, + DecommissionErasureLayout { data: 2, parity: 2 }, + PoolSpaceInfo { + free: 0, + total: 0, + used: 0, + }, + Some(&owned), + ); + + assert_eq!(capacity, (400, 40, 12)); + } + #[test] fn decommission_terminal_transitions_release_capacity_reservations() { let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(1); From e1bc7313314c48fae8c42c112bcee7da7a5a7dea Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 21 Sep 2026 15:43:07 +0800 Subject: [PATCH 20/30] fix(lifecycle): reject empty expiration actions (#8045) * fix(lifecycle): reject empty expiration actions * style: apply rustfmt to lifecycle validation --- crates/lifecycle/src/core.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/lifecycle/src/core.rs b/crates/lifecycle/src/core.rs index d125723ec..fbbf35090 100644 --- a/crates/lifecycle/src/core.rs +++ b/crates/lifecycle/src/core.rs @@ -241,7 +241,9 @@ impl RuleValidate for LifecycleRule { return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_DEL_MARKER_EXPIRATION_DAYS)); } // Rule must have at least one action - let has_expiration = self.expiration.is_some(); + let has_expiration = self.expiration.as_ref().is_some_and(|expiration| { + expiration.days.is_some() || expiration.date.is_some() || expiration.expired_object_delete_marker.is_some() + }); let has_transition = self.transitions.as_ref().is_some_and(|t| !t.is_empty()); // `NewerNoncurrentVersions` on its own is a MinIO extension, not an AWS // form: it keeps the newest N noncurrent versions and expires the rest @@ -4455,6 +4457,19 @@ mod tests { .expect_err("a negative retention count on a transition must be rejected"); } + #[tokio::test] + async fn validate_rejects_empty_expiration_action() { + let mut rule = rule_with_filter(LifecycleRuleFilter::default()); + rule.expiration = Some(LifecycleExpiration::default()); + + let err = config_with_rules(vec![rule]) + .validate(&ObjectLockConfiguration::default()) + .await + .expect_err("an empty Expiration object must not count as an action"); + + assert_eq!(err.to_string(), ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION); + } + #[tokio::test] async fn zero_newer_noncurrent_versions_means_no_count_constraint() { // Zero carries no constraint, matching how the batch limit path has From 077b961232f6a6bd98b70e90cea89ed0ab807212 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 21 Sep 2026 19:15:39 +0800 Subject: [PATCH 21/30] fix(obs): default span events to none and gate span list by level (#8047) JSON log layers synthesized span lifecycle events for every span and embedded the full ancestor span list on every record. With INFO spans on object hot paths that produced multi-megabyte lines, which journald truncated at 48 KiB: the payload was silently dropped and the volume itself became the dominant load. Add RUSTFS_OBS_LOG_SPAN_EVENTS (none|close|full) with a default of none, and keep with_span_list enabled only for verbose (debug/trace) levels so the ancestor chain stops being re-serialized per record. The request_id promotion path reads the span scope, not the span list, so it is unaffected. Refs rustfs/backlog#2642 --- crates/config/src/observability/mod.rs | 11 ++ crates/obs/src/telemetry/filter.rs | 12 ++ crates/obs/src/telemetry/local.rs | 156 ++++++++++++++++++++++--- crates/obs/src/telemetry/otel.rs | 9 +- 4 files changed, 171 insertions(+), 17 deletions(-) diff --git a/crates/config/src/observability/mod.rs b/crates/config/src/observability/mod.rs index b23e5f2c6..ccc3d1d54 100644 --- a/crates/config/src/observability/mod.rs +++ b/crates/config/src/observability/mod.rs @@ -55,6 +55,13 @@ pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME"; pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME"; pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES"; +/// Span lifecycle events emitted by the JSON log layers (`none`|`close`|`full`). +/// +/// Defaults to `none`: a record per span lifecycle multiplies log volume on +/// object hot paths, and emitting the ancestor span list on every record +/// produced multi-megabyte lines that journald truncated (backlog#2642). +pub const ENV_OBS_LOG_SPAN_EVENTS: &str = "RUSTFS_OBS_LOG_SPAN_EVENTS"; + /// Log cleanup related configurations pub const ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES"; pub const ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: &str = "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES"; @@ -99,6 +106,8 @@ pub const DEFAULT_OBS_LOG_DRY_RUN: bool = false; pub const DEFAULT_OBS_LOG_MATCH_MODE_PREFIX: &str = "prefix"; pub const DEFAULT_OBS_LOG_MATCH_MODE: &str = "suffix"; +pub const DEFAULT_OBS_LOG_SPAN_EVENTS: &str = "none"; + /// Default values for observability configuration // ### Supported Environment Values // - `production` - Secure file-only logging @@ -141,6 +150,7 @@ mod tests { assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME"); assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME"); assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES"); + assert_eq!(ENV_OBS_LOG_SPAN_EVENTS, "RUSTFS_OBS_LOG_SPAN_EVENTS"); assert_eq!(ENV_OBS_TRACES_EXPORT_ENABLED, "RUSTFS_OBS_TRACES_EXPORT_ENABLED"); assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED"); assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED"); @@ -178,6 +188,7 @@ mod tests { assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM_GZIP, "gzip"); assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM_ZSTD, "zstd"); assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE_PREFIX, "prefix"); + assert_eq!(DEFAULT_OBS_LOG_SPAN_EVENTS, "none"); assert_eq!(DEFAULT_OBS_LOG_MATCH_MODE, "suffix"); assert_eq!(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM, "zstd"); } diff --git a/crates/obs/src/telemetry/filter.rs b/crates/obs/src/telemetry/filter.rs index 1237ca502..860a71fd5 100644 --- a/crates/obs/src/telemetry/filter.rs +++ b/crates/obs/src/telemetry/filter.rs @@ -89,6 +89,18 @@ fn rust_log_requests_verbose(rust_log: &str) -> bool { }) } +/// Returns `true` when the effective logging configuration selects a verbose +/// (`debug`/`trace`) level. +/// +/// Follows the same precedence as [`build_env_filter`]: `RUST_LOG` overrides +/// `logger_level` when it is set. +pub(super) fn effective_verbose_logging(logger_level: &str) -> bool { + match env::var("RUST_LOG").ok().filter(|value| !value.trim().is_empty()) { + Some(rust_log) => rust_log_requests_verbose(&rust_log), + None => is_verbose_level(logger_level), + } +} + fn should_suppress_noisy_crates(logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool { if let Some(level) = default_level { return !is_verbose_level(level); diff --git a/crates/obs/src/telemetry/local.rs b/crates/obs/src/telemetry/local.rs index fedc28dbf..49bf94f85 100644 --- a/crates/obs/src/telemetry/local.rs +++ b/crates/obs/src/telemetry/local.rs @@ -36,7 +36,7 @@ use crate::cleaner::LogCleaner; use crate::cleaner::types::{CompressionAlgorithm, FileMatchMode}; use crate::config::OtelConfig; use crate::global::{METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL, METRIC_LOG_CLEANER_RUNS_TOTAL, set_observability_metric_enabled}; -use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter}; +use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter}; use crate::telemetry::rolling::{RollingAppender, Rotation}; use metrics::counter; use rustfs_config::observability::{ @@ -44,10 +44,11 @@ use rustfs_config::observability::{ DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM, DEFAULT_OBS_LOG_DELETE_EMPTY_FILES, DEFAULT_OBS_LOG_DRY_RUN, DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_MATCH_MODE, DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS, DEFAULT_OBS_LOG_PARALLEL_COMPRESS, - DEFAULT_OBS_LOG_PARALLEL_WORKERS, DEFAULT_OBS_LOG_ZSTD_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, - DEFAULT_OBS_LOG_ZSTD_WORKERS, + DEFAULT_OBS_LOG_PARALLEL_WORKERS, DEFAULT_OBS_LOG_SPAN_EVENTS, DEFAULT_OBS_LOG_ZSTD_COMPRESSION_LEVEL, + DEFAULT_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, DEFAULT_OBS_LOG_ZSTD_WORKERS, ENV_OBS_LOG_SPAN_EVENTS, }; use rustfs_config::{APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME}; +use rustfs_utils::get_env_str; use serde_json::Value as JsonValue; use std::sync::Arc; use std::{collections::BTreeMap, fmt}; @@ -78,6 +79,37 @@ pub(super) fn resolve_file_stdout_mirror(configured: Option, is_production configured.unwrap_or(!is_production) } +/// Resolves the span lifecycle events emitted by the JSON log layers. +/// +/// Defaults to [`FmtSpan::NONE`]. A record per span lifecycle multiplies log +/// volume on object hot paths and, combined with the ancestor span list, +/// produced multi-megabyte lines that journald truncated (backlog#2642). +/// Operators opt back in with `RUSTFS_OBS_LOG_SPAN_EVENTS=close|full`. +pub(super) fn resolve_span_events() -> FmtSpan { + match get_env_str(ENV_OBS_LOG_SPAN_EVENTS, DEFAULT_OBS_LOG_SPAN_EVENTS) + .trim() + .to_ascii_lowercase() + .as_str() + { + "close" => FmtSpan::CLOSE, + "full" => FmtSpan::FULL, + "none" | "" => FmtSpan::NONE, + unknown => { + // An unrecognised value must not silently re-enable span events. + eprintln!("{STDERR_WARNING_PREFIX} obs: unknown {ENV_OBS_LOG_SPAN_EVENTS}={unknown:?}; falling back to none"); + FmtSpan::NONE + } + } +} + +/// Whether JSON log records embed the full ancestor span list. +/// +/// `with_span_list(true)` re-serializes every ancestor span on each record, so +/// it is reserved for verbose levels where that context earns its volume. +pub(super) fn resolve_span_list(logger_level: &str) -> bool { + effective_verbose_logging(logger_level) +} + #[cfg(unix)] pub(super) fn validate_stdout_sink(file_appender: &RollingAppender) -> Result<(), TelemetryError> { use std::os::fd::AsFd; @@ -222,7 +254,12 @@ where request_id } -pub(super) fn build_json_log_layer(writer: W, enable_ansi: bool, span_events: FmtSpan) -> impl tracing_subscriber::Layer +pub(super) fn build_json_log_layer( + writer: W, + enable_ansi: bool, + span_events: FmtSpan, + span_list: bool, +) -> impl tracing_subscriber::Layer where S: Subscriber + for<'span> LookupSpan<'span>, W: for<'writer> tracing_subscriber::fmt::MakeWriter<'writer> + Send + Sync + 'static, @@ -239,7 +276,7 @@ where .json() .flatten_event(true) .with_current_span(true) - .with_span_list(true) + .with_span_list(span_list) .with_span_events(span_events) .map_event_format(RequestIdJsonFormat::new) } @@ -311,11 +348,12 @@ pub(super) fn init_local_logging( fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: bool) -> Result { let env_filter = build_env_filter(logger_level, None); let (nb, guard) = tracing_appender::non_blocking(std::io::stdout()); - let span_events = if is_production { FmtSpan::CLOSE } else { FmtSpan::FULL }; + let span_events = resolve_span_events(); + let span_list = resolve_span_list(logger_level); // Keep stdout formatting JSON-shaped even in local-only mode so operators // can ship the same log schema to external collectors if needed. - let fmt_layer = build_json_log_layer(nb, std::io::stdout().is_terminal(), span_events); + let fmt_layer = build_json_log_layer(nb, std::io::stdout().is_terminal(), span_events, span_list); tracing_subscriber::registry() .with(env_filter) @@ -413,11 +451,12 @@ fn init_file_logging_internal( // ── 4. Build subscriber layers ──────────────────────────────────────────── let env_filter = build_env_filter(logger_level, None); - let span_events = if is_production { FmtSpan::CLOSE } else { FmtSpan::FULL }; + let span_events = resolve_span_events(); + let span_list = resolve_span_list(logger_level); // File output stays machine-readable and free of ANSI sequences so the // resulting files are safe to parse or ship to log processors. - let file_layer = build_json_log_layer(non_blocking, false, span_events.clone()); + let file_layer = build_json_log_layer(non_blocking, false, span_events.clone(), span_list); // Optional stdout mirror: enabled explicitly via `log_stdout_enabled`, or // unconditionally in non-production environments so developers still see @@ -425,7 +464,10 @@ fn init_file_logging_internal( let (stdout_layer, stdout_guard) = if resolve_file_stdout_mirror(config.log_stdout_enabled, is_production) { let (stdout_nb, stdout_guard) = tracing_appender::non_blocking(std::io::stdout()); let enable_color = std::io::stdout().is_terminal(); - (Some(build_json_log_layer(stdout_nb, enable_color, span_events)), Some(stdout_guard)) + ( + Some(build_json_log_layer(stdout_nb, enable_color, span_events, span_list)), + Some(stdout_guard), + ) } else { (None, None) }; @@ -773,7 +815,7 @@ mod tests { fn render_json_log(enable_ansi: bool) -> (String, Value) { let writer = SharedWriter::default(); - let layer = build_json_log_layer(writer.clone(), enable_ansi, FmtSpan::NONE); + let layer = build_json_log_layer(writer.clone(), enable_ansi, FmtSpan::NONE, false); let subscriber = Registry::default().with(layer); tracing::subscriber::with_default(subscriber, || { @@ -793,9 +835,32 @@ mod tests { (raw, parsed) } + fn render_event_in_nested_spans(depth: usize, span_list: bool) -> String { + let writer = SharedWriter::default(); + let layer = build_json_log_layer(writer.clone(), false, FmtSpan::NONE, span_list); + let subscriber = Registry::default().with(layer); + + tracing::subscriber::with_default(subscriber, || { + let spans = (0..depth) + .map(|level| tracing::info_span!("layer", depth = level, payload = "x".repeat(1024))) + .collect::>(); + let _guards = spans.iter().map(tracing::Span::enter).collect::>(); + info!(component = "obs_contract_test", "nested event"); + }); + + String::from_utf8( + writer + .inner + .lock() + .expect("shared writer lock should not be poisoned") + .clone(), + ) + .expect("log output should be valid UTF-8") + } + fn render_json_log_with_request_span() -> Value { let writer = SharedWriter::default(); - let layer = build_json_log_layer(writer.clone(), false, FmtSpan::NONE); + let layer = build_json_log_layer(writer.clone(), false, FmtSpan::NONE, false); let subscriber = Registry::default().with(layer); tracing::subscriber::with_default(subscriber, || { @@ -818,7 +883,7 @@ mod tests { fn render_json_log_with_recovery_monitor_child_span() -> Value { let writer = SharedWriter::default(); - let layer = build_json_log_layer(writer.clone(), false, FmtSpan::NONE); + let layer = build_json_log_layer(writer.clone(), false, FmtSpan::NONE, true); let subscriber = Registry::default().with(layer); tracing::subscriber::with_default(subscriber, || { @@ -1075,6 +1140,10 @@ mod tests { assert_eq!(parsed["request-id"], Value::String("req-123".to_string())); assert_eq!(parsed["message"], Value::String("inside request span".to_string())); assert_eq!(parsed["span"]["request_id"], Value::String("req-123".to_string())); + assert!( + parsed.get("spans").is_none(), + "ancestor span list must stay out of records when span_list is disabled: {parsed}" + ); } #[test] @@ -1112,4 +1181,65 @@ mod tests { assert_eq!(resolve_log_cleanup_file_pattern(&config, "rustfs.log"), "rustfs.log"); } + + /// Regression for backlog#2642: records must not embed the ancestor span + /// list, so their size stays flat as the enclosing span chain deepens. + #[test] + fn record_size_does_not_grow_with_span_depth_when_span_list_is_disabled() { + let shallow = render_event_in_nested_spans(1, false); + let deep = render_event_in_nested_spans(32, false); + + assert!( + deep.len() <= shallow.len() * 2, + "ancestor spans leaked into the record: shallow={} deep={}", + shallow.len(), + deep.len() + ); + + // The flag is what keeps it bounded: re-enabling the list must show the + // growth the default avoids. + let deep_with_list = render_event_in_nested_spans(32, true); + assert!( + deep_with_list.len() > deep.len(), + "enabling the span list must reintroduce per-ancestor growth" + ); + } + + #[test] + fn span_events_default_to_none_and_parse_overrides() { + temp_env::with_var(ENV_OBS_LOG_SPAN_EVENTS, None::<&str>, || { + assert_eq!(resolve_span_events(), FmtSpan::NONE); + }); + temp_env::with_var(ENV_OBS_LOG_SPAN_EVENTS, Some("close"), || { + assert_eq!(resolve_span_events(), FmtSpan::CLOSE); + }); + temp_env::with_var(ENV_OBS_LOG_SPAN_EVENTS, Some("FULL"), || { + assert_eq!(resolve_span_events(), FmtSpan::FULL); + }); + temp_env::with_var(ENV_OBS_LOG_SPAN_EVENTS, Some("verbose"), || { + assert_eq!( + resolve_span_events(), + FmtSpan::NONE, + "an unrecognised value must not re-enable span events" + ); + }); + } + + #[test] + fn span_list_is_reserved_for_verbose_levels() { + temp_env::with_var("RUST_LOG", None::<&str>, || { + assert!(!resolve_span_list("info"), "info records must not embed the ancestor span list"); + assert!(!resolve_span_list("warn")); + assert!(!resolve_span_list("error")); + assert!(resolve_span_list("debug")); + assert!(resolve_span_list("trace")); + }); + + temp_env::with_var("RUST_LOG", Some("debug"), || { + assert!( + resolve_span_list("info"), + "RUST_LOG=debug overrides the base level and should keep span context" + ); + }); + } } diff --git a/crates/obs/src/telemetry/otel.rs b/crates/obs/src/telemetry/otel.rs index e03d450b3..dd297600d 100644 --- a/crates/obs/src/telemetry/otel.rs +++ b/crates/obs/src/telemetry/otel.rs @@ -68,7 +68,7 @@ use std::{fs, io::IsTerminal, time::Duration}; use tracing::{info, warn}; use tracing_error::ErrorLayer; use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer}; -use tracing_subscriber::{fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; const GET_OBJECT_DURATION_HISTOGRAM_METRICS: &[&str] = &[ "rustfs_io_get_object_request_duration_seconds", @@ -184,7 +184,8 @@ pub(super) fn init_observability_http( // active, the OpenTelemetry bridge is the authoritative sink for // `tracing` events unless local file logging is needed as a fallback. } - let span_events = if is_production { FmtSpan::CLOSE } else { FmtSpan::FULL }; + let span_events = crate::telemetry::local::resolve_span_events(); + let span_list = crate::telemetry::local::resolve_span_list(logger_level); // ── Case 2: File Logging // If a log directory is configured and OTLP log export is unavailable, use // the same rolling-file behavior as the local-only telemetry backend. @@ -221,7 +222,7 @@ pub(super) fn init_observability_http( crate::telemetry::local::validate_stdout_sink(&file_appender)?; let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); - let file_layer = build_json_log_layer(non_blocking, false, span_events.clone()); + let file_layer = build_json_log_layer(non_blocking, false, span_events.clone(), span_list); let cleanup_handle = spawn_cleanup_task(config, log_directory, log_filename, keep_files); Ok((file_layer, guard, cleanup_handle, rotation_str)) })(); @@ -267,7 +268,7 @@ pub(super) fn init_observability_http( if force_stdout_logging || crate::telemetry::local::resolve_file_stdout_mirror(config.log_stdout_enabled, is_production) { let (stdout_nb, stdout_g) = tracing_appender::non_blocking(std::io::stdout()); stdout_guard = Some(stdout_g); - stdout_layer_opt = Some(build_json_log_layer(stdout_nb, std::io::stdout().is_terminal(), span_events)); + stdout_layer_opt = Some(build_json_log_layer(stdout_nb, std::io::stdout().is_terminal(), span_events, span_list)); } let local_file_fallback_enabled = file_layer_opt.is_some(); let stdout_mirror_enabled = stdout_guard.is_some(); From cd2fa50e50b1b4644974479c0a143c3db7485082 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 21 Sep 2026 19:23:58 +0800 Subject: [PATCH 22/30] fix: rename admin metrics endpoint to realtime (#8046) --- crates/e2e_test/src/distributed/observability_test.rs | 4 ++-- docs/architecture/minio-rustfs-router-compatibility.md | 2 +- docs/operations/internode-grpc-benchmark-runbook.md | 2 +- docs/operations/s3-write-failure-diagnostics.md | 2 +- docs/operations/scanner-benchmark-runbook.md | 2 +- docs/operations/scanner-runtime-controls.md | 4 ++-- rustfs/src/admin/handlers/mod.rs | 4 ++-- rustfs/src/admin/handlers/{metrics.rs => realtime.rs} | 10 +++++----- rustfs/src/admin/handlers/system.rs | 6 +++--- rustfs/src/admin/route_policy.rs | 4 ++-- rustfs/src/admin/route_registration_test.rs | 4 ++-- rustfs/src/server/layer.rs | 2 +- scripts/check_architecture_migration_rules.sh | 2 +- 13 files changed, 24 insertions(+), 24 deletions(-) rename rustfs/src/admin/handlers/{metrics.rs => realtime.rs} (96%) diff --git a/crates/e2e_test/src/distributed/observability_test.rs b/crates/e2e_test/src/distributed/observability_test.rs index 280bb6a46..494f1ce09 100644 --- a/crates/e2e_test/src/distributed/observability_test.rs +++ b/crates/e2e_test/src/distributed/observability_test.rs @@ -184,7 +184,7 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() let (status, metrics_body) = admin_request( &node.url, Method::GET, - "/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true", + "/rustfs/admin/v3/realtime?n=1&by-host=true&by-disk=true", None, &dist.cluster.access_key, &dist.cluster.secret_key, @@ -257,7 +257,7 @@ async fn node_admin_body(dist: &DistCluster, node: usize, path: &str) -> TestRes } async fn http_put_counts(dist: &DistCluster, node: usize) -> TestResult<[u64; 2]> { - let body = node_admin_body(dist, node, "/rustfs/admin/v3/metrics?types=512&by-host=true&n=1").await?; + let body = node_admin_body(dist, node, "/rustfs/admin/v3/realtime?types=512&by-host=true&n=1").await?; let sample: RealtimeMetrics = serde_json::from_str(body.lines().next().ok_or("empty HTTP metrics stream")?)?; assert!(sample.errors.is_empty(), "HTTP metrics returned errors: {:?}", sample.errors); let http = sample.aggregated.http.ok_or("HTTP metrics missing at WARN log level")?; diff --git a/docs/architecture/minio-rustfs-router-compatibility.md b/docs/architecture/minio-rustfs-router-compatibility.md index 806ccc098..5677b78c8 100644 --- a/docs/architecture/minio-rustfs-router-compatibility.md +++ b/docs/architecture/minio-rustfs-router-compatibility.md @@ -41,7 +41,7 @@ Every route asserted in `rustfs/src/admin/route_registration_test.rs` is registe | Pools decommission / cancel / clear | 部分兼容 | `rustfs/src/admin/handlers/pools.rs` returns `NotImplemented` when endpoints are not initialized (single-pool or uninitialized clusters). | | `/v3/top/drives`, `/v3/top/net` | 缺失 | Only `/v3/top/locks` is registered (`rustfs/src/admin/handlers/diagnostics.rs`). | | Bucket / site replication per-object diff | 缺失 | `/v3/replicationmetrics` and site-replication status exist; no diff endpoint. | -| MRF (most-recent-failures) replication metrics breakdown | 缺失 | Only the generic `/v3/metrics` stream and replication metrics wire (`rustfs/src/admin/replication_metrics_wire.rs`). | +| MRF (most-recent-failures) replication metrics breakdown | 缺失 | Only the generic `/v3/realtime` stream and replication metrics wire (`rustfs/src/admin/replication_metrics_wire.rs`). | Formerly-missing families that are now registered and therefore not exceptions: `/v3/healthinfo`, `/v3/obdinfo`, `/v3/force-unlock`, `/v3/top/locks`, `/v3/speedtest*`, `/v3/log`, `/v3/trace`, `/v3/profile`, `/v3/profiling/*`, `/v3/idp/{ldap|openid}/*`, `/v3/idp-config/*`. diff --git a/docs/operations/internode-grpc-benchmark-runbook.md b/docs/operations/internode-grpc-benchmark-runbook.md index 2c2fb6385..ed7095009 100644 --- a/docs/operations/internode-grpc-benchmark-runbook.md +++ b/docs/operations/internode-grpc-benchmark-runbook.md @@ -11,7 +11,7 @@ Every stage is env-gated, so "before" and "after" run the *same binary* with dif | --- | --- | | RPC secret | Internode RPC fails closed: remote endpoints with default credentials and no `RUSTFS_RPC_SECRET` abort startup with `store init aborted: endpoints include remote nodes but ...` (`crates/ecstore/src/store/init.rs`). Set a non-default `RUSTFS_RPC_SECRET`, identical on every node. | | systemd start timeout | `deploy/build/rustfs.service` is `Type=notify` and ships `TimeoutStartSec=120s`; READY fires only after quorum. If freshly purged disks need longer, raise it in a drop-in rather than lowering it. | -| Metrics export | RustFS has no Prometheus pull endpoint (`/admin/v3/metrics` is NDJSON, see `rustfs/src/admin/handlers/metrics.rs`); it pushes OTLP. Run an otel-collector (OTLP receiver → Prometheus exporter) and set `RUSTFS_OBS_ENDPOINT`, `RUSTFS_OBS_METRICS_EXPORT_ENABLED=true`, `RUSTFS_OBS_METER_INTERVAL=5`. For lock p99 also set `RUSTFS_OBJECT_LOCK_DIAG_ENABLE=true` (default off). | +| Metrics export | RustFS has no Prometheus pull endpoint (`/admin/v3/realtime` is NDJSON, see `rustfs/src/admin/handlers/realtime.rs`); it pushes OTLP. Run an otel-collector (OTLP receiver → Prometheus exporter) and set `RUSTFS_OBS_ENDPOINT`, `RUSTFS_OBS_METRICS_EXPORT_ENABLED=true`, `RUSTFS_OBS_METER_INTERVAL=5`. For lock p99 also set `RUSTFS_OBJECT_LOCK_DIAG_ENABLE=true` (default off). | | Server env | `RUSTFS_INTERNODE_*` are **server** env. For p0/p1/p2, source the emitted `server-env.sh` on every node and restart before the run; the driver cannot mutate a running server. | ## Driver diff --git a/docs/operations/s3-write-failure-diagnostics.md b/docs/operations/s3-write-failure-diagnostics.md index 16d56e788..3e146d000 100644 --- a/docs/operations/s3-write-failure-diagnostics.md +++ b/docs/operations/s3-write-failure-diagnostics.md @@ -20,7 +20,7 @@ sum by (instance) (rate(rustfs_s3_http_requests_total{method="PUT",outcome=~"[1- Inspect `service_error` and `cancelled` separately; neither implies a received HTTP status. A zero denominator or absent series means no observed traffic, not proof of health. Use `rate` or reset-aware deltas because counters restart with the process. The older `rustfs_s3_operations_total` counter measures handler entries and excludes pre-dispatch rejections; it is not this HTTP denominator. -The authenticated admin metrics endpoint exposes the same counters through the optional `http` field in `aggregated` and `by_host`. Request `/rustfs/admin/v3/metrics?types=512&by-host=true&n=1` on each node for HTTP-only data. The default type selection also includes HTTP outcomes. This endpoint remains an NDJSON stream and does not become a cluster-wide peer fanout. `http.requests` contains `method`, `operation`, `outcome`, and `total`; `http.collected` timestamps collection. Compare consecutive samples from the same host. Concurrent snapshots are not atomic across series. +The authenticated admin metrics endpoint exposes the same counters through the optional `http` field in `aggregated` and `by_host`. Request `/rustfs/admin/v3/realtime?types=512&by-host=true&n=1` on each node for HTTP-only data. The default type selection also includes HTTP outcomes. This endpoint remains an NDJSON stream and does not become a cluster-wide peer fanout. `http.requests` contains `method`, `operation`, `outcome`, and `total`; `http.collected` timestamps collection. Compare consecutive samples from the same host. Concurrent snapshots are not atomic across series. A missing `http` field means an older or non-reporting node, not zero failures. Old map-encoded RPC readers ignore the additive field; new readers accept older snapshots. In mixed-version deployments, check reporting coverage before aggregating a fleet-wide ratio. diff --git a/docs/operations/scanner-benchmark-runbook.md b/docs/operations/scanner-benchmark-runbook.md index 9d245a641..46684d268 100644 --- a/docs/operations/scanner-benchmark-runbook.md +++ b/docs/operations/scanner-benchmark-runbook.md @@ -317,7 +317,7 @@ Use the harness around a post-start bucket creation workload to cover the timing 1. Start RustFS from an empty data path. 2. Start the harness before creating buckets. 3. Create a bucket, upload objects, and keep the harness running until at least one usage save is observed. -4. Compare `scanner-summary.csv` with `/rustfs/admin/v3/metrics?types=1&n=1` bucket metrics. +4. Compare `scanner-summary.csv` with `/rustfs/admin/v3/realtime?types=1&n=1` bucket metrics. Expected evidence: dirty usage is marked, `life_time_scan_cycle` or `life_time_scan_bucket_drive` advances, `life_time_scan_object` advances for object workloads, and `life_time_save_usage` plus `usage_last_save_result=success` appear before non-zero bucket usage metrics are accepted as fresh. diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index c28288738..1c55d0097 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -481,7 +481,7 @@ Read this snapshot before changing scanner controls: `blocked_source` with `life ## Reading Distributed Metrics -`/rustfs/admin/v3/scanner/status` and `/rustfs/admin/v3/metrics` report the node that handles the HTTP request; the metrics endpoint does not fan out to peers. In distributed deployments, query every node explicitly and keep `by-host=true` so each response includes that node's host view: +`/rustfs/admin/v3/scanner/status` and `/rustfs/admin/v3/realtime` report the node that handles the HTTP request; the metrics endpoint does not fan out to peers. In distributed deployments, query every node explicitly and keep `by-host=true` so each response includes that node's host view: ```bash for endpoint in http://node-a:9000 http://node-b:9000 http://node-c:9000; do @@ -493,7 +493,7 @@ for endpoint in http://node-a:9000 http://node-b:9000 http://node-c:9000; do --access_key "$RUSTFS_ACCESS_KEY" \ --secret_key "$RUSTFS_SECRET_KEY" \ --request GET \ - "${endpoint}/rustfs/admin/v3/metrics?types=1&by-host=true&n=1" \ + "${endpoint}/rustfs/admin/v3/realtime?types=1&by-host=true&n=1" \ > "artifacts/scanner-metrics.${node}.$(date -u +%Y%m%dT%H%M%SZ).ndjson" done ``` diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 45b2f2d67..cf709f22e 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -43,7 +43,6 @@ pub mod kms_key_metadata; pub mod kms_keys; pub mod kms_management; pub mod kms_rekey; -pub mod metrics; pub mod mfa; pub mod module_switch; mod notify_runtime_access; @@ -58,6 +57,7 @@ pub mod pools; pub mod profile; pub mod profile_admin; pub mod quota; +pub mod realtime; pub mod rebalance; pub mod replication; pub mod scanner; @@ -145,7 +145,7 @@ mod tests { let _inspect_data_handler = system::InspectDataHandler {}; let _storage_info_handler = system::StorageInfoHandler {}; let _data_usage_handler = system::DataUsageInfoHandler {}; - let _metrics_handler = metrics::MetricsHandler {}; + let _metrics_handler = realtime::MetricsHandler {}; let _profile_handler = profile_admin::ProfileHandler {}; let _profile_status_handler = profile_admin::ProfileStatusHandler {}; let _tls_status_handler = tls_debug::TlsStatusHandler {}; diff --git a/rustfs/src/admin/handlers/metrics.rs b/rustfs/src/admin/handlers/realtime.rs similarity index 96% rename from rustfs/src/admin/handlers/metrics.rs rename to rustfs/src/admin/handlers/realtime.rs index 298338508..b13f10b8c 100644 --- a/rustfs/src/admin/handlers/metrics.rs +++ b/rustfs/src/admin/handlers/realtime.rs @@ -14,7 +14,7 @@ //! Console realtime metrics API. //! -//! This preserves the console's fixed `/admin/v3/metrics` contract while +//! This preserves the console's fixed `/admin/v3/realtime` contract while //! keeping the response format explicitly NDJSON. It is not a Prometheus text //! exposition endpoint. @@ -302,7 +302,7 @@ mod tests { #[test] fn metrics_params_default_to_single_sample() { - let uri: Uri = "/rustfs/admin/v3/metrics".parse().unwrap(); + let uri: Uri = "/rustfs/admin/v3/realtime".parse().unwrap(); let mp = extract_metrics_init_params(&uri); assert_eq!(resolve_sample_count(&mp), DEFAULT_METRICS_SAMPLES); @@ -310,7 +310,7 @@ mod tests { #[test] fn metrics_params_treat_zero_as_single_sample() { - let uri: Uri = "/rustfs/admin/v3/metrics?n=0".parse().unwrap(); + let uri: Uri = "/rustfs/admin/v3/realtime?n=0".parse().unwrap(); let mp = extract_metrics_init_params(&uri); assert_eq!(resolve_sample_count(&mp), DEFAULT_METRICS_SAMPLES); @@ -318,7 +318,7 @@ mod tests { #[test] fn metrics_params_cap_samples_to_safety_limit() { - let uri: Uri = "/rustfs/admin/v3/metrics?n=9999".parse().unwrap(); + let uri: Uri = "/rustfs/admin/v3/realtime?n=9999".parse().unwrap(); let mp = extract_metrics_init_params(&uri); assert_eq!(resolve_sample_count(&mp), MAX_METRICS_SAMPLES); @@ -332,7 +332,7 @@ mod tests { #[tokio::test] async fn metrics_handler_rejects_missing_credentials() { let result = MetricsHandler {} - .call(build_metrics_request("/rustfs/admin/v3/metrics"), Params::new()) + .call(build_metrics_request("/rustfs/admin/v3/realtime"), Params::new()) .await; let err = match result { Ok(_) => panic!("metrics handler must reject unauthenticated requests"), diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index 8b8ea09bd..81b1a2df8 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{cluster_snapshot, metrics}; +use super::{cluster_snapshot, realtime}; use crate::admin::auth::authorize_admin_request; use crate::admin::handlers::account::{ACCOUNT_INFO_ROUTE, ACCOUNT_PASSWORD_ROUTE}; use crate::admin::handlers::mfa::{ACCOUNT_MFA_ROUTE, MFA_CHALLENGE_ROUTE, USER_MFA_ROUTE}; @@ -167,8 +167,8 @@ pub fn register_system_route(r: &mut S3Router) -> std::io::Resul r.insert( Method::GET, - format!("{}{}", ADMIN_PREFIX, "/v3/metrics").as_str(), - AdminOperation(&metrics::MetricsHandler {}), + format!("{}{}", ADMIN_PREFIX, "/v3/realtime").as_str(), + AdminOperation(&realtime::MetricsHandler {}), )?; r.insert( diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 8080aa379..54cd46e4f 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -320,7 +320,7 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ ), admin(HttpMethod::Get, "/rustfs/admin/v3/info", SERVER_INFO, RouteRiskLevel::Sensitive), admin(HttpMethod::Get, "/rustfs/admin/v3/storageinfo", STORAGE_INFO, RouteRiskLevel::Sensitive), - admin(HttpMethod::Get, "/rustfs/admin/v3/metrics", GET_METRICS, RouteRiskLevel::Sensitive), + admin(HttpMethod::Get, "/rustfs/admin/v3/realtime", GET_METRICS, RouteRiskLevel::Sensitive), admin( HttpMethod::Get, "/rustfs/admin/v3/object-data-cache/stats", @@ -2303,7 +2303,7 @@ mod tests { #[test] fn route_policy_maps_metrics_to_explicit_admin_action() { - assert_action(HttpMethod::Get, "/rustfs/admin/v3/metrics", GET_METRICS); + assert_action(HttpMethod::Get, "/rustfs/admin/v3/realtime", GET_METRICS); } #[test] diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 496c93436..b4be4fc4b 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -190,7 +190,7 @@ fn expected_admin_route_matrix() -> Vec { admin_route(Method::GET, "/v3/storageinfo"), admin_route(Method::GET, "/v3/datausageinfo"), admin_route_sample(Method::GET, "/v3/usage/{bucket}", "/v3/usage/test-bucket"), - admin_route(Method::GET, "/v3/metrics"), + admin_route(Method::GET, "/v3/realtime"), admin_route(Method::GET, "/v3/object-data-cache/stats"), admin_route(Method::POST, "/v3/object-data-cache/flush"), admin_route(Method::GET, "/v3/pools/list"), @@ -1333,7 +1333,7 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::POST, &admin_path("/v3/service")); assert_route(&router, Method::GET, &admin_path("/v3/info")); assert_route(&router, Method::GET, &admin_path("/v3/storageinfo")); - assert_route(&router, Method::GET, &admin_path("/v3/metrics")); + assert_route(&router, Method::GET, &admin_path("/v3/realtime")); assert_route(&router, Method::GET, &admin_path("/v3/pools/list")); assert_route(&router, Method::GET, &admin_path("/v3/decommission/status")); diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index f0e9f484e..fc04368ed 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -2538,7 +2538,7 @@ mod tests { assert!(futures::poll!(cancelled.as_mut()).is_pending()); drop(cancelled); for path in [ - "/rustfs/admin/v3/metrics", + "/rustfs/admin/v3/realtime", "/minio/admin/v3/storageinfo", CONSOLE_PREFIX, "/rustfs/rpc/test", diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index d6f1a600f..7b6a2e3af 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -2277,7 +2277,7 @@ fi rustfs/src/admin/router.rs \ rustfs/src/admin/console.rs \ rustfs/src/admin/handlers/heal.rs \ - rustfs/src/admin/handlers/metrics.rs \ + rustfs/src/admin/handlers/realtime.rs \ rustfs/src/admin/handlers/object_zip_download.rs || true ) >"$RUSTFS_APP_ADMIN_STORAGE_API_BYPASS_HITS_FILE" From 8ba3309624fda970158320fdfc3fc0d5bee2bdae Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 21 Sep 2026 19:27:58 +0800 Subject: [PATCH 23/30] fix(rebalance): retry fleet proof before start activation (#8049) Retry the transient missing fleet capability proof in both the admin start flow and node RPC activation. Return a retryable 503 with Retry-After after rollback when readiness does not converge. --- rustfs/src/admin/handlers/rebalance.rs | 238 +++++++++++++++++++++++-- rustfs/src/storage/rpc/node_service.rs | 101 ++++++++++- 2 files changed, 319 insertions(+), 20 deletions(-) diff --git a/rustfs/src/admin/handlers/rebalance.rs b/rustfs/src/admin/handlers/rebalance.rs index 81eb853de..557228264 100644 --- a/rustfs/src/admin/handlers/rebalance.rs +++ b/rustfs/src/admin/handlers/rebalance.rs @@ -43,13 +43,21 @@ use s3s::{ s3_error, }; use serde::{Deserialize, Serialize}; -use std::{sync::Arc, time::Duration}; +use std::{ + future::Future, + sync::Arc, + time::{Duration, Instant}, +}; use time::OffsetDateTime; use tracing::{error, info, warn}; const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_SUBSYSTEM_REBALANCE: &str = "rebalance"; const EVENT_ADMIN_REBALANCE_STATE: &str = "admin_rebalance_state"; +const REBALANCE_START_FLEET_PROOF_MARKER: &str = "pool activation requires a live fleet capability proof"; +const REBALANCE_START_FLEET_PROOF_RETRY_BUDGET: Duration = Duration::from_secs(30); +const REBALANCE_START_FLEET_PROOF_RETRY_DELAY: Duration = Duration::from_secs(5); +const REBALANCE_START_RETRY_AFTER_SECS: &str = "5"; fn admin_request_id(headers: &HeaderMap) -> Option<&str> { headers @@ -105,6 +113,80 @@ fn rebalance_internal_error(message: impl Into) -> S3Error { S3Error::with_message(S3ErrorCode::InternalError, message.into()) } +fn rebalance_fleet_proof_unavailable_error() -> S3Error { + let mut err = S3Error::with_message( + S3ErrorCode::ServiceUnavailable, + "rebalance start is waiting for cluster capability readiness; retry later", + ); + let mut headers = HeaderMap::new(); + headers.insert(http::header::RETRY_AFTER, HeaderValue::from_static(REBALANCE_START_RETRY_AFTER_SECS)); + err.set_headers(headers); + err +} + +fn is_rebalance_start_fleet_proof_retryable(err: &StorageError) -> bool { + crate::storage_api::capacity::is_pool_activation_fleet_proof_error(err) + && err.to_string().contains(REBALANCE_START_FLEET_PROOF_MARKER) +} + +fn rebalance_start_failure_error(start_err: &StorageError, rollback_result: &Result<(), String>) -> S3Error { + if crate::storage_api::capacity::is_pool_activation_fleet_proof_error(start_err) { + if rollback_result.is_ok() { + return rebalance_fleet_proof_unavailable_error(); + } + return rebalance_internal_error("failed to roll back rebalance start after fleet capability readiness failure"); + } + + rebalance_internal_error(rebalance_start_rollback_error(&start_err.to_string(), rollback_result)) +} + +async fn retry_rebalance_start_operation( + operation: &'static str, + rebalance_id: &str, + request_id: &str, + actor: &str, + remote_addr: &str, + deadline: Instant, + mut operation_fn: F, +) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut attempt = 1_u32; + loop { + match operation_fn().await { + Ok(value) => return Ok(value), + Err(err) if is_rebalance_start_fleet_proof_retryable(&err) => { + let now = Instant::now(); + if now >= deadline { + return Err(err); + } + let delay = REBALANCE_START_FLEET_PROOF_RETRY_DELAY.min(deadline.saturating_duration_since(now)); + warn!( + event = EVENT_ADMIN_REBALANCE_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REBALANCE, + action = "start", + state = "fleet_proof_retry_scheduled", + result = "retrying", + operation, + attempt, + retry_delay_ms = delay.as_millis(), + request_id = %request_id, + actor = %actor, + remote_addr = %remote_addr, + rebalance_id = %rebalance_id, + "admin rebalance state" + ); + tokio::time::sleep(delay).await; + attempt = attempt.saturating_add(1); + } + Err(err) => return Err(err), + } + } +} + fn rebalance_rollback_stop_failure_message(rebalance_id: &str, failures: &[String]) -> String { format!("cluster stop_rebalance rollback for {rebalance_id} partial: {}", failures.join("; ")) } @@ -209,7 +291,7 @@ async fn rollback_rebalance_start_for_admin( store: &Arc, notification_sys: Option<&NotificationSys>, rebalance_id: &str, - start_err: &str, + start_err: &StorageError, request_id: &str, actor: &str, remote_addr: &str, @@ -246,7 +328,7 @@ async fn rollback_rebalance_start_for_admin( ), } - Err(rebalance_internal_error(rebalance_start_rollback_error(start_err, &rollback_result))) + Err(rebalance_start_failure_error(start_err, &rollback_result)) } pub fn register_rebalance_route(r: &mut S3Router) -> std::io::Result<()> { @@ -562,6 +644,7 @@ impl Operation for RebalanceStart { "admin rebalance state" ); let notification_sys = current_notification_system(); + let fleet_proof_deadline = Instant::now() + REBALANCE_START_FLEET_PROOF_RETRY_BUDGET; for step in rebalance_start_steps(notification_sys.is_some()) { match step { RebalanceStartStep::PropagateFence => { @@ -592,13 +675,11 @@ impl Operation for RebalanceStart { error = %err, "admin rebalance state" ); - - let start_err = err.to_string(); rollback_rebalance_start_for_admin( &store, Some(notification_sys), &id, - &start_err, + &err, &request_id, &actor, &remote_addr, @@ -608,7 +689,17 @@ impl Operation for RebalanceStart { } } RebalanceStartStep::StartLocal => { - if let Err(err) = store.start_rebalance_for_id(&id).await { + let start_result = retry_rebalance_start_operation( + "start_rebalance_for_id", + &id, + &request_id, + &actor, + &remote_addr, + fleet_proof_deadline, + || store.start_rebalance_for_id(&id), + ) + .await; + if let Err(err) = start_result { error!( event = EVENT_ADMIN_REBALANCE_STATE, component = LOG_COMPONENT_ADMIN, @@ -682,6 +773,9 @@ impl Operation for RebalanceStart { ))); } } + if crate::storage_api::capacity::is_pool_activation_fleet_proof_error(&err) { + return Err(rebalance_fleet_proof_unavailable_error()); + } return Err(rebalance_internal_error(format!( "failed to start rebalance after metadata initialized for {id}; local metadata was finalized as failed: {start_err}" ))); @@ -701,7 +795,17 @@ impl Operation for RebalanceStart { rebalance_id = %id, "admin rebalance state" ); - if let Err(err) = notification_sys.load_rebalance_meta(true).await { + let worker_result = retry_rebalance_start_operation( + "load_rebalance_meta(start=true)", + &id, + &request_id, + &actor, + &remote_addr, + fleet_proof_deadline, + || notification_sys.load_rebalance_meta(true), + ) + .await; + if let Err(err) = worker_result { error!( event = EVENT_ADMIN_REBALANCE_STATE, component = LOG_COMPONENT_ADMIN, @@ -715,13 +819,11 @@ impl Operation for RebalanceStart { error = %err, "admin rebalance state" ); - - let start_err = err.to_string(); rollback_rebalance_start_for_admin( &store, Some(notification_sys), &id, - &start_err, + &err, &request_id, &actor, &remote_addr, @@ -1059,12 +1161,14 @@ mod rebalance_handler_tests { use super::calculate_rebalance_progress; use super::{ Body, HeaderMap, Method, Operation, Params, RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, RebalanceStart, - RebalanceStartStep, RebalanceStatus, RebalanceStop, RebalanceStopPropagationStatus, S3ErrorCode, S3Request, Uri, - build_rebalance_admin_status, build_rebalance_pool_statuses, build_rebalance_stop_propagation_status, - rebalance_pool_used, rebalance_query_present, rebalance_remaining_buckets, rebalance_rollback_failure_message, - rebalance_rollback_stop_failure_message, rebalance_start_rollback_error, rebalance_start_steps, rebalance_stop_target_id, - rebalance_used_pct, rollback_result_label, stop_rebalance_admission_first, + RebalanceStartStep, RebalanceStatus, RebalanceStop, RebalanceStopPropagationStatus, S3ErrorCode, S3Request, StatusCode, + Uri, build_rebalance_admin_status, build_rebalance_pool_statuses, build_rebalance_stop_propagation_status, + is_rebalance_start_fleet_proof_retryable, rebalance_pool_used, rebalance_query_present, rebalance_remaining_buckets, + rebalance_rollback_failure_message, rebalance_rollback_stop_failure_message, rebalance_start_failure_error, + rebalance_start_rollback_error, rebalance_start_steps, rebalance_stop_target_id, rebalance_used_pct, + retry_rebalance_start_operation, rollback_result_label, stop_rebalance_admission_first, }; + use crate::admin::storage_api::error::StorageError; use crate::admin::storage_api::rebalance::{ DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, encode_rebalance_stop_propagation_record, @@ -1328,6 +1432,108 @@ mod rebalance_handler_tests { assert!(message.contains("rollback error: peer b stop_rebalance failed: timeout")); } + #[test] + fn test_rebalance_start_fleet_proof_maps_to_retryable_503() { + let start_err = StorageError::other(super::REBALANCE_START_FLEET_PROOF_MARKER); + + let err = rebalance_start_failure_error(&start_err, &Ok(())); + + assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable); + assert_eq!(err.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE)); + assert_eq!( + err.headers() + .and_then(|headers| headers.get(http::header::RETRY_AFTER)) + .and_then(|value| value.to_str().ok()), + Some(super::REBALANCE_START_RETRY_AFTER_SECS) + ); + assert!( + !err.message() + .expect("retryable rebalance start error should have a message") + .contains(super::REBALANCE_START_FLEET_PROOF_MARKER) + ); + } + + #[test] + fn test_rebalance_start_fleet_proof_rollback_failure_is_generic_internal_error() { + let start_err = StorageError::other(super::REBALANCE_START_FLEET_PROOF_MARKER); + let rollback_result = Err("peer rollback failed".to_string()); + + let err = rebalance_start_failure_error(&start_err, &rollback_result); + + assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert!( + !err.message() + .expect("internal rebalance start error should have a message") + .contains(super::REBALANCE_START_FLEET_PROOF_MARKER) + ); + } + + #[test] + fn test_rebalance_start_unrelated_failure_remains_internal_error() { + let start_err = StorageError::other("disk read failed"); + + let err = rebalance_start_failure_error(&start_err, &Ok(())); + + assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert!(err.message().is_some_and(|message| message.contains("disk read failed"))); + } + + #[test] + fn test_rebalance_start_fleet_proof_retry_only_matches_missing_live_proof() { + let missing = StorageError::other(super::REBALANCE_START_FLEET_PROOF_MARKER); + let expired = StorageError::other(format!( + "rebalance meta save failed during start_rebalance: {}", + "pool activation fleet capability proof expired before commit" + )); + + assert!(is_rebalance_start_fleet_proof_retryable(&missing)); + assert!(!is_rebalance_start_fleet_proof_retryable(&expired)); + } + + #[tokio::test(start_paused = true)] + async fn test_rebalance_start_retry_waits_for_fleet_proof() { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + let attempts = Arc::new(AtomicUsize::new(0)); + let task = tokio::spawn({ + let attempts = Arc::clone(&attempts); + async move { + retry_rebalance_start_operation( + "load_rebalance_meta(start=true)", + "rebalance-id", + "request-id", + "actor", + "remote", + std::time::Instant::now() + super::REBALANCE_START_FLEET_PROOF_RETRY_BUDGET, + move || { + let attempts = Arc::clone(&attempts); + async move { + let attempt = attempts.fetch_add(1, Ordering::SeqCst); + if attempt == 0 { + Err(StorageError::other(super::REBALANCE_START_FLEET_PROOF_MARKER)) + } else { + Ok(()) + } + } + }, + ) + .await + } + }); + + tokio::task::yield_now().await; + assert_eq!(attempts.load(Ordering::SeqCst), 1); + + tokio::time::advance(super::REBALANCE_START_FLEET_PROOF_RETRY_DELAY).await; + task.await + .expect("retry task should not panic") + .expect("fleet proof retry should eventually succeed"); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + } + #[test] fn test_rebalance_rollback_stop_failure_message_lists_failures() { let failures = vec![ diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 72247fc08..45e11792a 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -52,9 +52,11 @@ use serde::Deserialize; use sha2::{Digest, Sha256}; use std::{ collections::HashMap, + future::Future, io::Cursor, pin::Pin, sync::{Arc, LazyLock, OnceLock}, + time::Instant, }; use time::OffsetDateTime; use tokio::spawn; @@ -78,6 +80,9 @@ const EVENT_RPC_REQUEST_FAILED: &str = "rpc_request_failed"; const EVENT_RPC_RESPONSE_EMITTED: &str = "rpc_response_emitted"; const EVENT_RPC_BACKGROUND_TASK_SPAWNED: &str = "rpc_background_task_spawned"; const EVENT_RPC_BACKGROUND_TASK_FAILED: &str = "rpc_background_task_failed"; +const REBALANCE_START_FLEET_PROOF_MARKER: &str = "pool activation requires a live fleet capability proof"; +const REBALANCE_START_FLEET_PROOF_RETRY_BUDGET: Duration = Duration::from_secs(30); +const REBALANCE_START_FLEET_PROOF_RETRY_DELAY: Duration = Duration::from_secs(5); const HEAL_CONTROL_REPLAY_CACHE_MAX_ENTRIES: usize = 4096; const TIER_MUTATION_PEER_STATE_UNSPECIFIED_WIRE: i32 = 0; const TIER_MUTATION_PEER_STATE_PREPARED_WIRE: i32 = 1; @@ -501,6 +506,48 @@ fn background_rebalance_start_error_message(result: StorageResult<()>) -> Option result.err().map(|err| format!("start_rebalance failed: {err}")) } +fn is_rebalance_start_fleet_proof_retryable(err: &Error) -> bool { + crate::storage::storage_api::ecstore_capacity::is_pool_activation_fleet_proof_error(err) + && err.to_string().contains(REBALANCE_START_FLEET_PROOF_MARKER) +} + +async fn retry_rebalance_start_fleet_proof(mut operation: F) -> StorageResult<()> +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let deadline = Instant::now() + REBALANCE_START_FLEET_PROOF_RETRY_BUDGET; + let mut attempt = 1_u32; + + loop { + match operation().await { + Ok(()) => return Ok(()), + Err(err) if is_rebalance_start_fleet_proof_retryable(&err) => { + let now = Instant::now(); + if now >= deadline { + return Err(err); + } + let delay = REBALANCE_START_FLEET_PROOF_RETRY_DELAY.min(deadline.saturating_duration_since(now)); + warn!( + event = EVENT_RPC_BACKGROUND_TASK_FAILED, + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + operation = "start_rebalance", + state = "fleet_proof_retry_scheduled", + result = "retrying", + attempt, + retry_delay_ms = delay.as_millis(), + error = %err, + "node rpc background task retry" + ); + tokio::time::sleep(delay).await; + attempt = attempt.saturating_add(1); + } + Err(err) => return Err(err), + } + } +} + fn stop_rebalance_response(result: StorageResult<()>) -> StopRebalanceResponse { match result { Ok(_) => StopRebalanceResponse { @@ -2800,7 +2847,9 @@ impl Node for NodeService { if start_rebalance { log_background_rebalance_task_spawned!(start_rebalance); - if let Some(message) = background_rebalance_start_error_message(store.start_rebalance().await) { + if let Some(message) = + background_rebalance_start_error_message(retry_rebalance_start_fleet_proof(|| store.start_rebalance()).await) + { error!( event = EVENT_RPC_BACKGROUND_TASK_FAILED, component = LOG_COMPONENT_STORAGE, @@ -2975,9 +3024,10 @@ mod tests { SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_PUBLICATION_LEASE_TTL_MS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message, execute_heal_control_envelope_with_manager, - initialize_heal_topology_fingerprint, initialize_heal_topology_fingerprint_with_probe, legacy_scanner_activity_response, - make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context, - make_tier_mutation_control_server_for_context, previous_scanner_activity_response, remove_heal_control_replay, + initialize_heal_topology_fingerprint, initialize_heal_topology_fingerprint_with_probe, + is_rebalance_start_fleet_proof_retryable, legacy_scanner_activity_response, make_heal_control_server, + make_heal_control_server_with_cache, make_server, make_server_for_context, make_tier_mutation_control_server_for_context, + previous_scanner_activity_response, remove_heal_control_replay, retry_rebalance_start_fleet_proof, scanner_activity_response_v7, start_decommission_failure_response, stop_rebalance_response, validate_admin_heal_control_start, }; @@ -8121,6 +8171,49 @@ mod tests { assert!(message.contains("boom")); } + #[test] + fn test_rebalance_start_retry_ignores_expired_fleet_proof() { + let expired = Error::other("pool activation fleet capability proof expired before commit"); + + assert!(!is_rebalance_start_fleet_proof_retryable(&expired)); + } + + #[tokio::test(start_paused = true)] + async fn test_retry_rebalance_start_waits_for_fleet_proof() { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + let attempts = Arc::new(AtomicUsize::new(0)); + let task = tokio::spawn({ + let attempts = Arc::clone(&attempts); + async move { + retry_rebalance_start_fleet_proof(move || { + let attempts = Arc::clone(&attempts); + async move { + let attempt = attempts.fetch_add(1, Ordering::SeqCst); + if attempt == 0 { + Err(Error::other(super::REBALANCE_START_FLEET_PROOF_MARKER)) + } else { + Ok(()) + } + } + }) + .await + } + }); + + tokio::task::yield_now().await; + assert_eq!(attempts.load(Ordering::SeqCst), 1); + + tokio::time::advance(super::REBALANCE_START_FLEET_PROOF_RETRY_DELAY).await; + task.await + .expect("retry task should not panic") + .expect("fleet proof retry should eventually succeed"); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + } + #[test] fn test_stop_rebalance_response_reports_local_stop_error() { let response = stop_rebalance_response(Err(Error::other("boom"))); From bdea44f332460dcf592d89d13fea3935f413b510 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:13:56 +0800 Subject: [PATCH 24/30] fix(ci): align FT evidence contract with scenario E (44 cases) (#8052) The fault-tolerance suite now registers and runs scenario E (rustfs/auto-testing#100): --all covers A, B, C, C2, D, E with 38 + 6 = 44 expected cases. The chain-evidence assertion still hardcoded [A, B, C, C2, D] and 38 everywhere, which would fail every green FT run's evidence validation once E is registered. Bump the scenario list to include E and all count assertions from 38 to 44; step name updated to match. No other lanes reference 38. --- .../workflows/rustfs-fault-tolerance-test.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/rustfs-fault-tolerance-test.yml b/.github/workflows/rustfs-fault-tolerance-test.yml index ab79aceaf..7127a2e73 100644 --- a/.github/workflows/rustfs-fault-tolerance-test.yml +++ b/.github/workflows/rustfs-fault-tolerance-test.yml @@ -149,7 +149,7 @@ jobs: CHAIN_MANIFEST: ${{ inputs.chain_manifest }} run: python3 scripts/prepare_functional_package.py prepare - - name: Run fault-tolerance scenarios (A, B, C, C2, D) + - name: Run fault-tolerance scenarios (A, B, C, C2, D, E) timeout-minutes: 45 id: test # Standalone case failures are reported by the backlog manager. @@ -194,21 +194,21 @@ jobs: jq -e --arg testing_sha "${AUTO_TESTING_SHA}" ' .schema_version == 1 and .auto_testing_sha == $testing_sha and - .selected_scenarios == ["A", "B", "C", "C2", "D"] and + .selected_scenarios == ["A", "B", "C", "C2", "D", "E"] and .complete == true and - .counts.expected == 38 and - .counts.seen == 38 and + .counts.expected == 44 and + .counts.seen == 44 and .counts.harness_errors == 0 and - (.expected_cases | length) == 38 and - (.expected_cases | unique | length) == 38 and - (.seen_cases | length) == 38 and - (.seen_cases | unique | length) == 38 and + (.expected_cases | length) == 44 and + (.expected_cases | unique | length) == 44 and + (.seen_cases | length) == 44 and + (.seen_cases | unique | length) == 44 and (.seen_cases | sort) == (.expected_cases | sort) and (.missing_cases | length) == 0 and (.harness_errors | length) == 0 and - (.cases | length) == 38 and + (.cases | length) == 44 and ([.cases[].case_id] | sort) == (.expected_cases | sort) and - ([.cases[].case_id] | unique | length) == 38 and + ([.cases[].case_id] | unique | length) == 44 and all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and .counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and .counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length) From 1afaf4491ebcbbdd62d43231df6d5662ae56a355 Mon Sep 17 00:00:00 2001 From: cxymds Date: Mon, 21 Sep 2026 20:56:26 +0800 Subject: [PATCH 25/30] fix(ecstore): bound decommission capacity gate waits (#8044) * fix(ecstore): bound decommission capacity gate waits * fix(ecstore): keep rebalance meta save lock errors retryable The rebalance metadata retry policy only retries when the source chain still carries the typed lock error, but resolve_rebalance_meta_save_result collapsed every failure into a plain string. A transient rebalance.bin write-lock timeout therefore bypassed retry_rebalance_metadata_access entirely and surfaced as a hard failure of the periodic and terminal rebalance metadata saves. Wrap the failure with the data-movement stage context instead, which keeps the original lock error reachable through rebalance_error_source so the existing transient lock policy applies. The rendered message is unchanged. Add a regression test that feeds the wrapped meta save lock timeout through the retry helper and asserts the retry engages. * chore(ecstore): refresh error format ratchet baseline Removing the string-wrapped meta save failure drops one `::other(format!` call site in the rebalance worker, so the shrink-only baseline must be regenerated in the same change. --- crates/ecstore/src/core/pools.rs | 35 +++++++++++++++++-- .../ecstore/src/services/rebalance/runtime.rs | 19 +++++----- .../ecstore/src/services/rebalance/worker.rs | 31 +++++++++++++++- scripts/error-other-format-baseline.txt | 2 +- 4 files changed, 75 insertions(+), 12 deletions(-) diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 84a1f3c62..636b3c04e 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -135,6 +135,7 @@ const DECOMMISSION_CAPACITY_RELEASE_FAILED: &str = "failed"; const DECOMMISSION_CAPACITY_RELEASE_COMPLETED: &str = "completed"; pub(crate) const DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX: &str = "decommission/capacity-target"; const DECOMMISSION_CAPACITY_TARGET_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(250); +const DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS: usize = 12; const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX: &str = "target pool "; const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX: &str = " target capacity mutation gate is busy"; const METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL: &str = "rustfs_decommission_capacity_conflicts_total"; @@ -158,6 +159,10 @@ const DECOMMISSION_COPY_RETRY_DELAY: std::time::Duration = std::time::Duration:: const DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT: usize = 100; const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1); const DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS: usize = 3; + +fn decommission_capacity_target_gate_retry_exhausted(attempt: usize) -> bool { + attempt.saturating_add(1) >= DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS +} const DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT: &str = "decommission/ilm-receipts"; const DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT: &str = "decommission/ilm-manifests"; const DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA: &str = "v2"; @@ -13957,6 +13962,20 @@ impl ECStore { match self.acquire_decommission_capacity_target_guard(target_pool_index).await { Ok(guard) => break guard, Err(err) if is_decommission_capacity_target_gate_busy(&err) => { + if decommission_capacity_target_gate_retry_exhausted(wait_attempt) { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + target_pool_index, + attempts = wait_attempt.saturating_add(1), + state = "capacity_gate_retry_exhausted", + error = %err, + "Decommission target capacity gate remained busy; pausing for supervised retry" + ); + return Err(err); + } wait_attempt = wait_attempt.saturating_add(1); } Err(err) => return Err(err), @@ -23162,8 +23181,9 @@ mod pools_tests { with_decommission_entry_context, }; use super::{ - DecommissionCapacityAdmission, DecommissionCapacityOwner, DecommissionCapacityReleaseProof, - DecommissionCapacityReservation, DecommissionCapacityTemporaryMutation, decommission_capacity_mutation_id, + DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS, DecommissionCapacityAdmission, DecommissionCapacityOwner, + DecommissionCapacityReleaseProof, DecommissionCapacityReservation, DecommissionCapacityTemporaryMutation, + decommission_capacity_mutation_id, decommission_capacity_target_gate_retry_exhausted, ensure_decommission_target_owner_admission, ensure_exact_delete_capacity_namespace_fences, ensure_external_decommission_target_admission, is_decommission_capacity_blocked_error, plan_exact_delete_capacity_reconciliations, record_decommission_target_consumption, release_decommission_target_inflight, @@ -23270,6 +23290,17 @@ mod pools_tests { identity: StdMutex, String)>>, } + #[test] + fn decommission_target_gate_retry_is_bounded() { + assert!(!decommission_capacity_target_gate_retry_exhausted(0)); + assert!(!decommission_capacity_target_gate_retry_exhausted( + DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS - 2 + )); + assert!(decommission_capacity_target_gate_retry_exhausted( + DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS - 1 + )); + } + #[tokio::test] async fn pool_meta_recovery_reconciles_prepare_and_partial_commit_without_format_downgrade() { for previous_version in [POOL_META_V1_VERSION, POOL_META_VERSION, POOL_META_GENERATION_VERSION] { diff --git a/crates/ecstore/src/services/rebalance/runtime.rs b/crates/ecstore/src/services/rebalance/runtime.rs index 2795c4a00..39302d592 100644 --- a/crates/ecstore/src/services/rebalance/runtime.rs +++ b/crates/ecstore/src/services/rebalance/runtime.rs @@ -7,8 +7,9 @@ use super::meta::{ validate_start_rebalance_state, }; use super::worker::{ - resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result, resolve_rebalance_save_task_result, - resolve_rebalance_terminal_error, send_rebalance_done_signal, + rebalance_max_attempts, resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result, + resolve_rebalance_save_task_result, resolve_rebalance_terminal_error, retry_rebalance_metadata_access, + send_rebalance_done_signal, }; use super::{ EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, @@ -449,13 +450,14 @@ impl ECStore { }; if terminal_state_present { - if let Err(err) = store - .save_rebalance_stats_inner( + if let Err(err) = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || { + store.save_rebalance_stats_inner( pool_index, RebalSaveOpt::Stats, Some(save_rebalance_id.as_ref()), ) - .await + }) + .await { let mut rebalance_meta = store.rebalance_meta.write().await; *rebalance_meta = previous_meta; @@ -475,9 +477,10 @@ impl ECStore { } if !terminal_state_saved - && let Err(err) = store - .save_rebalance_stats_for_id(pool_index, RebalSaveOpt::Stats, save_rebalance_id.as_ref()) - .await + && let Err(err) = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || { + store.save_rebalance_stats_for_id(pool_index, RebalSaveOpt::Stats, save_rebalance_id.as_ref()) + }) + .await { let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}")); error!("{} err: {:?}", msg, wrapped); diff --git a/crates/ecstore/src/services/rebalance/worker.rs b/crates/ecstore/src/services/rebalance/worker.rs index 7b6dc85f0..00ffa7ce3 100644 --- a/crates/ecstore/src/services/rebalance/worker.rs +++ b/crates/ecstore/src/services/rebalance/worker.rs @@ -109,7 +109,13 @@ pub(super) fn resolve_rebalance_save_task_result( } pub(super) fn resolve_rebalance_meta_save_result(result: Result<()>, stage: &str) -> Result<()> { - result.map_err(|err| Error::other(format!("rebalance meta save failed during {stage}: {err}"))) + // Keep the source error reachable: the metadata retry policy classifies + // transient lock timeouts by inspecting the source chain, so collapsing the + // failure into a plain string here would make that retry a no-op. + result.map_err(|err| { + let rendered = format!("rebalance meta save failed during {stage}: {err}"); + crate::data_movement::data_movement_context_error(rendered, err) + }) } pub(super) fn rebalance_meta_lock_error(err: rustfs_lock::LockError, mode: &'static str) -> Error { @@ -739,6 +745,29 @@ mod error_source_tests { } } + #[tokio::test] + async fn rebalance_metadata_retry_engages_for_wrapped_meta_save_lock_timeout() { + let mut attempts = 0; + let result = retry_rebalance_metadata_access(None, 3, || { + attempts += 1; + std::future::ready(resolve_rebalance_meta_save_result( + Err(rebalance_meta_lock_error( + rustfs_lock::LockError::timeout(".rustfs.sys/rebalance.bin@latest", Duration::from_secs(5)), + "write", + )), + "save_rebalance_stats for pool 0 opt Stats", + )) + }) + .await; + + assert_eq!(attempts, 3, "a wrapped meta save lock timeout must stay retryable"); + let err = result.expect_err("persistent lock contention must not become success"); + assert!(matches!( + rebalance_error_source(&err), + Error::Lock(rustfs_lock::LockError::Timeout { .. }) + )); + } + #[tokio::test] async fn rebalance_metadata_retry_cancels_a_pending_attempt() { struct DropProbe(Arc); diff --git a/scripts/error-other-format-baseline.txt b/scripts/error-other-format-baseline.txt index 5939a7027..6c39b9f26 100644 --- a/scripts/error-other-format-baseline.txt +++ b/scripts/error-other-format-baseline.txt @@ -50,7 +50,7 @@ 1|crates/ecstore/src/services/rebalance/entry.rs 8|crates/ecstore/src/services/rebalance/meta.rs 8|crates/ecstore/src/services/rebalance/runtime.rs -18|crates/ecstore/src/services/rebalance/worker.rs +17|crates/ecstore/src/services/rebalance/worker.rs 33|crates/ecstore/src/services/tier/tier.rs 1|crates/ecstore/src/services/tier/tier_config.rs 1|crates/ecstore/src/services/tier/warm_backend.rs From 4b0118520e822416a7078610398be70d971da5f2 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Tue, 22 Sep 2026 00:18:40 +0800 Subject: [PATCH 26/30] test(ci): align FT workflow contract tests with scenario E (44 cases) (#8054) #8052 renamed the fault-tolerance step to 'Run fault-tolerance scenarios (A, B, C, C2, D, E)' and raised the evidence gate to 44 cases, but scripts/test_security_workflow.py still expected the old step name and a 38-case fixture, breaking Quick Checks with three KeyErrors and one assertion failure. Update DIRECT_TESTS and the write_result fixture to the new contract. --- scripts/test_security_workflow.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py index 52d07bb03..ccaab5306 100644 --- a/scripts/test_security_workflow.py +++ b/scripts/test_security_workflow.py @@ -407,7 +407,7 @@ class FunctionalWorkflowTests(unittest.TestCase): "kms": "Run KMS suite", "storage": "Run storage engine suite", "s3-compat": "Run S3 compatibility suite", "upgrade": "Run upgrade compatibility suite", "replication": "Run replication suite", - "table": "Run table suite", "fault-tolerance": "Run fault-tolerance scenarios (A, B, C, C2, D)", + "table": "Run table suite", "fault-tolerance": "Run fault-tolerance scenarios (A, B, C, C2, D, E)", } def test_failure_and_always_step_wiring(self) -> None: @@ -538,10 +538,10 @@ class FaultToleranceWorkflowContractTests(unittest.TestCase): "GITHUB_RUN_ID": "314159", } - def write_result(self, *, seen: int = 38, complete: bool = True, + def write_result(self, *, seen: int = 44, complete: bool = True, duplicate: bool = False, testing_sha: str | None = None, unrelated_seen: bool = False, missing_verdict: bool = False) -> None: - expected = [f"CASE-{index:02d}" for index in range(38)] + expected = [f"CASE-{index:02d}" for index in range(44)] seen_cases = expected[:seen] if unrelated_seen: seen_cases = [f"OTHER-{index:02d}" for index in range(seen)] @@ -553,14 +553,14 @@ class FaultToleranceWorkflowContractTests(unittest.TestCase): payload = { "schema_version": 1, "auto_testing_sha": testing_sha or self.TESTING_SHA, - "selected_scenarios": ["A", "B", "C", "C2", "D"], + "selected_scenarios": ["A", "B", "C", "C2", "D", "E"], "complete": complete, "expected_cases": expected, "seen_cases": seen_cases, "missing_cases": expected[seen:], "harness_errors": [], "counts": { - "expected": 38, + "expected": 44, "seen": seen, "unexpected": 1, "known_divergence": 0, From d7c42d409985da231828939e6619d2e6dfb02e59 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Tue, 22 Sep 2026 10:25:36 +0800 Subject: [PATCH 27/30] fix(ci): accept the testing-sha staleness fallback in chain evidence (#8056) The 09-22 nightly chain failed all 12 lanes in seconds at the 'Bind functional candidate' step: ValueError: private script pin differs from chain resolve_functional_candidate.py's >24h staleness fallback (added by #8026, made functional by #8041's token fix) legitimately sets manifest.testing_sha to auto-testing main HEAD, but current_chain() still required it to equal .config/functional-script-revision.txt - a check written for the pre-fallback world where the two could never diverge. Once the fallback finally fired, prepare produced testing_sha 21edcf4 while the pin file still holds 27e9584 and every lane aborted before checking out the test scripts. Drop the pin-file comparison and keep what the lane actually needs to guarantee: testing_sha is a valid commit sha (current_chain), the lane checked out exactly that sha (record: private_head == testing_sha, kept as-is), and the health checker validates the same format instead of re-reading the pin file. Tests updated: a fallback testing_sha that differs from the pin is accepted; a non-sha testing_sha is rejected. Verified: python3 -m unittest test_functional_chain test_functional_chain_health -> 39 tests OK. --- scripts/functional_chain_evidence.py | 7 ++++++- scripts/functional_chain_health.py | 5 +++-- scripts/test_functional_chain.py | 14 ++++++++++++++ scripts/test_functional_chain_health.py | 8 +++++++- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/scripts/functional_chain_evidence.py b/scripts/functional_chain_evidence.py index d070c7d6a..3f3cbbdb3 100644 --- a/scripts/functional_chain_evidence.py +++ b/scripts/functional_chain_evidence.py @@ -28,7 +28,12 @@ def current_chain(): require(sha(chain["workflow_sha"]) and chain["workflow_sha"] == os.environ["GITHUB_SHA"], "chain workflow source mismatch") head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() require(head == chain["workflow_sha"], "lane checkout differs from chain workflow source") - require(chain["testing_sha"] == (ROOT / ".config/functional-script-revision.txt").read_text().strip() and sha(chain["testing_sha"]), "private script pin differs from chain") + # testing_sha is either the committed pin or auto-testing main HEAD via + # resolve_functional_candidate.py's >24h staleness fallback, so pin + # equality is no longer an invariant (the 09-21 chain died on exactly + # that check once the fallback finally fired). Lanes check out exactly + # this sha, which is what the format check guards. + require(sha(chain["testing_sha"]), "private script revision is not a valid commit sha") candidate = chain["candidate"] require(isinstance(candidate, dict) and set(candidate) == {"manifest", "artifact_id", "artifact_digest", "workflow_sha", "workflow_ref", "build_started_at"}, "invalid candidate envelope") manifest = candidate["manifest"] diff --git a/scripts/functional_chain_health.py b/scripts/functional_chain_health.py index d32deaf1b..efcf54a05 100644 --- a/scripts/functional_chain_health.py +++ b/scripts/functional_chain_health.py @@ -32,8 +32,9 @@ def validate_summary(summary, run): candidate = chain["candidate"] manifest = candidate["manifest"] require(resolve(manifest["build_run_id"], manifest["build_run_attempt"]) == candidate, "producer candidate identity changed") - config = api(f"repos/{REPOSITORY}/contents/.config/functional-script-revision.txt?ref={run['head_sha']}") - require(base64.b64decode(config["content"]).decode().strip() == chain["testing_sha"], "private pin differs from workflow source") + # testing_sha may legitimately be auto-testing main HEAD via the prepare + # step's >24h staleness fallback (checked for sha format above), so pin + # equality is not an invariant; drop the pin-file comparison. completed = timestamp(summary["completed_at"]) require(timestamp(run["run_started_at"]) <= completed <= datetime.now(timezone.utc) + timedelta(minutes=5), "invalid completion timestamp") source_ref = manifest.get("source_ref", candidate["workflow_ref"]) diff --git a/scripts/test_functional_chain.py b/scripts/test_functional_chain.py index 4d737781a..b7fde514a 100644 --- a/scripts/test_functional_chain.py +++ b/scripts/test_functional_chain.py @@ -226,6 +226,20 @@ class EnvelopeTests(unittest.TestCase): evidence.current_chain() self.assertFalse((self.root / "env").exists()) + def test_testing_sha_fallback_is_accepted_while_garbage_is_rejected(self): + # prepare's >24h staleness fallback legitimately sets testing_sha to + # auto-testing main HEAD, which differs from the committed pin; only + # the sha format is an invariant now. + for testing_sha, ok in (("d" * 40, True), ("1" * 40, True), ("xyz", False), ("", False)): + chain = dict(self.chain, testing_sha=testing_sha) + env = dict(self.env, CHAIN_MANIFEST=json.dumps(chain)) + if ok: + with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, env), mock.patch.object(evidence.subprocess, "check_output", return_value="e" * 40): + evidence.consume(evidence.current_chain()) + else: + with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, env), mock.patch.object(evidence.subprocess, "check_output", return_value="e" * 40), self.assertRaises(ValueError): + evidence.current_chain() + def test_report_or_swallowed_test_failure_cannot_produce_valid_evidence(self): report = self.root / "cases.md" report.write_text("| Case | Name | Status |\n| --- | --- | --- |\n| KMS-1 | fixture | PASS |\n") diff --git a/scripts/test_functional_chain_health.py b/scripts/test_functional_chain_health.py index 225bb99b4..b03aedf7a 100644 --- a/scripts/test_functional_chain_health.py +++ b/scripts/test_functional_chain_health.py @@ -39,8 +39,14 @@ class HealthTests(unittest.TestCase): def test_substituted_producer_pin_attempt_or_empty_suite_fails(self): with self.assertRaises(ValueError): self.validate(candidate={**self.candidate, "workflow_sha": "e" * 40}) + # The prepare step's >24h staleness fallback legitimately sets + # testing_sha to auto-testing main HEAD, so the pin FILE is no longer + # consulted at all; the sha FORMAT of the chain's testing_sha is the + # remaining invariant. + wrong = copy.deepcopy(self.summary) + wrong["chain"]["testing_sha"] = "short" with self.assertRaises(ValueError): - self.validate(config={"content": base64.b64encode(b"wrong pin").decode()}) + self.validate(wrong) wrong = copy.deepcopy(self.summary) wrong["chain"]["attempt"] = 1 with self.assertRaises(ValueError): From e12100b4a1017d737213a703c48843e9090b5c22 Mon Sep 17 00:00:00 2001 From: cxymds Date: Tue, 22 Sep 2026 11:18:59 +0800 Subject: [PATCH 28/30] fix(s3): emit x-amz-expiration as HTTP-date (#8057) The expiry-date field of the x-amz-expiration response header was rendered as RFC 3339 (e.g. 2026-10-02T00:00:00Z). AWS SDKs parse this value with an RFC 822 parser; the Java SDK v1 rejects the ISO-8601 shape, drops the whole header, and logs a WARN from ObjectExpirationHeaderHandler. S3 specifies HTTP-date (RFC 1123 with a literal GMT zone), which is also what MinIO emits. Render via the shared HTTP-date formatter (extracted alongside format_expires_header) and pin the exact output in the unit test. --- rustfs/src/app/object/shared.rs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/rustfs/src/app/object/shared.rs b/rustfs/src/app/object/shared.rs index 62c7095ec..421c47cdf 100644 --- a/rustfs/src/app/object/shared.rs +++ b/rustfs/src/app/object/shared.rs @@ -673,7 +673,10 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option) -> S3Result S3Result { + format_http_date(expires) +} + +/// Render a date-valued S3 response header (`Expires`, `x-amz-expiration`) as +/// HTTP-date: RFC 1123 with a literal `GMT` zone, e.g. +/// `Fri, 23 Dec 2012 00:00:00 GMT`. +/// +/// Interoperability depends on this shape. `x-amz-expiration` in particular is +/// parsed with an RFC 822 parser by the AWS SDKs, which rejects an RFC 3339 +/// value such as `2026-10-02T00:00:00Z` and then discards the whole header. +pub(super) fn format_http_date(timestamp: &Timestamp) -> S3Result { let mut formatted = Vec::new(); - expires + timestamp .format(TimestampFormat::HttpDate, &mut formatted) - .map_err(|e| ApiError::from(StorageError::other(format!("Invalid expires timestamp: {e}"))))?; + .map_err(|e| ApiError::from(StorageError::other(format!("Invalid HTTP-date timestamp: {e}"))))?; Ok(String::from_utf8_lossy(&formatted).into_owned()) } @@ -1873,9 +1887,10 @@ mod tests { storage_class: String::new(), }; - let expiry_date = expire_time.format(&Rfc3339).unwrap(); - let expected = format!("expiry-date=\"{}\", rule-id=\"rule-1\"", expiry_date); - assert_eq!(build_put_object_expiration_header(&event), Some(expected)); + // HTTP-date, not RFC 3339: the AWS SDKs parse this field with an + // RFC 822 parser, so `2026-10-02T00:00:00Z` loses the metadata. + let expected = "expiry-date=\"Tue, 14 Nov 2023 22:13:20 GMT\", rule-id=\"rule-1\""; + assert_eq!(build_put_object_expiration_header(&event), Some(expected.to_string())); } #[test] From 0ff06757c8cc592b293f97e3a92044d4fc55ba80 Mon Sep 17 00:00:00 2001 From: Hauser Date: Tue, 22 Sep 2026 13:21:44 +0800 Subject: [PATCH 29/30] fix(obs): support custom OTLP CA bundles (#8060) Add RUSTFS_OBS_TLS_CA_FILE for additive private CA trust in OTLP HTTP exporters. Co-authored-by: zhi22915 --- Cargo.lock | 5 + crates/config/src/observability/mod.rs | 3 + crates/obs/Cargo.toml | 6 + crates/obs/README.md | 3 + crates/obs/src/config.rs | 27 ++- crates/obs/src/error.rs | 12 ++ crates/obs/src/telemetry/otel.rs | 262 ++++++++++++++++++++++++- 7 files changed, 310 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82d57995a..05711417c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10408,6 +10408,8 @@ dependencies = [ "opentelemetry_sdk", "percent-encoding", "pyroscope", + "rcgen", + "reqwest", "rustfs-audit", "rustfs-common", "rustfs-config", @@ -10421,6 +10423,8 @@ dependencies = [ "rustfs-storage-api", "rustfs-utils", "rustix", + "rustls", + "rustls-pki-types", "serde", "serde_json", "sysinfo", @@ -10428,6 +10432,7 @@ dependencies = [ "tempfile", "thiserror 2.0.20", "tokio", + "tokio-rustls", "tokio-util", "tracing", "tracing-appender", diff --git a/crates/config/src/observability/mod.rs b/crates/config/src/observability/mod.rs index ccc3d1d54..9a1acc6c4 100644 --- a/crates/config/src/observability/mod.rs +++ b/crates/config/src/observability/mod.rs @@ -30,6 +30,9 @@ pub const ENV_OBS_ENDPOINT_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TIMEOUT_M pub const ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS"; pub const ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS"; pub const ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS: &str = "RUSTFS_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS"; +/// Absolute path to a PEM bundle whose certificates are added to the system +/// trust roots used by OTLP/HTTP exporters. +pub const ENV_OBS_TLS_CA_FILE: &str = "RUSTFS_OBS_TLS_CA_FILE"; pub const ENV_OBS_PROFILING_ENDPOINT: &str = "RUSTFS_OBS_PROFILING_ENDPOINT"; pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT"; pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO"; diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 22fb71dc6..03f1a098b 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -139,6 +139,7 @@ opentelemetry-stdout = { workspace = true } opentelemetry-otlp = { workspace = true, features = ["gzip-http", "reqwest-rustls"] } opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] } percent-encoding = { workspace = true } +reqwest.workspace = true serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } tracing = { workspace = true, features = ["std", "attributes"] } @@ -167,3 +168,8 @@ tempfile = { workspace = true } temp-env = { workspace = true } log.workspace = true metrics-util = { workspace = true, features = ["debugging"] } +rcgen.workspace = true +rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "std"] } +rustls-pki-types.workspace = true +tokio = { workspace = true, features = ["io-util", "net"] } +tokio-rustls = { workspace = true, default-features = false, features = ["aws-lc-rs"] } diff --git a/crates/obs/README.md b/crates/obs/README.md index 43ab1abc3..4d8a1cc67 100644 --- a/crates/obs/README.md +++ b/crates/obs/README.md @@ -115,6 +115,7 @@ All configuration is read from environment variables at startup. | `RUSTFS_OBS_TRACE_ENDPOINT` | _(empty)_ | Dedicated trace endpoint (overrides root + `/v1/traces`) | | `RUSTFS_OBS_METRIC_ENDPOINT` | _(empty)_ | Dedicated metrics endpoint | | `RUSTFS_OBS_LOG_ENDPOINT` | _(empty)_ | Dedicated log endpoint | +| `RUSTFS_OBS_TLS_CA_FILE` | _(empty)_ | Absolute PEM CA bundle added to system trust for OTLP/HTTP | | `RUSTFS_OBS_PROFILING_ENDPOINT` | _(empty)_ | Dedicated profiling endpoint (e.g. Pyroscope) | | `RUSTFS_OBS_TRACES_EXPORT_ENABLED` | `true` | Toggle trace export | | `RUSTFS_OBS_METRICS_EXPORT_ENABLED` | `true` | Toggle metrics export | @@ -124,6 +125,8 @@ All configuration is read from environment variables at startup. | `RUSTFS_OBS_SAMPLE_RATIO` | `0.1` | Trace sampling ratio `0.0`–`1.0` | | `RUSTFS_OBS_METER_INTERVAL` | `30` | Metrics export interval (seconds) | +`RUSTFS_OBS_TLS_CA_FILE` is read only while OTLP exporters are initialized. Its PEM certificates are added to, not substituted for, the platform trust roots, and TLS chain, expiry, and hostname checks remain enabled. Restart RustFS after rotating the bundle. A missing, unreadable, non-absolute, empty, or invalid bundle fails OTLP initialization. + The export interval is separate from application metric collection. Without interval overrides, node/disk metrics refresh every 60 seconds (`RUSTFS_METRICS_NODE_INTERVAL_SEC`), cluster metrics every 60 seconds (`RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`), and per-bucket metrics every 300 seconds (`RUSTFS_METRICS_BUCKET_INTERVAL_SEC`). See the [collection defaults](src/metrics/config.rs) and [interval configuration](src/metrics/scheduler.rs) for collector-specific and global overrides. With OTLP → Collector → Prometheus, the time from a failure to an alert also includes underlying health detection, collection and export work, transport and Collector batching, Prometheus `scrape_interval` and `evaluation_interval`, and any alert rule `for` duration. A shorter scrape interval cannot refresh a value that RustFS has not yet collected and exported. These periodic intervals do not guarantee a fixed detection or alert latency; see the [Prometheus alert rule documentation](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) for the role of `for`. diff --git a/crates/obs/src/config.rs b/crates/obs/src/config.rs index ac067a50a..70cb43aca 100644 --- a/crates/obs/src/config.rs +++ b/crates/obs/src/config.rs @@ -40,7 +40,8 @@ use rustfs_config::observability::{ ENV_OBS_LOG_STDOUT_ENABLED, ENV_OBS_LOG_ZSTD_COMPRESSION_LEVEL, ENV_OBS_LOG_ZSTD_FALLBACK_TO_GZIP, ENV_OBS_LOG_ZSTD_WORKERS, ENV_OBS_LOGGER_LEVEL, ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT, ENV_OBS_METRICS_EXPORT_ENABLED, ENV_OBS_PROFILING_ENDPOINT, ENV_OBS_PROFILING_EXPORT_ENABLED, ENV_OBS_SAMPLE_RATIO, - ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED, ENV_OBS_USE_STDOUT, + ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TLS_CA_FILE, ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED, + ENV_OBS_USE_STDOUT, }; use rustfs_config::{ APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_FILENAME, @@ -139,6 +140,9 @@ pub struct OtelConfig { pub metric_timeout_millis: Option, /// Timeout (milliseconds) for log OTLP HTTP export. pub log_timeout_millis: Option, + /// Absolute path to a PEM bundle whose certificates are added to the + /// system trust roots used by OTLP/HTTP exporters. + pub tls_ca_file: Option, /// Dedicated profiling endpoint. pub profiling_endpoint: Option, /// Whether to export distributed traces (default: `true`). @@ -292,6 +296,9 @@ impl OtelConfig { trace_timeout_millis: get_env_opt_u64(ENV_OBS_ENDPOINT_TRACES_TIMEOUT_MILLIS), metric_timeout_millis: get_env_opt_u64(ENV_OBS_ENDPOINT_METRICS_TIMEOUT_MILLIS), log_timeout_millis: get_env_opt_u64(ENV_OBS_ENDPOINT_LOGS_TIMEOUT_MILLIS), + tls_ca_file: get_env_opt_str(ENV_OBS_TLS_CA_FILE) + .map(|path| path.trim().to_string()) + .filter(|path| !path.is_empty()), profiling_endpoint: get_env_opt_str(ENV_OBS_PROFILING_ENDPOINT), traces_export_enabled: Some(get_env_bool(ENV_OBS_TRACES_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED)), metrics_export_enabled: Some(get_env_bool(ENV_OBS_METRICS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED)), @@ -509,4 +516,22 @@ mod tests { }); }); } + + #[test] + fn otlp_tls_ca_file_environment_preserves_unset_empty_and_configured_values() { + with_profiling_env_lock(|| { + temp_env::with_var_unset(ENV_OBS_TLS_CA_FILE, || { + assert_eq!(OtelConfig::extract_otel_config_from_env(None).tls_ca_file, None); + }); + temp_env::with_var(ENV_OBS_TLS_CA_FILE, Some(""), || { + assert_eq!(OtelConfig::extract_otel_config_from_env(None).tls_ca_file, None); + }); + temp_env::with_var(ENV_OBS_TLS_CA_FILE, Some("/etc/rustfs/otlp-ca.pem"), || { + assert_eq!( + OtelConfig::extract_otel_config_from_env(None).tls_ca_file.as_deref(), + Some("/etc/rustfs/otlp-ca.pem") + ); + }); + }); + } } diff --git a/crates/obs/src/error.rs b/crates/obs/src/error.rs index be43129f1..5bdf5a2ef 100644 --- a/crates/obs/src/error.rs +++ b/crates/obs/src/error.rs @@ -60,6 +60,18 @@ pub enum TelemetryError { BuildMetricExporter(String), #[error("Log exporter build failed: {0}")] BuildLogExporter(String), + #[error("RUSTFS_OBS_TLS_CA_FILE must be an absolute path")] + OtlpTlsCaPathNotAbsolute, + #[error("Failed to read RUSTFS_OBS_TLS_CA_FILE")] + OtlpTlsCaRead(#[source] std::io::Error), + #[error("RUSTFS_OBS_TLS_CA_FILE does not contain a PEM certificate")] + OtlpTlsCaEmpty, + #[error("RUSTFS_OBS_TLS_CA_FILE does not contain a valid PEM certificate")] + OtlpTlsCaInvalid, + #[error("Failed to parse RUSTFS_OBS_TLS_CA_FILE as a PEM certificate bundle")] + OtlpTlsCaParse(#[source] reqwest::Error), + #[error("Failed to build OTLP HTTP client")] + BuildOtlpHttpClient(#[source] reqwest::Error), #[error("Install metrics recorder failed: {0}")] InstallMetricsRecorder(String), #[error("Tracing subscriber init failed: {0}")] diff --git a/crates/obs/src/telemetry/otel.rs b/crates/obs/src/telemetry/otel.rs index dd297600d..9ca8a0d2e 100644 --- a/crates/obs/src/telemetry/otel.rs +++ b/crates/obs/src/telemetry/otel.rs @@ -64,7 +64,7 @@ use rustfs_config::{ DEFAULT_OBS_METRICS_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO, }; use std::collections::HashMap; -use std::{fs, io::IsTerminal, time::Duration}; +use std::{fs, io::IsTerminal, path::Path, time::Duration}; use tracing::{info, warn}; use tracing_error::ErrorLayer; use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer}; @@ -83,6 +83,12 @@ const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[ 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, ]; +const DEFAULT_OTLP_HTTP_TIMEOUT: Duration = Duration::from_secs(10); +const OTEL_EXPORTER_OTLP_TIMEOUT: &str = "OTEL_EXPORTER_OTLP_TIMEOUT"; +const OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: &str = "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT"; +const OTEL_EXPORTER_OTLP_METRICS_TIMEOUT: &str = "OTEL_EXPORTER_OTLP_METRICS_TIMEOUT"; +const OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: &str = "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT"; + #[cfg(all( feature = "pyroscope", any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")) @@ -130,6 +136,8 @@ pub(super) fn init_observability_http( logger_level: &str, is_production: bool, ) -> Result { + let otlp_tls_ca_bundle = OtlpTlsCaBundle::load(config)?; + // ── Resource & sampling ────────────────────────────────────────────────── // Build the common resource once so all enabled signals report the same // service identity and deployment metadata. @@ -153,10 +161,12 @@ pub(super) fn init_observability_http( let log_ep = resolve_signal_endpoint(config.log_endpoint.as_deref(), root_ep, "/v1/logs"); // ── Tracer provider (HTTP) ──────────────────────────────────────────────── - let tracer_provider = build_tracer_provider(&trace_ep, config, res.clone(), sampler, use_stdout)?; + let tracer_provider = + build_tracer_provider(&trace_ep, config, otlp_tls_ca_bundle.as_ref(), res.clone(), sampler, use_stdout)?; // ── Meter provider (HTTP) ───────────────────────────────────────────────── - let meter_provider = build_meter_provider(&metric_ep, config, res.clone(), &service_name, use_stdout)?; + let meter_provider = + build_meter_provider(&metric_ep, config, otlp_tls_ca_bundle.as_ref(), res.clone(), &service_name, use_stdout)?; // ── Logger Logic ────────────────────────────────────────────────────────── // Logging is the only signal that may intentionally route to either OTLP @@ -175,7 +185,7 @@ pub(super) fn init_observability_http( // Init OTLP logger logic. // We initialize the OTLP collector and honor the configured stdout setting // (e.g. via RUSTFS_OBS_USE_STDOUT / config.use_stdout) when building the provider. - logger_provider = build_logger_provider(&log_ep, config, res, use_stdout)?; + logger_provider = build_logger_provider(&log_ep, config, otlp_tls_ca_bundle.as_ref(), res, use_stdout)?; // Build bridge to capture `tracing` events. otel_bridge = logger_provider.as_ref().map(OpenTelemetryTracingBridge::new); @@ -321,6 +331,7 @@ pub(super) fn init_observability_http( fn build_tracer_provider( trace_ep: &str, config: &OtelConfig, + otlp_tls_ca_bundle: Option<&OtlpTlsCaBundle>, res: opentelemetry_sdk::Resource, sampler: Sampler, use_stdout: bool, @@ -338,9 +349,13 @@ fn build_tracer_provider( if !trace_headers.is_empty() { exporter_builder = exporter_builder.with_headers(trace_headers); } - if let Some(timeout) = resolve_signal_timeout(config.endpoint_timeout_millis, config.trace_timeout_millis) { + let timeout = resolve_signal_timeout(config.endpoint_timeout_millis, config.trace_timeout_millis); + if let Some(timeout) = timeout { exporter_builder = exporter_builder.with_timeout(timeout); } + if let Some(http_client) = build_otlp_http_client(otlp_tls_ca_bundle, timeout, OTEL_EXPORTER_OTLP_TRACES_TIMEOUT)? { + exporter_builder = exporter_builder.with_http_client(http_client); + } let exporter = exporter_builder .build() .map_err(|e| TelemetryError::BuildSpanExporter(e.to_string()))?; @@ -409,6 +424,7 @@ fn resolve_signal_endpoint(signal_endpoint: Option<&str>, root_ep: &str, suffix: fn build_meter_provider( metric_ep: &str, config: &OtelConfig, + otlp_tls_ca_bundle: Option<&OtlpTlsCaBundle>, res: opentelemetry_sdk::Resource, service_name: &str, use_stdout: bool, @@ -427,9 +443,13 @@ fn build_meter_provider( if !metric_headers.is_empty() { exporter_builder = exporter_builder.with_headers(metric_headers); } - if let Some(timeout) = resolve_signal_timeout(config.endpoint_timeout_millis, config.metric_timeout_millis) { + let timeout = resolve_signal_timeout(config.endpoint_timeout_millis, config.metric_timeout_millis); + if let Some(timeout) = timeout { exporter_builder = exporter_builder.with_timeout(timeout); } + if let Some(http_client) = build_otlp_http_client(otlp_tls_ca_bundle, timeout, OTEL_EXPORTER_OTLP_METRICS_TIMEOUT)? { + exporter_builder = exporter_builder.with_http_client(http_client); + } let exporter = exporter_builder .build() .map_err(|e| TelemetryError::BuildMetricExporter(e.to_string()))?; @@ -485,6 +505,7 @@ fn is_get_object_duration_histogram_metric(name: &str) -> bool { fn build_logger_provider( log_ep: &str, config: &OtelConfig, + otlp_tls_ca_bundle: Option<&OtlpTlsCaBundle>, res: opentelemetry_sdk::Resource, use_stdout: bool, ) -> Result, TelemetryError> { @@ -501,9 +522,13 @@ fn build_logger_provider( if !log_headers.is_empty() { exporter_builder = exporter_builder.with_headers(log_headers); } - if let Some(timeout) = resolve_signal_timeout(config.endpoint_timeout_millis, config.log_timeout_millis) { + let timeout = resolve_signal_timeout(config.endpoint_timeout_millis, config.log_timeout_millis); + if let Some(timeout) = timeout { exporter_builder = exporter_builder.with_timeout(timeout); } + if let Some(http_client) = build_otlp_http_client(otlp_tls_ca_bundle, timeout, OTEL_EXPORTER_OTLP_LOGS_TIMEOUT)? { + exporter_builder = exporter_builder.with_http_client(http_client); + } let exporter = exporter_builder .build() .map_err(|e| TelemetryError::BuildLogExporter(e.to_string()))?; @@ -660,16 +685,239 @@ fn resolve_signal_timeout(common_timeout_millis: Option, signal_timeout_mil .map(Duration::from_millis) } +struct OtlpTlsCaBundle { + certificates: Vec, +} + +impl OtlpTlsCaBundle { + fn load(config: &OtelConfig) -> Result, TelemetryError> { + let Some(ca_file) = config.tls_ca_file.as_deref() else { + return Ok(None); + }; + let path = Path::new(ca_file); + if !path.is_absolute() { + return Err(TelemetryError::OtlpTlsCaPathNotAbsolute); + } + + let pem_bundle = fs::read(path).map_err(TelemetryError::OtlpTlsCaRead)?; + if pem_bundle.iter().all(u8::is_ascii_whitespace) { + return Err(TelemetryError::OtlpTlsCaEmpty); + } + let certificates = reqwest::Certificate::from_pem_bundle(&pem_bundle).map_err(TelemetryError::OtlpTlsCaParse)?; + if certificates.is_empty() { + return Err(TelemetryError::OtlpTlsCaInvalid); + } + Ok(Some(Self { certificates })) + } + + fn build_http_client( + &self, + configured_timeout: Option, + otel_signal_timeout_env: &str, + ) -> Result { + let timeout = configured_timeout.unwrap_or_else(|| resolve_otlp_http_timeout(otel_signal_timeout_env)); + reqwest::Client::builder() + .timeout(timeout) + .tls_certs_merge(self.certificates.clone()) + .build() + .map_err(TelemetryError::BuildOtlpHttpClient) + } +} + +fn build_otlp_http_client( + otlp_tls_ca_bundle: Option<&OtlpTlsCaBundle>, + configured_timeout: Option, + otel_signal_timeout_env: &str, +) -> Result, TelemetryError> { + otlp_tls_ca_bundle + .map(|bundle| bundle.build_http_client(configured_timeout, otel_signal_timeout_env)) + .transpose() +} + +fn resolve_otlp_http_timeout(signal_timeout_env: &str) -> Duration { + [signal_timeout_env, OTEL_EXPORTER_OTLP_TIMEOUT] + .into_iter() + .find_map(|name| std::env::var(name).ok().and_then(|value| value.parse::().ok())) + .map(Duration::from_millis) + .unwrap_or(DEFAULT_OTLP_HTTP_TIMEOUT) +} + #[cfg(test)] mod tests { use super::*; use std::env; use std::io::{self, Write}; + use std::path::Path; use std::process::Command; use std::sync::{Arc, Mutex}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; const LOG_TRACER_CHILD_ENV: &str = "RUSTFS_OBS_LOG_TRACER_CHILD"; + fn config_with_tls_ca_file(path: &Path) -> OtelConfig { + OtelConfig { + tls_ca_file: Some(path.display().to_string()), + ..OtelConfig::default() + } + } + + fn load_tls_ca_bundle(path: &Path) -> OtlpTlsCaBundle { + OtlpTlsCaBundle::load(&config_with_tls_ca_file(path)) + .expect("load CA bundle") + .expect("configured CA file should produce a bundle") + } + + async fn spawn_test_tls_server() -> (String, String, tokio::task::JoinHandle) { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let certified = + rcgen::generate_simple_self_signed(vec!["127.0.0.1".to_string()]).expect("generate TLS server certificate"); + let ca_pem = certified.cert.pem(); + let private_key = rustls_pki_types::PrivateKeyDer::try_from(certified.signing_key.serialize_der()) + .expect("convert TLS server private key"); + let config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(vec![certified.cert.der().clone()], private_key) + .expect("build TLS server config"); + let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config)); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind TLS test server"); + let endpoint = format!("https://{}", listener.local_addr().expect("read TLS test server address")); + let server = tokio::spawn(async move { + let Ok((stream, _)) = listener.accept().await else { + return false; + }; + let Ok(mut stream) = acceptor.accept(stream).await else { + return false; + }; + let mut request = Vec::new(); + let mut buffer = [0_u8; 1_024]; + loop { + let Ok(read) = stream.read(&mut buffer).await else { + return false; + }; + if read == 0 { + return false; + } + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + stream + .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\nok") + .await + .is_ok() + && stream.shutdown().await.is_ok() + }); + (endpoint, ca_pem, server) + } + + #[tokio::test] + async fn otlp_http_client_trusts_every_certificate_in_a_pem_bundle() { + let (first_endpoint, first_ca_pem, first_server) = spawn_test_tls_server().await; + let (second_endpoint, second_ca_pem, second_server) = spawn_test_tls_server().await; + let file = tempfile::NamedTempFile::new().expect("create CA bundle file"); + std::fs::write(file.path(), format!("{first_ca_pem}\n{second_ca_pem}")).expect("write CA bundle"); + + let bundle = load_tls_ca_bundle(file.path()); + let client = build_otlp_http_client(Some(&bundle), Some(Duration::from_millis(250)), OTEL_EXPORTER_OTLP_TRACES_TIMEOUT) + .expect("build client with CA bundle"); + let client = client.expect("custom CA should build a client"); + assert_eq!( + client + .get(first_endpoint) + .send() + .await + .expect("first bundled CA should complete TLS handshake") + .status(), + reqwest::StatusCode::OK + ); + assert_eq!( + client + .get(second_endpoint) + .send() + .await + .expect("second bundled CA should complete TLS handshake") + .status(), + reqwest::StatusCode::OK + ); + assert!(first_server.await.expect("first bundled TLS server task")); + assert!(second_server.await.expect("second bundled TLS server task")); + } + + #[test] + fn otlp_http_client_rejects_relative_missing_empty_and_invalid_ca_files() { + let relative = OtelConfig { + tls_ca_file: Some("otlp-ca.pem".to_string()), + ..OtelConfig::default() + }; + assert!(matches!(OtlpTlsCaBundle::load(&relative), Err(TelemetryError::OtlpTlsCaPathNotAbsolute))); + + let missing_directory = tempfile::tempdir().expect("create missing CA directory"); + let missing = OtelConfig { + tls_ca_file: Some(missing_directory.path().join("otlp-ca.pem").display().to_string()), + ..OtelConfig::default() + }; + assert!(matches!(OtlpTlsCaBundle::load(&missing), Err(TelemetryError::OtlpTlsCaRead(_)))); + + let empty = tempfile::NamedTempFile::new().expect("create empty CA file"); + assert!(matches!( + OtlpTlsCaBundle::load(&config_with_tls_ca_file(empty.path())), + Err(TelemetryError::OtlpTlsCaEmpty) + )); + + let invalid = tempfile::NamedTempFile::new().expect("create invalid CA file"); + std::fs::write(invalid.path(), b"not a PEM certificate").expect("write invalid CA file"); + assert!(matches!( + OtlpTlsCaBundle::load(&config_with_tls_ca_file(invalid.path())), + Err(TelemetryError::OtlpTlsCaInvalid) + )); + } + + #[tokio::test] + async fn otlp_http_client_trusts_the_configured_ca_and_rejects_another_ca() { + let (endpoint, ca_pem, trusted_server) = spawn_test_tls_server().await; + let trusted_file = tempfile::NamedTempFile::new().expect("create trusted CA file"); + std::fs::write(trusted_file.path(), ca_pem).expect("write trusted CA file"); + let trusted_bundle = load_tls_ca_bundle(trusted_file.path()); + let trusted_client = + build_otlp_http_client(Some(&trusted_bundle), Some(Duration::from_secs(1)), OTEL_EXPORTER_OTLP_TRACES_TIMEOUT) + .expect("build trusted client") + .expect("custom CA should build a client"); + assert_eq!( + trusted_client + .get(&endpoint) + .send() + .await + .expect("configured CA should complete TLS handshake") + .status(), + reqwest::StatusCode::OK + ); + assert!(trusted_server.await.expect("trusted TLS server task")); + + let (endpoint, _ca_pem, untrusted_server) = spawn_test_tls_server().await; + let wrong_ca = + rcgen::generate_simple_self_signed(vec!["wrong.test".to_string()]).expect("generate unrelated CA certificate"); + let untrusted_file = tempfile::NamedTempFile::new().expect("create unrelated CA file"); + std::fs::write(untrusted_file.path(), wrong_ca.cert.pem()).expect("write unrelated CA file"); + let untrusted_bundle = load_tls_ca_bundle(untrusted_file.path()); + let untrusted_client = + build_otlp_http_client(Some(&untrusted_bundle), Some(Duration::from_secs(1)), OTEL_EXPORTER_OTLP_TRACES_TIMEOUT) + .expect("build untrusted client") + .expect("custom CA should build a client"); + assert!(untrusted_client.get(&endpoint).send().await.is_err()); + assert!(!untrusted_server.await.expect("untrusted TLS server task")); + } + + #[test] + fn otlp_http_client_is_not_built_without_a_custom_ca_file() { + assert!( + build_otlp_http_client(None, None, OTEL_EXPORTER_OTLP_TRACES_TIMEOUT) + .expect("no CA file keeps the default client") + .is_none() + ); + } + #[derive(Clone)] struct TestWriter(Arc>>); From 9c30cc88513c5e1b5443b6ee86bf4bc7965e79e9 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:22:18 +0800 Subject: [PATCH 30/30] feat(ci): on-demand fault-tolerance matrix sweep workflow (#8058) Adds rustfs-fault-tolerance-matrix.yml, a manually dispatched workflow that runs the --matrix topology x EC outage sweep from rustfs/auto-testing (38 combinations, 320 cases). The scenario suite hardcodes --all and a 60-minute budget, so the exhaustive sweep had no CI entry point. --- .../rustfs-fault-tolerance-matrix.yml | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 .github/workflows/rustfs-fault-tolerance-matrix.yml diff --git a/.github/workflows/rustfs-fault-tolerance-matrix.yml b/.github/workflows/rustfs-fault-tolerance-matrix.yml new file mode 100644 index 000000000..ffd216cc0 --- /dev/null +++ b/.github/workflows/rustfs-fault-tolerance-matrix.yml @@ -0,0 +1,221 @@ +# RustFS Fault-Tolerance (degradation) Matrix Sweep +# +# Exhaustive topology x EC outage sweep, complementary to the scenario suite +# (rustfs-fault-tolerance-test.yml). For each topology (nodes x drives per +# node) and each EC level P in 1..N*drives/2 the runner installs that exact +# cluster shape and drives progressive node outages plus recovery: +# +# M-EC

-full full-capacity reads/writes must succeed +# M-EC

-n with k nodes online: reads succeed iff the online +# drives cover the read quorum (N-P), writes iff they +# cover the write quorum (N-P+1); probes below quorum +# assert the expected outage +# M-EC

-rec after restoring every node, reads/writes succeed +# +# 10 topologies x EC levels = 38 combinations, 320 cases. The full sweep runs +# 3-5 hours and is dispatched on demand (e.g. after a nightly chain goes +# green), not nightly. Case ids and expectations come from product source +# (default_parity_count, erasure set sizing) via rustfs/auto-testing. + +name: RustFS Fault-Tolerance Matrix + +on: + workflow_dispatch: + inputs: + package_url: + description: 'Direct .deb URL. Required unless the nightly default is wanted.' + required: false + type: string + auto_testing_ref: + description: 'auto-testing branch, tag, or SHA (defaults to main)' + required: false + default: main + type: string + matrix_topo: + description: 'Topology filter like 4x2; empty sweeps all ten topologies' + required: false + default: '' + type: string + matrix_ec: + description: 'EC/parity filter; empty sweeps every EC level' + required: false + default: '' + type: string + cleanup_before: + description: 'Reset the nodes before the test (DESTROYS existing data/config)' + type: boolean + default: true + cleanup_after: + description: 'Reset the nodes after the test (DESTROYS test data/config)' + type: boolean + default: true + +permissions: + contents: read + +# The suite stops services and revokes drive access on the shared fleet; only one +# functional suite may touch the environment at a time. +concurrency: + group: rustfs-shared-functional-tests-v2 + cancel-in-progress: false + +defaults: + run: + shell: bash + +env: + RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }} + RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }} + RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }} + RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }} + PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} + +jobs: + fault-tolerance-matrix: + runs-on: smoke-testing + # The full 38-combination sweep takes 3-5h; 360 is the runner job ceiling. + timeout-minutes: 360 + steps: + - name: Initialize functional evidence + id: evidence + run: | + set -euo pipefail + umask 077 + FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-fault-tolerance-matrix-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch" "${FUNCTIONAL_ARTIFACTS_DIR}/evidence" + { + printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'RESULTS_FILE=%s/results.json\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + } >> "${GITHUB_ENV}" + + - name: Checkout auto-testing scripts + timeout-minutes: 5 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + repository: rustfs/auto-testing + ref: ${{ inputs.auto_testing_ref }} + token: ${{ secrets.PF_TESTING_GH_TOKEN }} + path: auto-testing + persist-credentials: false + + - name: Record auto-testing revision + run: | + set -euo pipefail + AUTO_TESTING_SHA="$(git -C auto-testing rev-parse HEAD)" + printf 'AUTO_TESTING_SHA=%s\n' "${AUTO_TESTING_SHA}" >> "${GITHUB_ENV}" + echo "auto-testing revision: ${AUTO_TESTING_SHA}" + + - name: Show environment + run: | + uname -a + jq --version + aws --version + echo "auto-testing: ${AUTO_TESTING_SHA}" + df -h /data | tail -1 + + - name: Cleanup environment (before) + timeout-minutes: 5 + if: ${{ inputs.cleanup_before != 'false' }} + run: | + ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" + + - name: Run matrix sweep (topology x EC outage combinations) + timeout-minutes: 350 + id: test + # No continue-on-error: an UNEXPECTED verdict (a probe that should + # fail by quorum math but passes, or vice versa) is the finding this + # sweep exists to surface, and the red run is the signal until matrix + # runs get backlog integration like the scenario suite has. + env: + MATRIX_EC: ${{ inputs.matrix_ec }} + run: | + set -euo pipefail + ARGS=(--matrix -y --package-url "${{ inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}" --results-file "${RESULTS_FILE}") + if [ -n "${{ inputs.matrix_topo }}" ]; then + # Trailing value form: consumed as `--matrix-topo [ec]`. + ARGS+=(--matrix-topo "${{ inputs.matrix_topo }}") + if [ -n "${MATRIX_EC}" ]; then + ARGS+=("${MATRIX_EC}") + fi + fi + ./auto-testing/rustfs-fault-tolerance-test.sh "${ARGS[@]}" + + - name: Generate report + timeout-minutes: 2 + if: ${{ always() && steps.evidence.outcome == 'success' }} + run: | + set -euo pipefail + { + echo "# RustFS fault-tolerance matrix report" + echo "" + echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}" + echo "- auto-testing: \`${AUTO_TESTING_SHA}\`" + echo "- Topology filter: ${{ inputs.matrix_topo || 'all (10 topologies)' }}" + echo "- EC filter: ${{ inputs.matrix_ec || 'all levels' }}" + echo "" + echo "## Per-probe results" + echo "" + echo '```' + grep -E '^FT-(CASE|COMPLETE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)" + echo '```' + echo "" + echo "## Health snapshots" + echo "" + for f in "${FUNCTIONAL_ARTIFACTS_DIR}"/evidence/*.code; do + [ -e "${f}" ] || continue + printf '%s -> %s\n' "$(basename "${f}" .code)" "$(cat "${f}")" + done + } > "${REPORT_FILE}" + cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + + # An unfiltered sweep must land exactly 320 cases (38 topology x EC + # combinations at 2*(nodes+1) cases each); filtered sweeps are held + # to internal consistency. Missing/harness-error rows always fail. + EXPECTED_TOTAL="" + if [ -z "${{ inputs.matrix_topo }}" ] && [ -z "${MATRIX_EC}" ]; then + EXPECTED_TOTAL=320 + fi + jq -e --arg testing_sha "${AUTO_TESTING_SHA}" --argjson expected_total "${EXPECTED_TOTAL:-null}" ' + .schema_version == 1 and + .auto_testing_sha == $testing_sha and + .complete == true and + .counts.expected == ($expected_total // .counts.expected) and + .counts.expected == .counts.seen and + .counts.harness_errors == 0 and + (.expected_cases | length) == .counts.expected and + (.expected_cases | unique | length) == .counts.expected and + (.seen_cases | length) == .counts.seen and + (.missing_cases | length) == 0 and + (.harness_errors | length) == 0 and + (.cases | length) == .counts.seen and + ([.cases[].case_id] | unique | length) == .counts.seen and + ([.cases[].case_id] | sort) == (.expected_cases | sort) and + all(.cases[]; (.case_id | startswith("M-")) and (.verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED")) and + .counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and + .counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length) + ' "${RESULTS_FILE}" >/dev/null + + - name: Upload test logs & evidence + timeout-minutes: 2 + if: ${{ always() && steps.evidence.outcome == 'success' }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: rustfs-fault-tolerance-matrix-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results.json + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/ + if-no-files-found: warn + + - name: Cleanup environment (after) + timeout-minutes: 5 + if: ${{ always() && inputs.cleanup_after != 'false' }} + run: | + ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true