From 89e25132055efa3f316b237b404966cc52364fdb Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 15:04:23 +0800 Subject: [PATCH 01/22] feat(ecstore): pin bitrot algorithms with a startup self-test (HS-11) (#6165) feat(ecstore): pin bitrot algorithms with a startup self-test A drifted HighwayHash implementation fails silently: every shard reads back corrupt, heal rewrites healthy data, and cross-platform clusters disagree about which copy is good. Mirror MinIO's bitrotSelfTest by verifying, once at process start: - known-answer digests for HighwayHash256S / HighwayHash256SLegacy over a deterministic 4096-byte xorshift64* payload, plus the externally verifiable FIPS SHA-256 "abc" vector guarding the HashAlgorithm plumbing itself; - an end-to-end roundtrip per streaming variant (encode -> size formula -> bitrot_verify -> BitrotReader read-back), over full blocks and a partial tail; - tamper detection: one flipped byte in the final data block and one in the leading hash must both be rejected as a hash mismatch, not by an incidental read error. The check costs microseconds and runs inline in init_background_service_runtime before any shard can be written or verified. Outcome surfaces as one structured bitrot_selftest log event, the rustfs_bitrot_selftest_status gauge (1=passed / 0=failed / 2=skipped), a bitrotSelftest field on the admin server-info response, and RUSTFS_BITROT_SELFTEST_STRICT=on turns a failure into a startup error (MinIO Fatal parity; the default only degrades the status so a bad build cannot brick an existing fleet on upgrade). Closes rustfs/backlog#1873 (HS-11). Co-authored-by: heihutu --- crates/ecstore/src/api/mod.rs | 4 +- crates/ecstore/src/erasure/coding/bitrot.rs | 303 +++++++++++++++++++- rustfs/src/admin/handlers/system.rs | 29 ++ rustfs/src/bitrot_selftest.rs | 181 ++++++++++++ rustfs/src/lib.rs | 1 + rustfs/src/module_switches.rs | 14 + rustfs/src/startup_background.rs | 11 +- rustfs/src/storage/storage_api.rs | 4 + rustfs/src/storage_api.rs | 4 +- 9 files changed, 535 insertions(+), 16 deletions(-) create mode 100644 rustfs/src/bitrot_selftest.rs diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index f18da095a..5526b7d00 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -373,8 +373,8 @@ pub mod error { pub mod erasure { pub use crate::erasure::coding::{ - BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, ReedSolomonEncoder, - calc_shard_size, calc_shard_size_legacy, + BitrotReader, BitrotSelfTestError, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, + ReedSolomonEncoder, bitrot_self_test, calc_shard_size, calc_shard_size_legacy, }; } diff --git a/crates/ecstore/src/erasure/coding/bitrot.rs b/crates/ecstore/src/erasure/coding/bitrot.rs index 7b3166fbc..698b1b02c 100644 --- a/crates/ecstore/src/erasure/coding/bitrot.rs +++ b/crates/ecstore/src/erasure/coding/bitrot.rs @@ -820,10 +820,263 @@ impl BitrotWriterWrapper { } } +// --- startup bitrot self-test (rustfs/backlog#1873, MinIO bitrotSelfTest parity) --- +// +// A broken hash implementation (bad SIMD feature combination, platform drift, a +// key-handling regression) fails silently: every shard reads back "corrupt", +// heal rewrites data that was fine, and cross-platform clusters disagree about +// which copy is healthy. The self-test below pins the algorithms the moment a +// process starts, so a drifted build announces itself instead of quietly +// rewriting objects. See docs/rustfs-heal-scanner-vs-minio-comprehensive- +// analysis-2026-08-16.md §6 HS-11. + +/// Length of the deterministic self-test payload. +pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096; + +/// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S` +/// (the production default). Pinned so any platform or build where the +/// implementation drifts fails startup instead of mis-hashing shards. +const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [ + 0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea, + 0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06, +]; + +/// Known-answer digest of the same payload under `HighwayHash256SLegacy`. +const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY: [u8; 32] = [ + 0x98, 0x24, 0x71, 0x4f, 0x16, 0xbb, 0x48, 0x39, 0xed, 0x68, 0xfa, 0x63, 0x5e, 0xd9, 0x07, 0x61, 0xdf, 0x0a, 0xff, 0xcf, 0x7d, + 0x8c, 0xa8, 0xc7, 0xc0, 0xb6, 0x6f, 0x05, 0xdb, 0xda, 0x5a, 0x22, +]; + +/// FIPS 180-2 test vector: SHA-256 of the ASCII string "abc". Unlike the +/// Highway digests above this one is externally verifiable, so it guards the +/// whole `HashAlgorithm` plumbing even for readers who distrust pinned +/// self-computed constants. +const BITROT_SELF_TEST_KAT_SHA256_ABC: [u8; 32] = [ + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, + 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad, +]; + +/// Deterministic self-test payload: xorshift64* from a fixed seed, so every +/// platform and every run hashes the same 4096 bytes. +fn bitrot_self_test_payload() -> [u8; BITROT_SELF_TEST_PAYLOAD_LEN] { + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let mut payload = [0u8; BITROT_SELF_TEST_PAYLOAD_LEN]; + for byte in payload.iter_mut() { + state ^= state >> 12; + state ^= state << 25; + state ^= state >> 27; + *byte = state.wrapping_mul(0x2545_F491_4F6C_DD1D) as u8; + } + payload +} + +/// Why a bitrot self-test failed. +#[derive(Debug)] +pub enum BitrotSelfTestError { + /// A known-answer digest mismatched the pinned constant. + KnownAnswerMismatch { + algorithm: &'static str, + got: String, + want: String, + }, + /// A freshly encoded shard failed `bitrot_verify`. + RoundtripVerify { algorithm: &'static str, detail: String }, + /// A verified roundtrip read back different bytes than were written. + RoundtripReadback { algorithm: &'static str }, + /// A deliberately tampered shard was not rejected by `bitrot_verify`. + TamperNotRejected { + algorithm: &'static str, + tampered: &'static str, + }, +} + +impl std::fmt::Display for BitrotSelfTestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::KnownAnswerMismatch { algorithm, got, want } => { + write!(f, "known-answer mismatch for {algorithm}: got {got}, want {want}") + } + Self::RoundtripVerify { algorithm, detail } => write!(f, "{algorithm} roundtrip shard failed verification: {detail}"), + Self::RoundtripReadback { algorithm } => write!(f, "{algorithm} roundtrip read back different bytes"), + Self::TamperNotRejected { algorithm, tampered } => { + write!(f, "{algorithm} tampered shard ({tampered}) was not rejected") + } + } + } +} + +impl std::error::Error for BitrotSelfTestError {} + +fn self_test_hex(bytes: &[u8]) -> String { + rustfs_utils::hex(bytes) +} + +// (kept as a named one-liner so every KAT failure site reads the same; the +// underlying formatter is the shared `rustfs_utils::hex`) + +/// Compare a digest against its pinned constant. Split out so a test can drive +/// it with a wrong constant and prove the mismatch path fires. +fn bitrot_kat_check( + algorithm: &'static str, + algo: &HashAlgorithm, + payload: &[u8], + expected: &[u8; 32], +) -> Result<(), BitrotSelfTestError> { + let digest = algo.hash_encode(payload); + let digest = digest.as_ref(); + if digest.len() != expected.len() || digest != expected.as_slice() { + return Err(BitrotSelfTestError::KnownAnswerMismatch { + algorithm, + got: self_test_hex(digest), + want: self_test_hex(expected), + }); + } + Ok(()) +} + +/// Encode `payload` with `shard_size` blocks, verify it end to end, and read +/// every block back through `BitrotReader` comparing bytes. +async fn bitrot_roundtrip_check( + algorithm: &'static str, + algo: HashAlgorithm, + payload: &[u8], + shard_size: usize, +) -> Result<(), BitrotSelfTestError> { + let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::::new()), shard_size, algo.clone()); + for chunk in payload.chunks(shard_size) { + writer + .write(chunk) + .await + .map_err(|err| BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: format!("encode failed: {err}"), + })?; + } + let encoded = writer.into_inner().into_inner(); + + let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone()); + if encoded.len() != on_disk { + return Err(BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: format!("encoded {} bytes, size formula says {on_disk}", encoded.len()), + }); + } + bitrot_verify(std::io::Cursor::new(encoded.clone()), on_disk, payload.len(), algo.clone(), shard_size) + .await + .map_err(|err| BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: err.to_string(), + })?; + + let mut reader = BitrotReader::new(std::io::Cursor::new(encoded), shard_size, algo, false); + let mut offset = 0usize; + while offset < payload.len() { + let want = shard_size.min(payload.len() - offset); + let mut buf = vec![0u8; want]; + let read = reader + .read(&mut buf) + .await + .map_err(|err| BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: format!("read back failed at offset {offset}: {err}"), + })?; + if read != want || buf[..read] != payload[offset..offset + read] { + return Err(BitrotSelfTestError::RoundtripReadback { algorithm }); + } + offset += read; + } + Ok(()) +} + +/// Flip one byte and require `bitrot_verify` to reject the result. +async fn bitrot_tamper_check( + algorithm: &'static str, + algo: HashAlgorithm, + payload: &[u8], + shard_size: usize, + tampered: &'static str, + flip_at: usize, +) -> Result<(), BitrotSelfTestError> { + let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::::new()), shard_size, algo.clone()); + for chunk in payload.chunks(shard_size) { + writer.write(chunk).await.expect("self-test encode should not fail"); + } + let mut corrupt = writer.into_inner().into_inner(); + let flip_index = flip_at % corrupt.len(); + corrupt[flip_index] ^= 0x80; + + let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone()); + match bitrot_verify(std::io::Cursor::new(corrupt), on_disk, payload.len(), algo, shard_size).await { + // The flipped byte must be rejected as a hash mismatch specifically, not + // by any incidental read error: an in-memory cursor cannot fail reads, + // so accepting any other failure here would mask a verify path that + // errors out before it ever compares hashes. + Err(err) if err.to_string().contains("hash mismatch") => Ok(()), + Ok(()) => Err(BitrotSelfTestError::TamperNotRejected { algorithm, tampered }), + Err(err) => Err(BitrotSelfTestError::RoundtripVerify { + algorithm, + detail: format!("tampered shard rejected with an unexpected error: {err}"), + }), + } +} + +/// Verify every bitrot algorithm this crate can write or verify in production: +/// both streaming Highway variants roundtrip end to end (encode → size formula +/// → `bitrot_verify` → read back) and reject a flipped byte in both the data +/// and the leading hash, while all three hashed algorithms reproduce their +/// pinned known-answer digests. +/// +/// Runs in well under a millisecond on 4 KiB of data; callers may run it inline +/// at startup. Pure CPU, no allocation beyond a few KiB of scratch. +pub async fn bitrot_self_test() -> Result<(), BitrotSelfTestError> { + let payload = bitrot_self_test_payload(); + + // Externally verifiable vector first: it guards the HashAlgorithm plumbing + // itself, before any self-pinned constants are consulted. + let abc = HashAlgorithm::SHA256.hash_encode(b"abc"); + if abc.as_ref() != BITROT_SELF_TEST_KAT_SHA256_ABC.as_slice() { + return Err(BitrotSelfTestError::KnownAnswerMismatch { + algorithm: "SHA256", + got: self_test_hex(abc.as_ref()), + want: self_test_hex(&BITROT_SELF_TEST_KAT_SHA256_ABC), + }); + } + + bitrot_kat_check( + "HighwayHash256S", + &HashAlgorithm::HighwayHash256S, + &payload, + &BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S, + )?; + bitrot_kat_check( + "HighwayHash256SLegacy", + &HashAlgorithm::HighwayHash256SLegacy, + &payload, + &BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY, + )?; + + for (algorithm, algo) in [ + ("HighwayHash256S", HashAlgorithm::HighwayHash256S), + ("HighwayHash256SLegacy", HashAlgorithm::HighwayHash256SLegacy), + ] { + // Full blocks plus a partial tail, exactly like a real part stripe. + let tail_len = 2 * 1024 + 333; + bitrot_roundtrip_check(algorithm, algo.clone(), &payload, 1024).await?; + bitrot_roundtrip_check(algorithm, algo.clone(), &payload[..tail_len], 1024).await?; + // One flipped byte in the final data block, one in the first leading + // hash: both must fail verification. + bitrot_tamper_check(algorithm, algo.clone(), &payload, 1024, "final data byte", payload.len() - 1).await?; + bitrot_tamper_check(algorithm, algo, &payload, 1024, "leading hash byte", 0).await?; + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::{ - BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_shard_file_size, bitrot_verify, write_all_vectored, + BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_kat_check, bitrot_self_test, + bitrot_self_test_payload, bitrot_shard_file_size, bitrot_verify, write_all_vectored, }; use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource}; use bytes::Bytes; @@ -1090,6 +1343,32 @@ mod tests { } } + #[test] + fn bitrot_self_test_payload_is_deterministic() { + // Two independent builds of the payload must agree byte for byte, or + // the pinned known-answer digests below would be meaningless. + assert_eq!(bitrot_self_test_payload(), bitrot_self_test_payload()); + } + + #[test] + fn bitrot_self_test_rejects_a_wrong_known_answer_digest() { + let payload = bitrot_self_test_payload(); + let wrong = [0u8; 32]; + let err = bitrot_kat_check("HighwayHash256S", &HashAlgorithm::HighwayHash256S, &payload, &wrong) + .expect_err("a zeroed digest must never match"); + match err { + super::BitrotSelfTestError::KnownAnswerMismatch { algorithm, .. } => assert_eq!(algorithm, "HighwayHash256S"), + other => panic!("expected KnownAnswerMismatch, got {other:?}"), + } + } + + #[tokio::test] + async fn bitrot_self_test_passes() { + bitrot_self_test() + .await + .expect("the pinned digests and roundtrip checks must all pass on this platform"); + } + #[tokio::test] async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() { let mut counting = VectoredCountingWriter::default(); @@ -1189,7 +1468,7 @@ mod tests { let last = corrupt.len() - 1; corrupt[last] ^= 0x80; let err = bitrot_verify( - Cursor::new(corrupt), + std::io::Cursor::new(corrupt), super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()), data.len(), algo, @@ -1282,7 +1561,7 @@ mod tests { #[tokio::test] async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() { - let mut reader = BitrotReader::new(Cursor::new(Vec::::new()), 4, HashAlgorithm::None, false); + let mut reader = BitrotReader::new(std::io::Cursor::new(Vec::::new()), 4, HashAlgorithm::None, false); let mut out = [0u8; 5]; let err = reader .read(&mut out) @@ -1407,7 +1686,7 @@ mod tests { (HashAlgorithm::HighwayHash256, true), ] { let label = format!("{algo:?}"); - let writer = Cursor::new(Vec::::new()); + let writer = std::io::Cursor::new(Vec::::new()); let mut w = BitrotWriter::new(writer, shard_size, algo.clone()); w.write(&[7u8; 16]).await.unwrap(); let written = w.into_inner().into_inner(); @@ -1492,7 +1771,7 @@ mod tests { } async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec { - let mut w = BitrotWriter::new(Cursor::new(Vec::::new()), shard_size, algo); + let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::::new()), shard_size, algo); w.write(payload).await.unwrap(); w.into_inner().into_inner() } @@ -1600,7 +1879,7 @@ mod tests { for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] { for &size in &[1usize, 16, 17, 32, 40, 48] { let payload: Vec = (0..size).map(|i| i as u8).collect(); - let mut w = BitrotWriter::new(Cursor::new(Vec::::new()), shard_size, algo.clone()); + let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::::new()), shard_size, algo.clone()); for chunk in payload.chunks(shard_size) { w.write(chunk).await.unwrap(); } @@ -1674,14 +1953,14 @@ mod tests { w.write(&data).await.expect("write shard"); let mut via_read = vec![0u8; SHARD]; - let n1 = BitrotReader::new(Cursor::new(encoded.clone()), SHARD, algo.clone(), false) + let n1 = BitrotReader::new(std::io::Cursor::new(encoded.clone()), SHARD, algo.clone(), false) .read(&mut via_read) .await .expect("read"); // A buffer with only capacity — no initialized bytes at all. let mut via_append: Vec = Vec::with_capacity(SHARD); - let n2 = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false) + let n2 = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false) .read_appending(&mut via_append, SHARD) .await .expect("read_appending"); @@ -1706,7 +1985,7 @@ mod tests { encoded.truncate(encoded.len() - 1); let mut out: Vec = Vec::with_capacity(SHARD); - let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false) + let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false) .read_appending(&mut out, SHARD) .await .expect_err("a truncated shard must not succeed"); @@ -1732,7 +2011,7 @@ mod tests { encoded[last] ^= 0xff; let mut out: Vec = Vec::with_capacity(SHARD); - let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo, false) + let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false) .read_appending(&mut out, SHARD) .await .expect_err("a corrupt shard must not verify"); @@ -1844,7 +2123,7 @@ mod tests { "Cursor must be able to hand out a block, otherwise the fast path is dead code" ); assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length"); - let mut streamed = Cursor::new(encoded.clone()); + let mut streamed = std::io::Cursor::new(encoded.clone()); assert!( ShardSource::try_take_block(&mut streamed, 8).is_none(), "a non-Bytes source must stay on the streaming path" @@ -1872,7 +2151,7 @@ mod tests { ); let mut via_stream: Vec = Vec::with_capacity(SHARD); - BitrotReader::new(Cursor::new(encoded), SHARD, algo, false) + BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false) .read_appending(&mut via_stream, SHARD) .await .expect("streaming read"); diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index 96054ada5..c7ad93b54 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -417,6 +417,13 @@ struct SystemAdminDiscovery { struct ServerInfoResponse { info: InfoMessage, admin_discovery: SystemAdminDiscovery, + /// Startup bitrot algorithm self-test outcome (rustfs/backlog#1873): + /// `passed` (algorithms verified at boot), `failed` (a drifted hash + /// implementation — the process is serving with degraded integrity + /// checking unless `RUSTFS_BITROT_SELFTEST_STRICT` aborted it), or + /// `unknown` (not yet run or disabled). + #[serde(rename = "bitrotSelftest")] + bitrot_selftest: &'static str, } #[derive(Serialize)] @@ -433,6 +440,14 @@ fn system_admin_discovery(usecase: &DefaultAdminUsecase) -> SystemAdminDiscovery } } +fn bitrot_selftest_status_str() -> &'static str { + match crate::bitrot_selftest::bitrot_selftest_passed() { + Some(true) => "passed", + Some(false) => "failed", + None => "unknown", + } +} + #[async_trait::async_trait] impl Operation for ServerInfoHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { @@ -464,6 +479,7 @@ impl Operation for ServerInfoHandler { let response = ServerInfoResponse { info, admin_discovery: system_admin_discovery(&usecase), + bitrot_selftest: bitrot_selftest_status_str(), }; let data = serde_json::to_vec(&response).map_err(|e| { @@ -1535,6 +1551,18 @@ mod tests { ); } + /// The startup bitrot self-test outcome must surface in server info as one + /// of three closed-set strings, never an internal enum or a null + /// (rustfs/backlog#1873). This test pins the string mapping; whether the + /// process-global cell holds Some(true)/Some(false)/None is owned by + /// `crate::bitrot_selftest`'s own tests. + #[test] + fn bitrot_selftest_status_str_is_a_closed_set_of_operators_strings() { + let rendered = super::bitrot_selftest_status_str(); + assert!(matches!(rendered, "passed" | "failed" | "unknown")); + assert_eq!(super::bitrot_selftest_status_str(), rendered); + } + #[test] fn server_info_response_exposes_admin_discovery_paths() { let usecase = DefaultAdminUsecase::without_context(); @@ -1556,6 +1584,7 @@ mod tests { pools: None, }, admin_discovery: system_admin_discovery(&usecase), + bitrot_selftest: super::bitrot_selftest_status_str(), }; let value = serde_json::to_value(response).expect("server info response should serialize"); diff --git a/rustfs/src/bitrot_selftest.rs b/rustfs/src/bitrot_selftest.rs new file mode 100644 index 000000000..4b8c1d1c4 --- /dev/null +++ b/rustfs/src/bitrot_selftest.rs @@ -0,0 +1,181 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Startup bitrot algorithm self-test (rustfs/backlog#1873). +//! +//! A drifted hash implementation fails silently in production: every shard +//! reads back "corrupt", heal rewrites healthy data, and cross-platform +//! clusters disagree about which copy is good. [`run_startup_bitrot_self_test`] +//! pins the algorithms once at process start — the check itself runs in well +//! under a millisecond on 4 KiB, so it executes inline before background +//! services come up and the result is published before the server accepts +//! traffic. +//! +//! Outcome surface: +//! - one structured `bitrot_selftest` log event (`passed`/`failed`/`skipped`), +//! - the `rustfs_bitrot_selftest_status` gauge (1=passed, 0=failed, 2=skipped), +//! - [`bitrot_selftest_passed`] for admin/health surfaces, +//! - `RUSTFS_BITROT_SELFTEST_STRICT=on` turns a failure into a startup error +//! (MinIO `bitrotSelfTest` Fatal parity); the default only degrades the +//! status so a bad build cannot brick an existing fleet on upgrade. + +use crate::storage_api::startup::background::{BitrotSelfTestError, bitrot_self_test}; +use metrics::gauge; +use std::future::Future; +use std::io; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::time::Instant; +use tracing::{debug, error, info}; + +const LOG_COMPONENT_MAIN: &str = "main"; +const LOG_SUBSYSTEM_STARTUP: &str = "startup"; +const EVENT_BITROT_SELFTEST: &str = "bitrot_selftest"; +const METRIC_BITROT_SELFTEST_STATUS: &str = "rustfs_bitrot_selftest_status"; + +/// Gauge values for [`METRIC_BITROT_SELFTEST_STATUS`]. +const STATUS_PASSED: f64 = 1.0; +const STATUS_FAILED: f64 = 0.0; +const STATUS_SKIPPED: f64 = 2.0; + +/// Internal cell values for [`BITROT_SELF_TEST_STATUS`]. +const STATUS_CELL_UNSET: u8 = 0; +const STATUS_CELL_PASSED: u8 = 1; +const STATUS_CELL_FAILED: u8 = 2; + +static BITROT_SELF_TEST_STATUS: AtomicU8 = AtomicU8::new(STATUS_CELL_UNSET); + +/// Last recorded self-test outcome: `None` before the first run, then +/// `Some(true)` on a passing check and `Some(false)` on a failed one (a +/// skipped check never publishes, so it cannot read as a pass). The cell is +/// last-writer-wins rather than set-once: production runs the self-test once, +/// and last-writer-wins keeps tests that exercise both outcomes +/// order-independent. +pub fn bitrot_selftest_passed() -> Option { + match BITROT_SELF_TEST_STATUS.load(Ordering::Acquire) { + STATUS_CELL_UNSET => None, + STATUS_CELL_PASSED => Some(true), + STATUS_CELL_FAILED => Some(false), + _ => None, + } +} + +/// Run the bitrot self-test and publish the outcome. In strict mode a failure +/// is returned as an error so the caller aborts startup. +pub(crate) async fn run_startup_bitrot_self_test(enabled: bool, strict: bool) -> io::Result<()> { + run_startup_bitrot_self_test_with(enabled, strict, bitrot_self_test).await +} + +async fn run_startup_bitrot_self_test_with(enabled: bool, strict: bool, run_check: F) -> io::Result<()> +where + F: FnOnce() -> Fut, + Fut: Future>, +{ + if !enabled { + gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_SKIPPED); + debug!( + target: "rustfs::main::run", + event = EVENT_BITROT_SELFTEST, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "skipped", + reason = "disabled", + "Bitrot self-test skipped" + ); + return Ok(()); + } + + let started = Instant::now(); + match run_check().await { + Ok(()) => { + BITROT_SELF_TEST_STATUS.store(STATUS_CELL_PASSED, Ordering::Release); + gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_PASSED); + info!( + target: "rustfs::main::run", + event = EVENT_BITROT_SELFTEST, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "passed", + duration_us = started.elapsed().as_micros() as u64, + "Bitrot self-test passed" + ); + } + Err(err) => { + BITROT_SELF_TEST_STATUS.store(STATUS_CELL_FAILED, Ordering::Release); + gauge!(METRIC_BITROT_SELFTEST_STATUS).set(STATUS_FAILED); + error!( + target: "rustfs::main::run", + event = EVENT_BITROT_SELFTEST, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "failed", + duration_us = started.elapsed().as_micros() as u64, + error = %err, + "Bitrot self-test failed" + ); + if strict { + return Err(io::Error::other(format!("bitrot self-test failed: {err}"))); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{BITROT_SELF_TEST_STATUS, STATUS_CELL_UNSET, bitrot_selftest_passed, run_startup_bitrot_self_test_with}; + use crate::storage_api::startup::background::BitrotSelfTestError; + use std::future::ready; + use std::sync::atomic::Ordering; + + fn failing_check() -> impl Future> { + ready(Err(BitrotSelfTestError::RoundtripReadback { + algorithm: "HighwayHash256S", + })) + } + + /// All scenarios run sequentially inside one test: the status cell is + /// process-global, so parallel per-scenario tests would race the reset and + /// read each other's outcomes (the exact order-dependent flake class this + /// module exists to avoid). + #[tokio::test] + async fn startup_self_test_publishes_outcome_and_strict_gates_abort() { + BITROT_SELF_TEST_STATUS.store(STATUS_CELL_UNSET, Ordering::Release); + + // Skipped: publishes nothing, never fails, never aborts. + run_startup_bitrot_self_test_with(false, true, || async { Ok(()) }) + .await + .expect("a disabled self-test must not fail even in strict mode"); + assert_eq!(bitrot_selftest_passed(), None, "a skipped run must leave the status unset"); + + // Passing: publishes Some(true), never fails. + run_startup_bitrot_self_test_with(true, false, || async { Ok(()) }) + .await + .expect("a passing check must never fail startup"); + assert_eq!(bitrot_selftest_passed(), Some(true), "a passing run must publish Some(true)"); + + // Failing, non-strict: publishes Some(false) but startup continues. + run_startup_bitrot_self_test_with(true, false, failing_check) + .await + .expect("a failed check must not abort startup in non-strict mode"); + assert_eq!(bitrot_selftest_passed(), Some(false), "a failing run must publish Some(false)"); + + // Failing, strict: startup error carries the failure and the published + // outcome stays a failure. + let err = run_startup_bitrot_self_test_with(true, true, failing_check) + .await + .expect_err("strict mode must turn a failed check into a startup error"); + assert!(err.to_string().contains("bitrot self-test failed")); + assert_eq!(bitrot_selftest_passed(), Some(false)); + } +} diff --git a/rustfs/src/lib.rs b/rustfs/src/lib.rs index f8e9d893d..b60a2bd20 100644 --- a/rustfs/src/lib.rs +++ b/rustfs/src/lib.rs @@ -76,6 +76,7 @@ pub mod allocator_reclaim; pub mod app; pub mod auth; pub mod auth_keystone; +pub(crate) mod bitrot_selftest; pub mod capacity; pub mod cluster_snapshot; pub mod config; diff --git a/rustfs/src/module_switches.rs b/rustfs/src/module_switches.rs index fcb0ffcef..ced58b0ec 100644 --- a/rustfs/src/module_switches.rs +++ b/rustfs/src/module_switches.rs @@ -33,6 +33,8 @@ pub(crate) const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED"; pub(crate) const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER"; pub(crate) const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED"; pub(crate) const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL"; +pub(crate) const ENV_BITROT_SELFTEST_ENABLE: &str = "RUSTFS_BITROT_SELFTEST_ENABLE"; +pub(crate) const ENV_BITROT_SELFTEST_STRICT: &str = "RUSTFS_BITROT_SELFTEST_STRICT"; static AUDIT_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_AUDIT_ENABLE); static NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE); @@ -47,6 +49,18 @@ pub(crate) fn heal_enabled_from_env() -> bool { get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true) } +/// Whether the startup bitrot algorithm self-test runs, defaulting to on +/// (rustfs/backlog#1873). +pub(crate) fn bitrot_selftest_enabled_from_env() -> bool { + rustfs_utils::get_env_bool(ENV_BITROT_SELFTEST_ENABLE, true) +} + +/// Whether a failed bitrot self-test aborts startup instead of only logging +/// and exposing a failed status, defaulting to off. +pub(crate) fn bitrot_selftest_strict_from_env() -> bool { + rustfs_utils::get_env_bool(ENV_BITROT_SELFTEST_STRICT, false) +} + /// Last published audit-module state. pub fn is_audit_module_enabled() -> bool { AUDIT_MODULE_ENABLED.load(Ordering::Relaxed) diff --git a/rustfs/src/startup_background.rs b/rustfs/src/startup_background.rs index 8cee2c502..fd5366740 100644 --- a/rustfs/src/startup_background.rs +++ b/rustfs/src/startup_background.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::module_switches::{heal_enabled_from_env, scanner_enabled_from_env}; +use crate::bitrot_selftest::run_startup_bitrot_self_test; +use crate::module_switches::{ + bitrot_selftest_enabled_from_env, bitrot_selftest_strict_from_env, heal_enabled_from_env, scanner_enabled_from_env, +}; use crate::storage_api::startup::background::{ECStore, set_workload_admission_snapshot_provider}; use crate::workload_admission::RustFsWorkloadAdmissionSnapshotProvider; use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; @@ -27,6 +30,12 @@ const LOG_SUBSYSTEM_STARTUP: &str = "startup"; const EVENT_BACKGROUND_SERVICES_CONFIGURED: &str = "background_services_configured"; pub(crate) async fn init_background_service_runtime(store: Arc) -> Result { + // Pin the bitrot algorithms before anything can write or verify a shard: + // the check costs well under a millisecond, and in strict mode a drifted + // build must abort here rather than after it has touched data + // (rustfs/backlog#1873). + run_startup_bitrot_self_test(bitrot_selftest_enabled_from_env(), bitrot_selftest_strict_from_env()).await?; + let _ = create_ahm_services_cancel_token(); let enable_scanner = scanner_enabled_from_env(); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 5d217f18e..d1e7c0f3a 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -569,6 +569,10 @@ pub(crate) mod ecstore_erasure { pub(crate) use rustfs_ecstore::api::erasure::{BitrotReader, Erasure}; } +/// Startup bitrot algorithm self-test (rustfs/backlog#1873), re-exported for +/// the root facade's background-startup section. +pub(crate) use rustfs_ecstore::api::erasure::{BitrotSelfTestError, bitrot_self_test}; + pub(crate) mod ecstore_storage { #[cfg(test)] pub(crate) use rustfs_ecstore::api::storage::init_local_disks; diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index b9b8d3c64..e90c1f528 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -214,7 +214,9 @@ pub(crate) mod startup { } pub(crate) mod background { - pub(crate) use crate::storage::storage_api::{ECStore, set_workload_admission_snapshot_provider}; + pub(crate) use crate::storage::storage_api::{ + BitrotSelfTestError, ECStore, bitrot_self_test, set_workload_admission_snapshot_provider, + }; } pub(crate) mod bucket_metadata { From 23b17c2d5ac9d6e4e7ebdb44ee4efb0bad102f5c Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 15:04:49 +0800 Subject: [PATCH 02/22] feat(madmin): add a SigV4-signed admin client for heal and scanner APIs (HS-05) (#6166) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat(madmin): add a SigV4-signed admin client for heal and scanner APIs The madmin crate held only wire types; automation and mc-style tooling had no way to drive the heal/scanner admin surface without hand-rolled HTTP. Add `AdminClient`, which signs with the same rustfs-signer path the server authenticates (UNSIGNED-PAYLOAD marker, matching RustFS peer admin calls) and wraps: - heal_start / heal_status / heal_stop over POST /rustfs/admin/v3/heal/ (bucket/prefix path params percent-encoded per segment; stop models the server's two cancel branches: token-scoped task status vs path-scoped start-success receipt); - background_heal_status, scanner_status (freshness typed), plus ilm_expiry_status / replacement_recovery_status passthroughs; - a public get_json escape hatch for endpoints not wrapped yet. Wire types follow the madmin-go model (SDK-owned mirrors pinned by round-trip tests): HealOpts with serde defaults so partial settings objects decode, HealScanMode accepting both the numeric and name encodings, and status structs that type the fields operators branch on while flattening unknown nested payloads verbatim so server additions cannot break the client. Errors map to a closed AdminClientError enum (InvalidEndpoint / Transport / HttpStatus with body / Decode). Tests cover wire round-trips, path building, both stop branches, error mapping, and — via a dependency-free raw-TCP test server — that signed requests carry a SigV4 Authorization header, the right method/path/ query, and the expected JSON body. Closes rustfs/backlog#1869 (first increment; single-sourcing the wire structs server-side and an embedded-server e2e roundtrip are noted as follow-ups there). Co-authored-by: heihutu --- Cargo.lock | 5 + crates/madmin/Cargo.toml | 5 + crates/madmin/src/client.rs | 851 ++++++++++++++++++++++++++++++++++++ crates/madmin/src/lib.rs | 2 + 4 files changed, 863 insertions(+) create mode 100644 crates/madmin/src/client.rs diff --git a/Cargo.lock b/Cargo.lock index 38637d173..b521a731d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9825,14 +9825,19 @@ name = "rustfs-madmin" version = "1.0.0-rc.2" dependencies = [ "hotpath", + "http 1.5.0", "humantime", "hyper", "jiff", + "reqwest", "rmp-serde", + "rustfs-signer", + "s3s", "serde", "serde_json", "sysinfo", "time", + "tokio", ] [[package]] diff --git a/crates/madmin/Cargo.toml b/crates/madmin/Cargo.toml index 6a1fe0ff5..dfd346334 100644 --- a/crates/madmin/Cargo.toml +++ b/crates/madmin/Cargo.toml @@ -37,7 +37,11 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] [dependencies] hotpath.workspace = true humantime.workspace = true +http.workspace = true hyper = { workspace = true, features = ["http2", "http1", "server"] } +reqwest = { workspace = true, features = ["json"] } +rustfs-signer.workspace = true +s3s.workspace = true jiff = { workspace = true, features = ["serde"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value"] } @@ -49,3 +53,4 @@ doctest = false [dev-dependencies] rmp-serde.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net"] } diff --git a/crates/madmin/src/client.rs b/crates/madmin/src/client.rs new file mode 100644 index 000000000..273f922a3 --- /dev/null +++ b/crates/madmin/src/client.rs @@ -0,0 +1,851 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Admin API HTTP client for heal and scanner management (rustfs/backlog#1869). +//! +//! [`AdminClient`] speaks the `/rustfs/admin/v3` surface with S3 SigV4 +//! request signing (the same scheme the server's admin router authenticates), +//! so `mc`-style tooling and automation can drive heal start/query/cancel and +//! read background-heal / scanner status without hand-rolling HTTP. +//! +//! Wire structs in this module mirror the server-side shapes +//! (`rustfs/src/admin/handlers/heal.rs`, `handlers/scanner.rs`, +//! `rustfs-common/src/heal_channel.rs`), following the madmin-go model where +//! the SDK owns its own copies and round-trip tests pin the encoding. Deeply +//! nested status payloads that the server composes from runtime types are +//! carried through as `serde_json::Value` and flattened maps rather than +//! duplicated field-for-field, so the client cannot silently drift on fields +//! it never interprets. + +use crate::heal_commands::HealResultItem; +use http::Method; +use serde::{Deserialize, Serialize, de}; +use std::time::Duration; + +/// Default admin API path prefix on a RustFS endpoint. +pub const DEFAULT_ADMIN_API_PREFIX: &str = "/rustfs/admin"; +/// Default SigV4 region when the server has no explicit region configured. +pub const DEFAULT_REGION: &str = "us-east-1"; + +/// Scan mode for a heal request, mirroring the server's numeric-or-name wire +/// encoding (`0` unknown/default, `1` normal, `2` deep). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum HealScanMode { + /// Server default; behaves as [`HealScanMode::Normal`]. + #[default] + Unknown, + /// Metadata-level checks only. + Normal, + /// Full bitrot verification while healing. + Deep, +} + +impl HealScanMode { + fn wire_number(self) -> u8 { + match self { + Self::Unknown => 0, + Self::Normal => 1, + Self::Deep => 2, + } + } + + fn from_wire_number(value: u8) -> Option { + match value { + 0 => Some(Self::Unknown), + 1 => Some(Self::Normal), + 2 => Some(Self::Deep), + _ => None, + } + } + + fn from_wire_name(value: &str) -> Option { + match value { + "unknown" => Some(Self::Unknown), + "normal" => Some(Self::Normal), + "deep" => Some(Self::Deep), + _ => None, + } + } +} + +impl Serialize for HealScanMode { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_u8(self.wire_number()) + } +} + +impl<'de> Deserialize<'de> for HealScanMode { + fn deserialize>(deserializer: D) -> Result { + struct HealScanModeVisitor; + + impl de::Visitor<'_> for HealScanModeVisitor { + type Value = HealScanMode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a heal scan mode number or name") + } + + fn visit_u64(self, value: u64) -> Result { + u8::try_from(value) + .ok() + .and_then(HealScanMode::from_wire_number) + .ok_or_else(|| E::custom(format!("unknown heal scan mode number: {value}"))) + } + + fn visit_str(self, value: &str) -> Result { + HealScanMode::from_wire_name(value).ok_or_else(|| E::custom(format!("unknown heal scan mode name: {value}"))) + } + } + + deserializer.deserialize_any(HealScanModeVisitor) + } +} + +/// Heal options for an admin heal request (mirror of the server body type). +/// Fields default on decode: a client should tolerate a server response whose +/// settings object omits fields it never set. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct HealOpts { + #[serde(default)] + pub recursive: bool, + #[serde(rename = "dryRun", default)] + pub dry_run: bool, + #[serde(default)] + pub remove: bool, + #[serde(default)] + pub recreate: bool, + #[serde(rename = "scanMode", default)] + pub scan_mode: HealScanMode, + #[serde(rename = "updateParity", default)] + pub update_parity: bool, + #[serde(rename = "nolock", default)] + pub no_lock: bool, + #[serde(rename = "pool", default)] + pub pool: Option, + #[serde(rename = "set", default)] + pub set: Option, +} + +/// Successful heal start / path-scoped cancel response. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HealStartSuccess { + pub client_token: String, + pub client_address: String, + #[serde(default)] + pub start_time: String, +} + +/// Heal task status response (query, cancel-with-token, start-then-poll). +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HealTaskStatus { + /// `running` | `finished` | `stopped` | `notFound`. + pub summary: String, + /// Failure detail for stopped tasks; empty otherwise. + #[serde(rename = "detail", default)] + pub failure_detail: String, + #[serde(default)] + pub start_time: String, + #[serde(default)] + pub settings: HealOpts, + #[serde(default)] + pub items: Vec, + #[serde(default)] + pub truncated: bool, + /// Live progress snapshot; the exact shape is owned by the heal runtime. + #[serde(default)] + pub progress: Option, +} + +/// `POST /v3/background-heal/status` response. Known top-level fields are +/// typed; the flattened heal info and operations matrix pass through verbatim. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BackgroundHealStatus { + /// `disabled` | `uninitialized` | `idle` | `active` | `degraded`. + pub state: String, + #[serde(default)] + pub heal_queue_length: u64, + #[serde(default)] + pub heal_active_tasks: u64, + #[serde(default)] + pub cluster_status_complete: bool, + #[serde(default)] + pub progress: Option, + /// Remaining wire fields (flattened `BackgroundHealInfo` plus the + /// priority-by-source operations matrix), carried verbatim. + #[serde(flatten)] + pub extra: serde_json::Map, +} + +/// `GET /v3/scanner/status` response, typed at the fields operators branch +/// on; everything else passes through verbatim. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScannerStatus { + pub enabled: bool, + /// `fresh` | `stale` | `unknown`; absent when the scanner never completed + /// a cycle. + #[serde(default)] + pub freshness: Option, + #[serde(flatten)] + pub extra: serde_json::Map, +} + +/// Freshness block of the scanner status response. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScannerFreshness { + /// `fresh` | `stale` | `unknown`. + pub state: String, +} + +impl ScannerStatus { + /// Convenience accessor for the freshness state string. + pub fn freshness(&self) -> &str { + self.freshness + .as_ref() + .map(|freshness| freshness.state.as_str()) + .unwrap_or("unknown") + } +} + +/// Everything that can go wrong in an admin client call. +#[derive(Debug)] +pub enum AdminClientError { + /// The endpoint URL could not be parsed. + InvalidEndpoint(String), + /// Request build/send failed (DNS, connect, timeout, body read). + Transport(reqwest::Error), + /// The server answered a non-2xx status. + HttpStatus { status: u16, body: String }, + /// The response body did not decode into the expected shape. + Decode { message: String }, +} + +impl std::fmt::Display for AdminClientError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidEndpoint(message) => write!(f, "invalid admin endpoint: {message}"), + Self::Transport(err) => write!(f, "admin request transport failure: {err}"), + Self::HttpStatus { status, body } => write!(f, "admin request failed with HTTP {status}: {body}"), + Self::Decode { message } => write!(f, "admin response decode failure: {message}"), + } + } +} + +impl std::error::Error for AdminClientError {} + +impl From for AdminClientError { + fn from(err: reqwest::Error) -> Self { + Self::Transport(err) + } +} + +/// A signed client for a RustFS admin API. +#[derive(Debug, Clone)] +pub struct AdminClient { + endpoint: reqwest::Url, + access_key: String, + secret_key: String, + session_token: String, + region: String, + api_prefix: String, + http: reqwest::Client, +} + +impl AdminClient { + /// Build a client for `endpoint` (e.g. `http://127.0.0.1:9000`) using root + /// or admin credentials. Requests are SigV4-signed with the same scheme + /// the server's admin router authenticates. + pub fn new(endpoint: &str, access_key: &str, secret_key: &str) -> Result { + let url = reqwest::Url::parse(endpoint).map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?; + if url.host_str().is_none() { + return Err(AdminClientError::InvalidEndpoint("endpoint has no host".to_string())); + } + let http = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(30)) + .build() + .map_err(AdminClientError::Transport)?; + Ok(Self { + endpoint: url, + access_key: access_key.to_string(), + secret_key: secret_key.to_string(), + session_token: String::new(), + region: DEFAULT_REGION.to_string(), + api_prefix: DEFAULT_ADMIN_API_PREFIX.to_string(), + http, + }) + } + + /// Attach an STS session token (signed as `x-amz-security-token`). + pub fn with_session_token(mut self, session_token: impl Into) -> Self { + self.session_token = session_token.into(); + self + } + + /// Override the SigV4 region (defaults to `us-east-1`, matching a + /// region-less RustFS deployment). + pub fn with_region(mut self, region: impl Into) -> Self { + self.region = region.into(); + self + } + + /// Override the admin API path prefix (defaults to `/rustfs/admin`). + pub fn with_api_prefix(mut self, prefix: impl Into) -> Self { + self.api_prefix = prefix.into(); + self + } + + /// Start a heal. `bucket` empty and `prefix` empty heals the whole + /// deployment (requires `recursive` or a `pool`/`set` pair in `opts`, + /// enforced server-side); a bucket alone heals the bucket (the server + /// forces `recursive` for bucket heals). + pub async fn heal_start( + &self, + bucket: Option<&str>, + prefix: Option<&str>, + opts: &HealOpts, + force_start: bool, + ) -> Result { + let body = serde_json::to_vec(opts).map_err(|err| AdminClientError::Decode { + message: err.to_string(), + })?; + let mut query = Vec::new(); + if force_start { + query.push(("forceStart", "true".to_string())); + } + self.post_json(&heal_path(bucket, prefix), &query, body).await + } + + /// Query the status of the heal identified by `client_token` (the token + /// returned by [`Self::heal_start`]) at the path it was started on. + pub async fn heal_status( + &self, + bucket: Option<&str>, + prefix: Option<&str>, + client_token: &str, + ) -> Result { + self.post_json(&heal_path(bucket, prefix), &[("clientToken", client_token.to_string())], Vec::new()) + .await + } + + /// Stop a heal: with a `client_token` only that task is cancelled and its + /// final status returned; without one, every heal task at the path is + /// cancelled (the server answers with a start-success-shaped receipt). + pub async fn heal_stop( + &self, + bucket: Option<&str>, + prefix: Option<&str>, + client_token: Option<&str>, + ) -> Result { + let mut query = vec![("forceStop", "true".to_string())]; + if let Some(token) = client_token { + query.push(("clientToken", token.to_string())); + } + match client_token { + Some(_) => { + let status: HealTaskStatus = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?; + Ok(HealStopOutcome::Stopped(status)) + } + None => { + let success: HealStartSuccess = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?; + Ok(HealStopOutcome::PathStopped(success)) + } + } + } + + /// Cluster-aggregated background heal status. + pub async fn background_heal_status(&self) -> Result { + self.get_json("/v3/background-heal/status").await + } + + /// Data scanner status (enabled state, freshness, runtime config). + pub async fn scanner_status(&self) -> Result { + self.get_json("/v3/scanner/status").await + } + + /// ILM expiry worker status. The payload is owned by the expiry + /// subsystem and still evolving; returned verbatim. + pub async fn ilm_expiry_status(&self) -> Result { + self.get_json("/v3/ilm/expiry/status").await + } + + /// Durable replacement-recovery status (admin v4). The payload is owned + /// by the heal runtime; returned verbatim. + pub async fn replacement_recovery_status(&self) -> Result { + self.get_json("/v4/heal/replacement-recovery").await + } + + /// Signed GET returning a decoded JSON body; escape hatch for endpoints + /// this client does not wrap yet. + pub async fn get_json Deserialize<'de>>(&self, path: &str) -> Result { + let url = self.url_for(path, &[])?; + let request = self.sign_and_build(Method::GET, url, Vec::new(), None).await?; + self.execute(request).await + } + + /// Signed POST returning a decoded JSON body. + async fn post_json Deserialize<'de>>( + &self, + path: &str, + query: &[(&str, String)], + body: Vec, + ) -> Result { + let content_type = if body.is_empty() { None } else { Some("application/json") }; + let url = self.url_for(path, query)?; + let request = self.sign_and_build(Method::POST, url, body, content_type).await?; + self.execute(request).await + } + + fn url_for(&self, path: &str, query: &[(&str, String)]) -> Result { + let mut url = self + .endpoint + .join(&format!("{}{}", self.api_prefix.trim_end_matches('/'), path)) + .map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?; + if !query.is_empty() { + let mut pairs = url.query_pairs_mut(); + for (key, value) in query { + pairs.append_pair(key, value); + } + } + Ok(url) + } + + /// Build a SigV4-signed request via the same signer the server trusts, + /// then hand the signed headers to the HTTP client. The signature covers + /// method, path, query, and an unsigned-payload marker — the same shape + /// RustFS itself sends for peer admin calls. + async fn sign_and_build( + &self, + method: Method, + url: reqwest::Url, + body: Vec, + content_type: Option<&str>, + ) -> Result { + let authority = match (url.host_str(), url.port_or_known_default()) { + (Some(host), Some(port)) => format!("{host}:{port}"), + _ => return Err(AdminClientError::InvalidEndpoint("endpoint has no authority".to_string())), + }; + let mut builder = http::Request::builder() + .method(method.clone()) + .uri(url.as_str()) + .header(http::header::HOST, &authority) + .header("x-amz-content-sha256", rustfs_signer::constants::UNSIGNED_PAYLOAD); + if let Some(content_type) = content_type { + builder = builder.header(http::header::CONTENT_TYPE, content_type); + } + let unsigned = builder + .body(s3s::Body::empty()) + .map_err(|err| AdminClientError::InvalidEndpoint(format!("build request failed: {err}")))?; + let signed = rustfs_signer::sign_v4( + unsigned, + body.len() as i64, + &self.access_key, + &self.secret_key, + &self.session_token, + &self.region, + ); + + let mut request = self + .http + .request(method, url) + .body(body) + .build() + .map_err(AdminClientError::Transport)?; + let headers = request.headers_mut(); + for (name, value) in signed.headers().iter() { + // HOST is owned by the HTTP client; the signed value above was + // built from the same URL authority, so they always agree. + if name == http::header::HOST { + continue; + } + headers.insert(name, value.clone()); + } + Ok(request) + } + + async fn execute Deserialize<'de>>(&self, request: reqwest::Request) -> Result { + let response = self.http.execute(request).await?; + let status = response.status(); + let bytes = response.bytes().await?; + if !status.is_success() { + return Err(AdminClientError::HttpStatus { + status: status.as_u16(), + body: String::from_utf8_lossy(&bytes).into_owned(), + }); + } + serde_json::from_slice(&bytes).map_err(|err| AdminClientError::Decode { + message: err.to_string(), + }) + } +} + +/// Response of [`AdminClient::heal_stop`]: cancelling a single tokened task +/// answers with that task's status, cancelling a whole path answers with a +/// start-success-shaped receipt. +#[derive(Debug, Clone)] +pub enum HealStopOutcome { + Stopped(HealTaskStatus), + PathStopped(HealStartSuccess), +} + +fn heal_path(bucket: Option<&str>, prefix: Option<&str>) -> String { + match (bucket, prefix) { + (Some(bucket), Some(prefix)) if !bucket.is_empty() && !prefix.is_empty() => { + format!("/v3/heal/{}/{}", percent_encode_path_segment(bucket), percent_encode_path_segment(prefix)) + } + (Some(bucket), Some(_)) | (Some(bucket), None) if !bucket.is_empty() => { + format!("/v3/heal/{}", percent_encode_path_segment(bucket)) + } + _ => "/v3/heal/".to_string(), + } +} + +/// Encode a single path segment (slashes are content, not separators, inside +/// bucket/prefix path params). +fn percent_encode_path_segment(segment: &str) -> String { + let mut out = String::with_capacity(segment.len()); + for byte in segment.bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => out.push(byte as char), + _ => out.push_str(&format!("%{byte:02X}")), + } + } + out +} + +#[cfg(test)] +mod tests { + use super::{ + AdminClient, AdminClientError, BackgroundHealStatus, HealOpts, HealScanMode, HealStartSuccess, HealTaskStatus, + ScannerStatus, heal_path, percent_encode_path_segment, + }; + use serde_json::json; + use std::sync::{Arc, Mutex}; + + #[test] + fn heal_paths_cover_root_bucket_and_prefix() { + assert_eq!(heal_path(None, None), "/v3/heal/"); + assert_eq!(heal_path(Some(""), Some("")), "/v3/heal/"); + assert_eq!(heal_path(Some("bucket"), None), "/v3/heal/bucket"); + assert_eq!(heal_path(Some("bucket"), Some("pre/fix")), "/v3/heal/bucket/pre%2Ffix"); + } + + #[test] + fn path_segments_percent_encode_reserved_characters() { + assert_eq!(percent_encode_path_segment("a b"), "a%20b"); + assert_eq!(percent_encode_path_segment("a/b"), "a%2Fb"); + assert_eq!(percent_encode_path_segment("ü"), "%C3%BC"); + } + + #[test] + fn heal_opts_round_trip_through_the_server_wire_shape() { + let opts = HealOpts { + recursive: true, + dry_run: false, + remove: true, + recreate: false, + scan_mode: HealScanMode::Deep, + update_parity: true, + no_lock: false, + pool: Some(1), + set: Some(2), + }; + let wire = serde_json::to_value(&opts).unwrap(); + assert_eq!(wire["scanMode"], json!(2), "the server body decodes scanMode as a number"); + let back: HealOpts = serde_json::from_value(wire).unwrap(); + assert_eq!(back.scan_mode, HealScanMode::Deep); + assert_eq!(back.pool, Some(1)); + } + + #[test] + fn heal_scan_mode_accepts_both_wire_encodings() { + assert_eq!(serde_json::from_value::(json!(1)).unwrap(), HealScanMode::Normal); + assert_eq!(serde_json::from_value::(json!("deep")).unwrap(), HealScanMode::Deep); + assert!(serde_json::from_value::(json!(9)).is_err()); + assert!(serde_json::from_value::(json!("sideways")).is_err()); + } + + #[test] + fn heal_task_status_decodes_the_server_response_shape() { + let raw = json!({ + "summary": "finished", + "detail": "", + "startTime": "2026-08-17T00:00:00Z", + "settings": {"recursive": false, "scanMode": 1}, + "items": [{ + "resultId": 1, "type": "object", "bucket": "b", "object": "o", "versionId": "", "detail": "", + "parityBlocks": 2, "dataBlocks": 2, "diskCount": 4, "setCount": 1, + "before": {"drives": []}, "after": {"drives": []}, "objectSize": 128 + }], + "truncated": false + }); + let status: HealTaskStatus = serde_json::from_value(raw).unwrap(); + assert_eq!(status.summary, "finished"); + assert_eq!(status.items.len(), 1); + assert_eq!(status.settings.scan_mode, HealScanMode::Normal); + assert!(status.progress.is_none()); + } + + #[test] + fn background_heal_status_types_known_fields_and_passes_the_rest_through() { + let raw = json!({ + "state": "active", + "bitrotStartTime": "t", + "healQueueLength": 3, + "healActiveTasks": 1, + "healOperations": {"queueLength": 3}, + "clusterStatusComplete": true + }); + let status: BackgroundHealStatus = serde_json::from_value(raw).unwrap(); + assert_eq!(status.state, "active"); + assert_eq!(status.heal_queue_length, 3); + assert!(status.cluster_status_complete); + assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through"); + } + + #[test] + fn scanner_status_defaults_freshness_to_unknown() { + let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}}); + let status: ScannerStatus = serde_json::from_value(raw).unwrap(); + assert_eq!(status.freshness(), "stale"); + let bare: ScannerStatus = serde_json::from_value(json!({"enabled": false})).unwrap(); + assert_eq!(bare.freshness(), "unknown"); + } + + #[test] + fn invalid_endpoint_is_rejected_without_io() { + let err = AdminClient::new("not a url", "ak", "sk").unwrap_err(); + assert!(matches!(err, AdminClientError::InvalidEndpoint(_))); + } + + #[tokio::test] + async fn signed_requests_carry_sigv4_authorization_and_correct_target() { + let server = TestServer::spawn(r#"{"clientToken":"token-1","clientAddress":"127.0.0.1:9","startTime":"t"}"#, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "minioadmin", "minioadmin") + .expect("client builds against the test server"); + + let start: HealStartSuccess = client + .heal_start( + Some("bucket"), + None, + &HealOpts { + recursive: true, + ..Default::default() + }, + false, + ) + .await + .expect("signed heal start decodes"); + + assert_eq!(start.client_token, "token-1"); + let request = server.recorded(); + assert_eq!(request.method, "POST"); + assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket"); + assert!(!request.query.contains("forceStart"), "absent flags must not be sent"); + let auth = request.header("authorization").expect("request must be signed"); + assert!(auth.starts_with("AWS4-HMAC-SHA256"), "SigV4 scheme, got: {auth}"); + assert!(auth.contains("Credential=minioadmin/"), "credentials must be in the Authorization header"); + assert_eq!( + request.header("x-amz-content-sha256").as_deref(), + Some("UNSIGNED-PAYLOAD"), + "the client signs the same payload marker RustFS peer calls use" + ); + assert_eq!(request.header("content-type").as_deref(), Some("application/json")); + assert!(request.body.contains("\"recursive\":true")); + } + + #[tokio::test] + async fn query_sends_client_token_on_the_same_path() { + let body = r#"{"summary":"running","detail":"","settings":{"recursive":false},"items":[],"truncated":false}"#; + let server = TestServer::spawn(body, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + + let status = client + .heal_status(Some("bucket"), None, "token-1") + .await + .expect("status decodes"); + assert_eq!(status.summary, "running"); + let request = server.recorded(); + assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket"); + assert!(request.query.contains("clientToken=token-1")); + assert!(!request.query.contains("forceStop")); + } + + #[tokio::test] + async fn stop_without_token_takes_the_path_cancel_branch() { + let server = TestServer::spawn(r#"{"clientToken":"path","clientAddress":"c","startTime":"t"}"#, 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + + let outcome = client.heal_stop(Some("bucket"), None, None).await.expect("path stop decodes"); + assert!(matches!(outcome, super::HealStopOutcome::PathStopped(_))); + let request = server.recorded(); + assert!(request.query.contains("forceStop=true")); + assert!(!request.query.contains("clientToken")); + } + + #[tokio::test] + async fn http_error_status_maps_to_a_typed_error_with_body() { + let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + let err = client.scanner_status().await.unwrap_err(); + match err { + AdminClientError::HttpStatus { status, body } => { + assert_eq!(status, 403); + assert!(body.contains("AccessDenied")); + } + other => panic!("expected HttpStatus, got {other:?}"), + } + } + + #[tokio::test] + async fn malformed_success_body_maps_to_a_decode_error() { + let server = TestServer::spawn("not json", 200).await; + let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap(); + assert!(matches!(client.scanner_status().await.unwrap_err(), AdminClientError::Decode { .. })); + } + + /// One recorded request, parsed off the wire with the minimum needed for + /// assertions: method, path, query, headers, body. + #[derive(Debug, Clone)] + struct RecordedRequest { + method: String, + path: String, + query: String, + headers: Vec<(String, String)>, + body: String, + } + + impl RecordedRequest { + fn header(&self, name: &str) -> Option { + self.headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.clone()) + } + } + + /// Minimal HTTP/1.1 server: one canned response per connection, every + /// request recorded behind an `Arc`. Deliberately dependency-free — + /// the assertions only need the raw request bytes. + struct TestServer { + addr: std::net::SocketAddr, + requests: Arc>>, + } + + impl TestServer { + async fn spawn(response_body: &'static str, status: u16) -> Self { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral port"); + let addr = listener.local_addr().expect("local addr"); + let requests: Arc>> = Arc::new(Mutex::new(Vec::new())); + + let recorded = requests.clone(); + tokio::spawn(async move { + let reason = if status == 200 { "OK" } else { "Forbidden" }; + let response = format!( + "HTTP/1.1 {status} {reason}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{response_body}", + response_body.len() + ); + // Each request is a fresh connection (connection: close); a + // bounded loop serves every call a test makes while letting + // the task exit instead of lingering for the whole process. + for _ in 0..16 { + let Ok((mut stream, _)) = listener.accept().await else { + break; + }; + let mut buffer = Vec::with_capacity(2048); + let mut chunk = [0u8; 2048]; + // Read headers plus content-length body, or stop on close. + loop { + if let Some(end) = find_header_end(&buffer) { + let content_length = extract_content_length(&buffer[..end]); + if buffer.len() >= end + content_length { + break; + } + } + let n = match stream.read(&mut chunk).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + buffer.extend_from_slice(&chunk[..n]); + if buffer.len() > 64 * 1024 { + break; + } + } + if let Some(request) = parse_request(&buffer) { + recorded.lock().expect("recorded lock").push(request); + } + let _ = stream.write_all(response.as_bytes()).await; + let _ = stream.shutdown().await; + } + }); + + Self { addr, requests } + } + + fn recorded(&self) -> RecordedRequest { + self.requests + .lock() + .expect("recorded lock") + .last() + .cloned() + .expect("the client call must have produced one recorded request") + } + } + + fn find_header_end(buffer: &[u8]) -> Option { + buffer.windows(4).position(|window| window == b"\r\n\r\n").map(|pos| pos + 4) + } + + fn extract_content_length(headers: &[u8]) -> usize { + let text = String::from_utf8_lossy(headers).to_ascii_lowercase(); + text.lines() + .find_map(|line| line.strip_prefix("content-length:")) + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(0) + } + + fn parse_request(raw: &[u8]) -> Option { + let end = find_header_end(raw)?; + let head = String::from_utf8_lossy(&raw[..end]); + let body = String::from_utf8_lossy(&raw[end..]).into_owned(); + let mut lines = head.lines(); + let request_line = lines.next()?; + let mut parts = request_line.split_whitespace(); + let method = parts.next()?.to_string(); + let target = parts.next()?.to_string(); + let (path, query) = match target.split_once('?') { + Some((path, query)) => (path.to_string(), query.to_string()), + None => (target, String::new()), + }; + let headers = lines + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.trim().to_string(), value.trim().to_string())) + .collect(); + Some(RecordedRequest { + method, + path, + query, + headers, + body, + }) + } +} diff --git a/crates/madmin/src/lib.rs b/crates/madmin/src/lib.rs index 154663b11..a9d4bd8b9 100644 --- a/crates/madmin/src/lib.rs +++ b/crates/madmin/src/lib.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod client; pub mod group; pub mod heal_commands; pub mod health; @@ -25,6 +26,7 @@ pub mod trace; pub mod user; pub mod utils; +pub use client::*; pub use group::*; pub use info_commands::*; pub use policy::*; From 984c7057132936a4772d29c1e209903cca3db85a Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 16:24:54 +0800 Subject: [PATCH 03/22] docs(ecstore): fix bitrot comment typo (#6168) Co-authored-by: heihutu --- _typos.toml | 1 + crates/ecstore/src/erasure/coding/bitrot.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/_typos.toml b/_typos.toml index 12d50d60e..7ff843137 100644 --- a/_typos.toml +++ b/_typos.toml @@ -40,6 +40,7 @@ mak = "mak" gae = "gae" GAE = "GAE" thr = "thr" +mis = "mis" # s3-tests original test names (cannot be changed) nonexisted = "nonexisted" consts = "consts" diff --git a/crates/ecstore/src/erasure/coding/bitrot.rs b/crates/ecstore/src/erasure/coding/bitrot.rs index 698b1b02c..947bbb68e 100644 --- a/crates/ecstore/src/erasure/coding/bitrot.rs +++ b/crates/ecstore/src/erasure/coding/bitrot.rs @@ -835,7 +835,7 @@ pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096; /// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S` /// (the production default). Pinned so any platform or build where the -/// implementation drifts fails startup instead of mis-hashing shards. +/// implementation drifts fails startup instead of miss-hashing shards. const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [ 0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea, 0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06, From e0b87b0e7e48d5db66358dde292a80498df6e6ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Mon, 17 Aug 2026 17:47:36 +0800 Subject: [PATCH 04/22] fix(site-replication): admit only verifiable peer-edit fences (#6123) --- rustfs/src/admin/handlers/site_replication.rs | 219 +++++++++++++++++- 1 file changed, 214 insertions(+), 5 deletions(-) diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index fba311ba9..18eef001b 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -6023,7 +6023,10 @@ fn edit_generation_wall_clock() -> u64 { /// node's clock behind the clock that fed the previous lifetime) mints /// below the stale mark and the origin stays fenced — but only until real /// time passes the previous lifetime's last allocation, because every later -/// allocation takes the wall-clock floor again. Bounded by the skew, +/// allocation takes the wall-clock floor again (and never longer than +/// [`PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS`]: a regression past the window +/// leaves the mark implausibly distant and the origin runs unfenced +/// immediately). Bounded by the skew, /// self-healing, and no rollback window beyond the plain counter's: a /// delivery applies only at or above the receiver's mark, so the one /// cross-lifetime interleaving that can apply stale content — a @@ -6063,6 +6066,52 @@ fn peer_edit_fence(queries: &HashMap) -> Option<(String, u64)> { Some((origin.clone(), generation)) } +/// How far below the recorded high-water mark a delivery may sit and still +/// be fenced as stale. The distance a GENUINE superseded delivery can trail +/// its origin's mark is small: retransmissions re-run the sender flow and +/// mint a fresh generation (the retry queue keys on the bare path and never +/// replays a fenced URL), so only an in-flight straggler of the losing +/// fan-out race trails the mark, by delivery latency — minutes at the +/// outside. A mark further above than this window cannot be explained by +/// any genuine race, only by a forged fence (the shared service account +/// lets any peer stamp any origin) or by a persisted clock excursion the +/// origin has since left behind — and fencing on it would silently drop the +/// origin's real edits, so the stale check ignores it instead. +const PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS: u64 = 24 * 60 * 60 * 1_000_000_000; + +/// Whether an incoming fence may be honoured, as far as this site can vouch +/// for it. The sender's identity is unverifiable (shared service account), +/// so the check runs over what the receiving state knows: the claimed origin +/// must be a site this state currently replicates with — the same membership +/// rule the load-time mark pruning applies, so every mark recorded behind +/// this check is one a reload would keep — and not this site itself, which +/// never delivers edits to itself. The caller IGNORES an inadmissible fence +/// rather than failing the request: the delivery applies exactly as an +/// unstamped (pre-fence) delivery would, no high-water mark is read or +/// written, and the worst a forged fence achieves is forfeiting an ordering +/// guarantee its sender was never owed. The generation itself is NOT +/// bounded here: a genuine origin whose hybrid clock persisted a wall-clock +/// excursion allocates arbitrarily far in the future, and refusing to +/// record its marks would strip the ordering fence from exactly the +/// deliveries that still race — the staleness window on the read side is +/// what defuses forged marks instead. +fn peer_edit_fence_is_admissible(state: &SiteReplicationState, local_deployment_id: &str, fence: &(String, u64)) -> bool { + let (origin, generation) = fence; + if origin != local_deployment_id && state.peers.contains_key(origin) { + return true; + } + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "fence_origin_not_a_remote_peer", + origin = %origin, + generation = *generation, + "ignoring inadmissible peer-edit fence" + ); + false +} + /// True when a strictly newer edit from the same origin site already landed /// here. No lock on the sending side can order deliveries issued by two /// nodes of that site, so ordering is decided here, on the generation the @@ -6070,11 +6119,42 @@ fn peer_edit_fence(queries: &HashMap) -> Option<(String, u64)> { /// stale: one edit legitimately fans out several deliveries under a single /// generation (the ILM-expiry edit sends every peer's record), and a replay of /// an applied delivery re-applies the same edit idempotently. +/// +/// A mark more than [`PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS`] above the +/// delivery is implausible and does NOT fence: the shared service account +/// means any peer can stamp any origin, so a forged `u64::MAX`-scale mark +/// would otherwise silently swallow the origin's genuine edits for good. +/// Bounding the fence by distance instead of by an absolute ceiling keeps +/// ordering intact wherever the origin's clock actually operates — two +/// racing deliveries trail each other by seconds whether the hybrid clock +/// tracks wall time or persists a long-gone excursion far ahead of it — +/// while a mark no genuine race can explain merely downgrades the origin to +/// unfenced (pre-fence) delivery instead of dropping its edits. (One genuine +/// shape does land out here: a plain-counter straggler arriving after its +/// origin's first hybrid-clock edit. It gets the same downgrade — applied +/// unfenced — once, at upgrade time; fencing it instead would silence the +/// mirror case, a hybrid-clock origin downgraded back to the plain counter.) fn peer_edit_delivery_is_stale(state: &SiteReplicationState, origin: &str, generation: u64) -> bool { - state - .applied_edit_generations - .get(origin) - .is_some_and(|applied| *applied > generation) + let Some(applied) = state.applied_edit_generations.get(origin) else { + return false; + }; + if *applied <= generation { + return false; + } + if *applied - generation > PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "fence_mark_beyond_staleness_window", + origin, + generation, + applied_mark = *applied, + "ignoring implausibly distant peer-edit high-water mark" + ); + return false; + } + true } fn record_applied_peer_edit_generation(state: &mut SiteReplicationState, origin: &str, generation: u64) { @@ -10698,6 +10778,11 @@ impl Operation for SRPeerEditHandler { let outcome = update_site_replication_state_when_changed(move |state| { let mut incoming = incoming; let local_peer = local_peer_at_endpoint(commit_endpoint, state); + // The fence is self-reported — the shared service account means + // the sender cannot be identified — so it is honoured only after + // the admissibility check, against the same state it will gate. + let commit_fence = + commit_fence.filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence)); // Ordering fence: the sending site allocates the generation under // its state-object lock, so a delivery that lost the race carries // a generation this site has already passed. Applying it would @@ -13393,6 +13478,15 @@ mod tests { handler_block.contains("record_applied_peer_edit_generation(state, origin, *generation);"), "SRPeerEditHandler must record the applied generation so later stale deliveries are recognised" ); + // Fence hardening: origin and generation are self-reported by a + // caller the shared service account cannot identify, so the handler + // must pass the fence through the admissibility check — against the + // same state the fence gates, i.e. inside the transaction — before + // reading or raising any high-water mark. + assert!( + handler_block.contains(".filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence))"), + "SRPeerEditHandler must admit a fence only through peer_edit_fence_is_admissible inside the state transaction" + ); // P1-15 PR2: both halves of the fence and the edit they fence share // ONE transaction. Checking the fence against a state read outside the // lock would let the check pass on one snapshot and the write land on @@ -14769,6 +14863,121 @@ mod tests { assert!(peer_edit_delivery_is_stale(&state, origin, generation - 1)); } + /// A fence is self-reported: every site authenticates peer traffic with + /// the same site-replicator credential, so a compromised peer can stamp + /// ANY origin with ANY generation. An origin the receiver does not + /// replicate with — or the receiver itself — is ignored and plants no + /// mark; a mark a compromised peer plants for a CURRENT origin cannot + /// silence that origin, because the staleness window refuses to fence on + /// a mark implausibly far above the genuine deliveries. + #[test] + fn forged_peer_edit_fences_cannot_poison_the_high_water_marks() { + let mut state = SiteReplicationState { + peers: BTreeMap::from([ + ( + "site-local".to_string(), + PeerInfo { + deployment_id: "site-local".to_string(), + ..peer("local", "https://local.example:9000") + }, + ), + ( + "site-victim".to_string(), + PeerInfo { + deployment_id: "site-victim".to_string(), + ..peer("victim", "https://victim.example:9000") + }, + ), + ]), + ..Default::default() + }; + // An origin outside the current membership is refused outright... + let unknown = ("site-unknown".to_string(), 4u64); + assert!(!peer_edit_fence_is_admissible(&state, "site-local", &unknown)); + + // No site delivers edits to itself: a fence claiming the receiver as + // its origin is forged by construction, current peer or not. + let own = ("site-local".to_string(), 4u64); + assert!(!peer_edit_fence_is_admissible(&state, "site-local", &own)); + + // A current remote peer's fence is admitted and works end to end. + let genuine = ("site-victim".to_string(), 1u64); + assert!(peer_edit_fence_is_admissible(&state, "site-local", &genuine)); + assert!(!peer_edit_delivery_is_stale(&state, &genuine.0, genuine.1)); + record_applied_peer_edit_generation(&mut state, &genuine.0, genuine.1); + assert_eq!(state.applied_edit_generations.get("site-victim"), Some(&1)); + + // A forged u64::MAX-scale mark CAN be recorded — the shared service + // account means the receiver cannot tell the stamp was forged — but + // it is inert: the victim's genuine hybrid-clock deliveries sit far + // more than the staleness window below it, so they keep applying + // instead of being silently acked-and-dropped. + record_applied_peer_edit_generation(&mut state, "site-victim", u64::MAX); + assert!(!peer_edit_delivery_is_stale(&state, "site-victim", edit_generation_wall_clock())); + } + + /// The staleness window bounds the fence by DISTANCE from the mark, not + /// by an absolute clock ceiling, so ordering must hold wherever the + /// origin's hybrid clock actually operates. The regression that matters: + /// a temporary wall-clock excursion far in the future is persisted by + /// `next_peer_edit_generation` (`max(now, prev + 1)` never comes back + /// down), and two later edits g+1 then g can arrive in reverse order — + /// g must still be fenced, even though both generations dwarf the + /// receiver's clock. Conversely a mark further above a delivery than any + /// genuine race can explain must not fence it. + #[test] + fn peer_edit_fence_orders_a_persisted_future_clock_and_defuses_distant_marks() { + let mut state = SiteReplicationState { + peers: BTreeMap::from([( + "site-origin".to_string(), + PeerInfo { + deployment_id: "site-origin".to_string(), + ..peer("origin", "https://origin.example:9000") + }, + )]), + ..Default::default() + }; + + // The origin's clock once jumped ten years ahead; the hybrid clock + // keeps allocating from there long after the clock was corrected. + let excursion = edit_generation_wall_clock() + 10 * 365 * 24 * 60 * 60 * 1_000_000_000; + let fence = ("site-origin".to_string(), excursion + 1); + assert!(peer_edit_fence_is_admissible(&state, "site-local", &fence)); + record_applied_peer_edit_generation(&mut state, &fence.0, fence.1); + + // The reverse delivery of the race: g arrives after g+1 landed. + // Without the fence it would commit last and roll g+1 back. + assert!(peer_edit_delivery_is_stale(&state, "site-origin", excursion)); + // Equal generation (same edit's fan-out or a replay) still applies, + // as does the next edit. + assert!(!peer_edit_delivery_is_stale(&state, "site-origin", excursion + 1)); + assert!(!peer_edit_delivery_is_stale(&state, "site-origin", excursion + 2)); + + // The window's exact boundary: a delivery trailing the mark by the + // full window is still fenced; one nanosecond further is not — that + // distance is no longer explicable by a genuine race, only by a + // forged mark or an excursion the origin has left behind. + let mark = fence.1; + // A straggler trailing by a concrete hour must still be fenced — + // pins the window's real magnitude, not just its symbolic boundary. + assert!(peer_edit_delivery_is_stale(&state, "site-origin", mark - 60 * 60 * 1_000_000_000)); + assert!(peer_edit_delivery_is_stale( + &state, + "site-origin", + mark - PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS + )); + assert!(!peer_edit_delivery_is_stale( + &state, + "site-origin", + mark - PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS - 1 + )); + + // A pre-hybrid plain-counter origin trails such a mark by eons: it + // is not fenced (the rc.2-era downgrade case), it just runs + // unfenced until its counter regime catches up. + assert!(!peer_edit_delivery_is_stale(&state, "site-origin", 3)); + } + /// P1-15 review follow-up: a site that leaves the mesh drops below two /// peers, which clears its state object and restarts its generation /// counter at zero. A mark left over from its previous membership would From 59b7d13095e780fccbff829a19770a713f593884 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 17 Aug 2026 19:40:56 +0800 Subject: [PATCH 05/22] feat(scanner): expose prefix-level bucket usage via admin API (HS-08) (#6171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat(scanner): expose prefix-level bucket usage via admin API The scanner's per-bucket, per-set usage caches already hold a path-keyed prefix tree, but dui() flattened it only to bucket names — consoles and operators had no way to ask "what does this prefix hold" without an S3 listing sweep (rustfs/backlog#1872, MinIO loadPrefixUsageFromBackend parity). Add: - data-usage: prefix_usage_in_cache — a shared aggregation over the entry map (arbitrary prefix, full counters, one-level sub-prefix breakdown with names recovered from the literal-path cache keys), hardened like the scanner's checked flatten: cycles, dangling child links, over-deep trees, and overflowing counters yield None rather than unbounded recursion or wrapped totals. - ecstore: ECStore::all_set_disks — iterate every erasure set so a query can read each set's own cache copy; the hash-routed store path would always land on one set. - scanner: bucket_prefix_usage — per-set loads (5s budget each, a slow set degrades to not-reporting instead of stalling the caller), merged across sets with partial/compacted/truncated flags, served from a bounded 30s cache (128 entries, hard-capped) that bucket writes invalidate through the dirty-usage hook. - admin: GET /rustfs/admin/v3/usage/{bucket}?prefix=&max-entries= behind the same any-of gate as datausageinfo (DataUsageInfoAdminAction OR ListBucketAction), rejecting unknown query parameters and clamping max-entries to 1..=10000. Route registered in the policy table (deferred MultipleActions, matching datausageinfo) and the route matrix test. Closes rustfs/backlog#1872. Co-authored-by: heihutu --- crates/data-usage/src/data_usage.rs | 286 ++++++++++++++++ crates/ecstore/src/store/mod.rs | 10 + crates/scanner/src/data_usage_define.rs | 10 +- crates/scanner/src/lib.rs | 2 + crates/scanner/src/prefix_usage.rs | 349 ++++++++++++++++++++ crates/scanner/src/scanner_io.rs | 4 + rustfs/src/admin/handlers/mod.rs | 1 + rustfs/src/admin/handlers/system.rs | 8 +- rustfs/src/admin/handlers/usage_prefix.rs | 142 ++++++++ rustfs/src/admin/mod.rs | 4 +- rustfs/src/admin/route_policy.rs | 5 + rustfs/src/admin/route_registration_test.rs | 1 + 12 files changed, 816 insertions(+), 6 deletions(-) create mode 100644 crates/scanner/src/prefix_usage.rs create mode 100644 rustfs/src/admin/handlers/usage_prefix.rs diff --git a/crates/data-usage/src/data_usage.rs b/crates/data-usage/src/data_usage.rs index ebbd03261..b08281d0f 100644 --- a/crates/data-usage/src/data_usage.rs +++ b/crates/data-usage/src/data_usage.rs @@ -870,6 +870,157 @@ pub struct DataUsageCacheInfo { pub snapshot_complete: bool, } +/// Prefix-level usage over a raw entry map — the shared core behind +/// [`DataUsageCache::prefix_usage`], usable by any cache-shaped reader (the +/// scanner's writer-side cache has the same map type). +/// +/// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix +/// names come straight off the child keys — no reverse mapping exists or is +/// needed. A compacted prefix carries its aggregate but no children, which +/// the `compacted` flag reports so callers can say why the breakdown is +/// empty. `truncated` is set when the breakdown exceeded `max_entries` and +/// was cut (largest first). +pub fn prefix_usage_in_cache( + cache: &HashMap, + bucket: &str, + prefix: &str, + max_entries: usize, +) -> Option { + let prefix = prefix.trim_matches('/'); + let root = if prefix.is_empty() { + bucket.to_string() + } else { + format!("{bucket}/{prefix}") + }; + let entry = cache.get(&hash_path(&root).key())?.clone(); + + let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?); + + let child_prefix = format!("{root}/"); + let mut sub_prefixes: Vec = entry + .children + .iter() + .filter_map(|child_key| { + let child = cache.get(child_key)?; + let child_flat = flatten_entry(cache, child, 1)?; + // Child keys are literal `bucket/pre/name` paths; a trailing + // slash marks a directory object and is display-only here. + let name = child_key + .strip_prefix(child_prefix.as_str()) + .unwrap_or(child_key.as_str()) + .trim_end_matches('/') + .to_string(); + Some(PrefixUsageEntry { + prefix: name, + usage: PrefixUsageSummary::from_entry(&child_flat), + }) + }) + .collect(); + sub_prefixes.sort_by(|left, right| { + right + .usage + .size + .cmp(&left.usage.size) + .then_with(|| left.prefix.cmp(&right.prefix)) + }); + let truncated = sub_prefixes.len() > max_entries; + sub_prefixes.truncate(max_entries); + + Some(PrefixUsageQuery { + usage, + compacted: entry.compacted, + truncated, + sub_prefixes, + }) +} + +/// Maximum subtree depth [`flatten_entry`] will walk before declaring the +/// cache corrupt — the same bound the scanner's checked flatten uses. +const PREFIX_USAGE_MAX_DEPTH: usize = 1024; + +/// Flatten one entry's subtree into an aggregate: the free-function twin of +/// [`DataUsageCache::flatten`], carrying the scanner checked-flatten +/// hardening so a corrupt cache (cycles, over-deep trees, overflowing +/// counters) yields `None` instead of unbounded recursion or wrapped totals. +fn flatten_entry(cache: &HashMap, root: &DataUsageEntry, depth: usize) -> Option { + if depth > PREFIX_USAGE_MAX_DEPTH { + return None; + } + let mut flattened = DataUsageEntry::default(); + if !flattened.checked_merge(root) { + return None; + } + flattened.compacted = root.compacted; + // The root itself is not pre-seeded: it is merged above, and a corrupt + // child edge pointing back at the root's own key is still terminated by + // the visited set on first encounter. + let mut visited: HashSet<&str> = HashSet::new(); + let mut pending: Vec<(&String, usize)> = root.children.iter().map(|child| (child, depth + 1)).collect(); + while let Some((key, child_depth)) = pending.pop() { + if child_depth > PREFIX_USAGE_MAX_DEPTH || !visited.insert(key.as_str()) { + return None; + } + let entry = cache.get(key)?; + if !flattened.checked_merge(entry) { + return None; + } + pending.extend(entry.children.iter().map(|child| (child, child_depth + 1))); + } + flattened.children.clear(); + Some(flattened) +} + +/// Flattened counters of one prefix subtree, as returned by +/// [`DataUsageCache::prefix_usage`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PrefixUsageSummary { + pub size: u64, + pub objects: u64, + pub versions: u64, + pub delete_markers: u64, +} + +impl PrefixUsageSummary { + fn from_entry(entry: &DataUsageEntry) -> Self { + Self { + size: entry.size as u64, + objects: entry.objects as u64, + versions: entry.versions as u64, + delete_markers: entry.delete_markers as u64, + } + } + + /// Add another set's counters into this one (entries are partitioned by + /// set, so per-set results sum). + pub fn merge(&mut self, other: &Self) { + self.size = self.size.saturating_add(other.size); + self.objects = self.objects.saturating_add(other.objects); + self.versions = self.versions.saturating_add(other.versions); + self.delete_markers = self.delete_markers.saturating_add(other.delete_markers); + } +} + +/// One first-level sub-prefix row of a [`PrefixUsageQuery`]. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +pub struct PrefixUsageEntry { + pub prefix: String, + pub usage: PrefixUsageSummary, +} + +/// Result of [`DataUsageCache::prefix_usage`]. +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PrefixUsageQuery { + pub usage: PrefixUsageSummary, + /// The prefix entry was compacted by the scanner: its aggregate is valid + /// but no sub-prefix breakdown exists on disk. + pub compacted: bool, + /// The breakdown had more entries than `max_entries`; the largest remain. + pub truncated: bool, + pub sub_prefixes: Vec, +} + /// Read-only projection of a scanner-written `.usage-cache.bin` file. /// /// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`) @@ -997,6 +1148,21 @@ impl DataUsageCache { } } + /// Prefix-level usage for one bucket subtree, plus the one-level + /// breakdown below it (rustfs/backlog#1872, MinIO + /// `loadPrefixUsageFromBackend` parity and beyond: arbitrary prefixes and + /// full counters instead of first-level sizes only). + /// + /// Cache keys are cleaned literal paths (`bucket/pre/fix`), so sub-prefix + /// names come straight off the child keys — no reverse mapping exists or + /// is needed. A compacted prefix carries its aggregate but no children, + /// which the `compacted` flag reports so callers can say why the + /// breakdown is empty. `truncated` is set when the breakdown exceeded + /// `max_entries` and was cut (largest first). + pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option { + prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries) + } + pub fn force_compact(&mut self, limit: usize) { if self.cache.len() < limit { return; @@ -1898,6 +2064,126 @@ mod tests { ); } + /// Build a cache shaped like `bucket/{a,b/{c,d}},bucket/loose` with + /// distinct counters so aggregation is observable. + fn prefix_usage_fixture_cache() -> DataUsageCache { + let mut cache = DataUsageCache::default(); + let mut insert = |path: &str, parent: &str, size: usize, objects: usize, versions: usize, delete_markers: usize| { + cache.replace( + path, + parent, + DataUsageEntry { + size, + objects, + versions, + delete_markers, + ..Default::default() + }, + ); + }; + insert("bucket", "", 0, 0, 0, 0); + insert("bucket/a", "bucket", 100, 1, 1, 0); + insert("bucket/b", "bucket", 0, 0, 0, 0); + insert("bucket/b/c", "bucket/b", 200, 2, 2, 1); + insert("bucket/b/d", "bucket/b", 40, 1, 3, 0); + insert("bucket/loose", "bucket", 10, 1, 1, 1); + cache + } + + #[test] + fn prefix_usage_aggregates_bucket_root_and_one_level_below() { + let cache = prefix_usage_fixture_cache(); + + let root = cache + .prefix_usage("bucket", "", 100) + .expect("root query must find the bucket entry"); + assert_eq!(root.usage.size, 350, "root aggregate flattens the whole subtree"); + assert_eq!(root.usage.objects, 5); + assert_eq!(root.usage.versions, 7); + assert_eq!(root.usage.delete_markers, 2); + assert!(!root.compacted); + assert!(!root.truncated); + // Breakdown is one level: b (240) before a (100) before loose (10), + // each flattened to its own subtree total. + let names: Vec<(&str, u64)> = root + .sub_prefixes + .iter() + .map(|entry| (entry.prefix.as_str(), entry.usage.size)) + .collect(); + assert_eq!(names, vec![("b", 240), ("a", 100), ("loose", 10)]); + } + + #[test] + fn prefix_usage_drills_into_arbitrary_prefixes() { + let cache = prefix_usage_fixture_cache(); + + let b = cache.prefix_usage("bucket", "b", 100).expect("nested prefix must resolve"); + assert_eq!(b.usage.size, 240); + assert_eq!(b.usage.versions, 5); + let names: Vec<&str> = b.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect(); + assert_eq!(names, vec!["c", "d"]); + + // Prefix slashes are normalized away. + let slashed = cache.prefix_usage("bucket", "/b/", 100).expect("slash-insensitive lookup"); + assert_eq!(slashed.usage.size, 240); + + assert!(cache.prefix_usage("bucket", "absent", 100).is_none(), "unknown prefix must be a miss"); + assert!(cache.prefix_usage("other", "", 100).is_none(), "unknown bucket must be a miss"); + } + + #[test] + fn prefix_usage_reports_and_respects_truncation() { + let cache = prefix_usage_fixture_cache(); + let capped = cache.prefix_usage("bucket", "", 2).expect("root query"); + assert!(capped.truncated, "three children capped to two must flag truncation"); + let names: Vec<&str> = capped.sub_prefixes.iter().map(|entry| entry.prefix.as_str()).collect(); + assert_eq!(names, vec!["b", "a"], "largest prefixes survive the cut"); + } + + #[test] + fn prefix_usage_marks_compacted_entries() { + let mut cache = DataUsageCache::default(); + cache.replace( + "bucket", + "", + DataUsageEntry { + size: 999, + objects: 9, + compacted: true, + ..Default::default() + }, + ); + + let compacted = cache.prefix_usage("bucket", "", 100).expect("compacted root resolves"); + assert!(compacted.compacted, "compaction must be visible to callers"); + assert_eq!(compacted.usage.size, 999); + assert!(compacted.sub_prefixes.is_empty(), "a compacted entry carries no children"); + } + + #[test] + fn prefix_usage_rejects_cyclic_and_dangling_caches() { + // A self-referencing child (corrupt cache) must yield a miss for the + // whole query, not unbounded recursion. + let mut cache = prefix_usage_fixture_cache(); + if let Some(entry) = cache.cache.get_mut("bucket/b") { + entry.children.insert("bucket/b".to_string()); + } + assert!(cache.prefix_usage("bucket", "b", 100).is_none(), "a cyclic subtree must be rejected"); + // The unaffected sibling still answers. + assert!(cache.prefix_usage("bucket", "a", 100).is_some()); + + // A child key with no entry (dangling link) is rejected rather than + // silently dropped: half a tree would under-report usage. + let mut dangling = prefix_usage_fixture_cache(); + if let Some(entry) = dangling.cache.get_mut("bucket/b") { + entry.children.insert("bucket/b/ghost".to_string()); + } + assert!( + dangling.prefix_usage("bucket", "b", 100).is_none(), + "a dangling child link must be rejected" + ); + } + #[test] fn hash_path_uses_portable_slash_semantics() { for (input, expected) in [ diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index f51fa6df5..8d02ef441 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -216,6 +216,16 @@ impl std::fmt::Debug for ECStore { /// These delegate to the process-global statics. No local state — the globals /// remain the single source of truth until the migration is complete. impl ECStore { + /// Every erasure set across all pools, pool-major order. + /// + /// Read-only queries that must consult each set's own copy of a + /// per-bucket object (e.g. the scanner's `.usage-cache.bin`) iterate + /// this instead of the hash-routed store path, which would always land + /// on one set (rustfs/backlog#1872). + pub fn all_set_disks(&self) -> Vec> { + self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect() + } + /// Get server configuration (delegates to global) pub fn get_server_config(&self) -> Option { runtime_sources::server_config() diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 67a199661..8ac714314 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -28,7 +28,8 @@ use rustfs_common::heal_channel::HealScanMode; use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS; pub use rustfs_data_usage::{ AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME, - DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, TierStats, hash_path, + DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry, + PrefixUsageQuery, PrefixUsageSummary, TierStats, hash_path, prefix_usage_in_cache, }; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use tokio::time::{Duration, Instant, sleep, timeout}; @@ -430,6 +431,13 @@ pub(crate) enum DataUsageCachePrepareOutcome { } impl DataUsageCache { + /// Prefix-level usage query over this (writer-side) cache; see + /// [`prefix_usage_in_cache`] for the semantics + /// (rustfs/backlog#1872). + pub fn prefix_usage(&self, bucket: &str, prefix: &str, max_entries: usize) -> Option { + prefix_usage_in_cache(&self.cache, bucket, prefix, max_entries) + } + pub(crate) fn prepare_for_scan( &mut self, name: &str, diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index 4a5cc7543..36cc6b817 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -53,6 +53,7 @@ use tokio_util::sync::CancellationToken; pub mod data_usage_define; pub mod error; +pub mod prefix_usage; mod remote_scanner; pub mod runtime_config; pub mod scanner; @@ -64,6 +65,7 @@ pub(crate) mod storage_api; pub use data_usage_define::*; pub use error::ScannerError; +pub use prefix_usage::{BucketPrefixUsageResponse, bucket_prefix_usage, invalidate_prefix_usage_cache}; pub use remote_scanner::{ NS_SCANNER_MAX_REQUEST_BODY_SIZE, RemoteScannerAdmission, RemoteScannerRequest, admit_remote_scanner_request, claim_remote_scanner_request, decode_remote_scanner_request, preflight_remote_scanner_request, diff --git a/crates/scanner/src/prefix_usage.rs b/crates/scanner/src/prefix_usage.rs new file mode 100644 index 000000000..9f94957de --- /dev/null +++ b/crates/scanner/src/prefix_usage.rs @@ -0,0 +1,349 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Prefix-level bucket usage for admin/console consumers (rustfs/backlog#1872, +//! MinIO `loadPrefixUsageFromBackend` parity). +//! +//! The per-bucket, per-set `.usage-cache.bin` objects already hold a +//! path-keyed prefix tree; this module reads every set's copy through that +//! set's own object layer (the hash-routed store path would always land on +//! one set), aggregates the overlapping trees, and serves the result from a +//! bounded 30-second cache. Bucket writes poke the cache through the +//! dirty-usage hook so a fresh scan is visible immediately. + +use crate::data_usage_define::{DATA_USAGE_CACHE_NAME, DataUsageCache}; +use crate::error::ScannerError; +use crate::storage_api::owner::{ + EcstoreSetDisks, EcstoreStore, ecstore_is_reserved_or_invalid_bucket, ecstore_resolve_object_store_handle, +}; +use futures::future::join_all; +use rustfs_data_usage::{PrefixUsageEntry, PrefixUsageSummary}; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime}; +use tracing::{debug, warn}; + +const LOG_COMPONENT_SCANNER: &str = "scanner"; +const LOG_SUBSYSTEM_PREFIX_USAGE: &str = "prefix_usage"; +const EVENT_PREFIX_USAGE_CACHE_STATE: &str = "prefix_usage_cache_state"; + +/// How long a computed breakdown stays fresh. MinIO uses the same 30s for +/// its prefix-usage cache; bucket writes additionally invalidate on the spot. +const CACHE_TTL: Duration = Duration::from_secs(30); +/// Hard entry cap for the result cache; exceeded, expired entries go first +/// and the map clears rather than growing past the bound. +const CACHE_MAX_ENTRIES: usize = 128; +/// Per-set cache read budget. The underlying loader retries for up to a +/// minute per attempt on backend errors — far too long for an admin GET, so +/// a slow set degrades to "not reporting" instead of stalling the caller. +const PER_SET_LOAD_TIMEOUT: Duration = Duration::from_secs(5); + +/// Aggregated prefix-usage answer across every erasure set. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BucketPrefixUsageResponse { + pub bucket: String, + pub prefix: String, + pub usage: PrefixUsageSummary, + /// Every reporting set's prefix entry was compacted: the aggregate is + /// valid, the sub-prefix breakdown is empty on disk. + pub compacted: bool, + /// The sub-prefix breakdown is incomplete: at least one reporting set + /// had the prefix compacted (or absent while others found it), so its + /// objects cannot be attributed to a sub-prefix. + pub sub_prefixes_partial: bool, + /// The breakdown exceeded the caller's entry limit; largest remain. + pub truncated: bool, + pub sub_prefixes: Vec, + /// Sets whose cache held this bucket and prefix. + pub sets_reporting: usize, + pub sets_total: usize, + /// Newest `last_update` across reporting sets, unix seconds. + pub last_update_unix_secs: Option, +} + +#[derive(Clone)] +struct CachedResponse { + computed_at: std::time::Instant, + response: Arc, +} + +/// Cache key: (lowercased bucket, normalized prefix, max entries). +type PrefixUsageCacheKey = (String, String, usize); +type PrefixUsageCacheMap = Option>; + +static PREFIX_USAGE_CACHE: Mutex = Mutex::new(None); + +/// Drop cached results for `bucket` (empty string clears everything). Wired +/// into the dirty-usage recording path so a write makes the next prefix +/// query recompute instead of serving up to `CACHE_TTL` seconds of stale +/// numbers. +pub fn invalidate_prefix_usage_cache(bucket: &str) { + let mut guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner()); + let Some(map) = guard.as_mut() else { + return; + }; + if bucket.is_empty() { + map.clear(); + return; + } + map.retain(|(cached_bucket, ..), _| !cached_bucket.eq_ignore_ascii_case(bucket)); +} + +/// Query prefix usage for `bucket` (arbitrary `prefix`, empty = whole +/// bucket), merging every erasure set's own cache copy. `max_entries` bounds +/// the sub-prefix rows (largest first). +pub async fn bucket_prefix_usage( + bucket: &str, + prefix: &str, + max_entries: usize, +) -> Result { + if ecstore_is_reserved_or_invalid_bucket(bucket, true) { + return Err(ScannerError::Other(format!("invalid bucket name: {bucket}"))); + } + let normalized_prefix = prefix.trim_matches('/').to_string(); + let cache_key = (bucket.to_ascii_lowercase(), normalized_prefix.clone(), max_entries); + if let Some(response) = lookup_cached(&cache_key) { + return Ok((*response).clone()); + } + + let store = ecstore_resolve_object_store_handle() + .ok_or_else(|| ScannerError::Other("object store is not initialized".to_string()))?; + let response = Arc::new(compute_prefix_usage(store, bucket, &normalized_prefix, max_entries).await); + store_cached(cache_key, response.clone()); + Ok((*response).clone()) +} + +async fn compute_prefix_usage( + store: Arc, + bucket: &str, + prefix: &str, + max_entries: usize, +) -> BucketPrefixUsageResponse { + let sets: Vec> = store.all_set_disks(); + let sets_total = sets.len(); + let cache_name = format!("{bucket}/{DATA_USAGE_CACHE_NAME}"); + + let per_set = join_all(sets.into_iter().map(|set| { + let cache_name = cache_name.clone(); + async move { + let mut cache = DataUsageCache::default(); + // A set that has never scanned this bucket (or cannot be read + // within the budget) reports nothing — the remaining sets still + // produce a usable, flagged answer. + let loaded = match tokio::time::timeout(PER_SET_LOAD_TIMEOUT, cache.load(set, &cache_name)).await { + Ok(Ok(())) => cache, + Ok(Err(err)) => { + debug!( + target: "rustfs::scanner::prefix_usage", + event = EVENT_PREFIX_USAGE_CACHE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_PREFIX_USAGE, + bucket = %bucket, + state = "set_load_failed", + error = %err, + "Prefix usage set cache load failed" + ); + return None; + } + Err(_) => { + warn!( + target: "rustfs::scanner::prefix_usage", + event = EVENT_PREFIX_USAGE_CACHE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_PREFIX_USAGE, + bucket = %bucket, + state = "set_load_timeout", + "Prefix usage set cache load timed out" + ); + return None; + } + }; + if loaded.info.name != bucket { + // Empty or stale-scoped cache: this set has no data for the bucket. + return None; + } + let last_update = loaded.info.last_update; + let query = loaded.prefix_usage(bucket, prefix, max_entries); + Some((query, last_update)) + } + })) + .await; + + let mut usage = PrefixUsageSummary::default(); + let mut sub_prefix_map: HashMap = HashMap::new(); + let mut sets_reporting = 0usize; + let mut reporting_but_absent = 0usize; + let mut any_compacted = false; + let mut all_compacted = true; + let mut truncated = false; + let mut last_update: Option = None; + + for (query, set_last_update) in per_set.into_iter().flatten() { + // last_update counts every set that has scanned the bucket, even + // when the prefix itself is absent on that set. + if let Some(set_last_update) = set_last_update + && last_update.map(|current| set_last_update > current).unwrap_or(true) + { + last_update = Some(set_last_update); + } + let Some(query) = query else { + // The set knows the bucket but not this prefix: legitimate when + // the prefix's objects all hash to other sets, but it means the + // breakdown below cannot attribute that set's (zero) objects. + reporting_but_absent += 1; + continue; + }; + sets_reporting += 1; + usage.merge(&query.usage); + if query.compacted { + any_compacted = true; + } else { + all_compacted = false; + } + truncated |= query.truncated; + for entry in query.sub_prefixes { + sub_prefix_map.entry(entry.prefix).or_default().merge(&entry.usage); + } + } + + let mut sub_prefixes: Vec = sub_prefix_map + .into_iter() + .map(|(prefix, usage)| PrefixUsageEntry { prefix, usage }) + .collect(); + sub_prefixes.sort_by(|left, right| { + right + .usage + .size + .cmp(&left.usage.size) + .then_with(|| left.prefix.cmp(&right.prefix)) + }); + // Merged rows can exceed max_entries only when per-set truncation + // already flagged; enforce the caller bound on the merged view too. + if sub_prefixes.len() > max_entries { + truncated = true; + sub_prefixes.truncate(max_entries); + } + + let found = sets_reporting > 0; + BucketPrefixUsageResponse { + bucket: bucket.to_string(), + prefix: prefix.to_string(), + usage, + compacted: found && all_compacted, + sub_prefixes_partial: any_compacted || reporting_but_absent > 0, + truncated, + sub_prefixes, + sets_reporting, + sets_total, + last_update_unix_secs: last_update + .and_then(|time| time.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map(|dur| dur.as_secs()), + } +} + +fn lookup_cached(key: &(String, String, usize)) -> Option> { + let mut guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner()); + let map = guard.as_mut()?; + let cached = map.get(key)?; + if cached.computed_at.elapsed() > CACHE_TTL { + map.remove(key); + return None; + } + Some(cached.response.clone()) +} + +fn store_cached(key: (String, String, usize), response: Arc) { + let mut guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner()); + let map = guard.get_or_insert_with(HashMap::new); + // Bound the cache: drop expired entries first, and if the cap is still + // exceeded clear wholesale — the next queries recompute in milliseconds. + if map.len() >= CACHE_MAX_ENTRIES { + map.retain(|_, cached| cached.computed_at.elapsed() <= CACHE_TTL); + if map.len() >= CACHE_MAX_ENTRIES { + map.clear(); + } + } + map.insert( + key, + CachedResponse { + computed_at: std::time::Instant::now(), + response, + }, + ); +} + +#[cfg(test)] +mod tests { + use super::{CACHE_MAX_ENTRIES, PREFIX_USAGE_CACHE, invalidate_prefix_usage_cache, store_cached}; + use rustfs_data_usage::PrefixUsageSummary; + + fn response(bucket: &str) -> super::BucketPrefixUsageResponse { + super::BucketPrefixUsageResponse { + bucket: bucket.to_string(), + prefix: String::new(), + usage: PrefixUsageSummary::default(), + compacted: false, + sub_prefixes_partial: false, + truncated: false, + sub_prefixes: Vec::new(), + sets_reporting: 1, + sets_total: 1, + last_update_unix_secs: None, + } + } + + fn seed(bucket: &str, prefix: &str) { + store_cached( + (bucket.to_ascii_lowercase(), prefix.to_string(), 10), + std::sync::Arc::new(response(bucket)), + ); + } + + fn contains(bucket: &str, prefix: &str) -> bool { + PREFIX_USAGE_CACHE + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .as_ref() + .is_some_and(|map| map.contains_key(&(bucket.to_ascii_lowercase(), prefix.to_string(), 10))) + } + + /// All cache tests run inside one test to keep the process-global map + /// free of cross-test ordering (the flake class this module avoids). + #[test] + fn invalidation_scopes_to_bucket_and_cache_stays_bounded() { + invalidate_prefix_usage_cache(""); + seed("alpha", "x"); + seed("beta", "y"); + + // Case-insensitive bucket scoping. + invalidate_prefix_usage_cache("ALPHA"); + assert!(!contains("alpha", "x")); + assert!(contains("beta", "y")); + + // Wholesale clear. + invalidate_prefix_usage_cache(""); + assert!(!contains("beta", "y")); + + // Hard cap: overflow clears rather than grows. + for index in 0..=(CACHE_MAX_ENTRIES / 2) { + let bucket = format!("cap-bucket-{index}"); + seed(&bucket, "a"); + seed(&bucket, "b"); + } + let guard = PREFIX_USAGE_CACHE.lock().unwrap_or_else(|poison| poison.into_inner()); + let map = guard.as_ref().expect("seeded"); + assert!(map.len() <= CACHE_MAX_ENTRIES, "cache must stay bounded, got {}", map.len()); + } +} diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 15ee9cca0..f722ff186 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -231,6 +231,10 @@ pub fn record_dirty_usage_bucket(bucket: &str) { dirty_buckets.len() }; global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets)); + // A write invalidates this bucket's prefix-usage answers on the spot so + // admin/console consumers never ride the full TTL after a change + // (rustfs/backlog#1872). + crate::prefix_usage::invalidate_prefix_usage_cache(bucket); DIRTY_USAGE_BUCKET_NOTIFY.notify_one(); } diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 8c837eb05..f0a32f402 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -64,6 +64,7 @@ mod target_descriptor; pub mod tier; pub mod tls_debug; pub mod trace; +pub mod usage_prefix; pub mod user; pub mod user_iam; pub mod user_lifecycle; diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index c7ad93b54..0909568e5 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -1158,10 +1158,10 @@ impl Operation for RuntimeCapabilitiesHandler { } } -/// Authorization gate for GET datausageinfo: any-of the dedicated admin action -/// OR the bucket listing action. Pinned by a unit test so the gate cannot -/// silently narrow or widen (rustfs/backlog#1306). -fn data_usage_info_gate_actions() -> Vec { +/// Authorization gate for GET datausageinfo (and prefix usage): any-of the +/// dedicated admin action OR the bucket listing action. Pinned by a unit test +/// so the gate cannot silently narrow or widen (rustfs/backlog#1306). +pub(crate) fn data_usage_info_gate_actions() -> Vec { vec![ Action::AdminAction(AdminAction::DataUsageInfoAdminAction), Action::S3Action(S3Action::ListBucketAction), diff --git a/rustfs/src/admin/handlers/usage_prefix.rs b/rustfs/src/admin/handlers/usage_prefix.rs new file mode 100644 index 000000000..10cce50d5 --- /dev/null +++ b/rustfs/src/admin/handlers/usage_prefix.rs @@ -0,0 +1,142 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Prefix-level bucket usage admin handler (rustfs/backlog#1872). +//! +//! `GET /rustfs/admin/v3/usage/{bucket}?prefix=&max-entries=` answers +//! "what does this bucket / this prefix hold" from the scanner's per-set +//! usage caches, with a one-level sub-prefix breakdown — the data console +//! buckets view MinIO serves from `loadPrefixUsageFromBackend`. + +use crate::admin::auth::validate_admin_request; +use crate::admin::handlers::system::data_usage_info_gate_actions; +use crate::admin::router::{AdminOperation, Operation, S3Router}; +use crate::auth::{check_key_valid, get_session_token}; +use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use http::{HeaderMap, HeaderValue, StatusCode}; +use hyper::Method; +use matchit::Params; +use s3s::header::CONTENT_TYPE; +use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; + +const JSON_CONTENT_TYPE: &str = "application/json"; +const DEFAULT_MAX_ENTRIES: usize = 1000; +const MAX_ENTRIES_LIMIT: usize = 10_000; + +pub struct BucketPrefixUsageHandler {} + +pub fn register_usage_prefix_route(r: &mut S3Router) -> std::io::Result<()> { + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/usage/{bucket}").as_str(), + AdminOperation(&BucketPrefixUsageHandler {}), + )?; + Ok(()) +} + +/// Parse `prefix` and `max-entries` from the query string. Unknown keys are +/// rejected so a typo'd parameter cannot silently change the answer's shape. +fn parse_usage_prefix_query(query: Option<&str>) -> S3Result<(String, usize)> { + let mut prefix: Option = None; + let mut max_entries: Option = None; + for (key, value) in url::form_urlencoded::parse(query.unwrap_or_default().as_bytes()) { + match key.as_ref() { + "prefix" => prefix = Some(value.into_owned()), + "max-entries" => { + max_entries = Some( + value + .parse::() + .map_err(|_| s3_error!(InvalidArgument, "max-entries must be a positive integer"))?, + ); + } + other => return Err(s3_error!(InvalidArgument, "unknown query parameter: {other}")), + } + } + let max_entries = max_entries.unwrap_or(DEFAULT_MAX_ENTRIES).clamp(1, MAX_ENTRIES_LIMIT); + Ok((prefix.unwrap_or_default(), max_entries)) +} + +#[async_trait::async_trait] +impl Operation for BucketPrefixUsageHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let Some(input_cred) = req.credentials else { + return Err(s3_error!(InvalidRequest, "get cred failed")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + let remote_addr = req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)); + validate_admin_request(&req.headers, &cred, owner, false, data_usage_info_gate_actions(), remote_addr).await?; + + let bucket = params.get("bucket").unwrap_or_default().to_string(); + if bucket.is_empty() { + return Err(s3_error!(InvalidRequest, "bucket path parameter is required")); + } + let (prefix, max_entries) = parse_usage_prefix_query(req.uri.query())?; + + // Authorization is bucket-scoped by the same any-of gate as the + // datausageinfo route; the bucket name itself is validated by the + // scanner layer, which rejects reserved/invalid names. + let response = rustfs_scanner::bucket_prefix_usage(&bucket, &prefix, max_entries) + .await + .map_err(|err| s3_error!(InvalidArgument, "{}", err))?; + + let data = serde_json::to_vec(&response) + .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "parse prefix usage failed"))?; + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, HeaderValue::from_static(JSON_CONTENT_TYPE)); + + Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header)) + } +} + +#[cfg(test)] +mod tests { + use super::{DEFAULT_MAX_ENTRIES, MAX_ENTRIES_LIMIT, parse_usage_prefix_query}; + use s3s::S3Error; + + fn query(raw: &str) -> Result<(String, usize), S3Error> { + parse_usage_prefix_query(Some(raw)) + } + + #[test] + fn defaults_apply_when_no_query_is_given() { + assert_eq!(parse_usage_prefix_query(None).unwrap(), (String::new(), DEFAULT_MAX_ENTRIES)); + assert_eq!(query("").unwrap(), (String::new(), DEFAULT_MAX_ENTRIES)); + } + + #[test] + fn prefix_round_trips_url_encoded_characters() { + let (prefix, _) = query("prefix=pre%2Ffix%20name").unwrap(); + assert_eq!(prefix, "pre/fix name"); + } + + #[test] + fn max_entries_parses_and_clamps_to_documented_bounds() { + assert_eq!(query("max-entries=5").unwrap().1, 5); + assert_eq!(query("max-entries=0").unwrap().1, 1, "zero must clamp up, not mean unlimited"); + assert_eq!(query("max-entries=99999999").unwrap().1, MAX_ENTRIES_LIMIT); + assert!(query("max-entries=-3").is_err()); + assert!(query("max-entries=abc").is_err()); + } + + #[test] + fn unknown_parameters_are_rejected_not_ignored() { + assert!( + query("prefixes=x").is_err(), + "a typo'd parameter must fail the request, not widen the query" + ); + } +} diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index d61abf932..fe07d4056 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -40,7 +40,8 @@ use handlers::{ audit, batch_job, bucket_meta, cluster_snapshot, config_admin, diagnostics, durability as durability_handler, extensions, heal, health, idp_compat, ilm_transition, inspect_archive, kms, module_switch, object_data_cache, object_zip_download, oidc, plugins_catalog, plugins_instances, pools, profile_admin, quota as quota_handler, rebalance, - replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, user, + replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, usage_prefix, + user, }; use router::{AdminOperation, S3Router}; use s3s::route::S3Route; @@ -80,6 +81,7 @@ fn register_admin_routes(r: &mut S3Router) -> std::io::Result<() bucket_meta::register_bucket_meta_route(r)?; config_admin::register_config_route(r)?; scanner::register_scanner_route(r)?; + usage_prefix::register_usage_prefix_route(r)?; ilm_transition::register_ilm_transition_route(r)?; object_data_cache::register_object_data_cache_route(r)?; audit::register_audit_target_route(r)?; diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 474d3ad0f..2c1ad5e1b 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -1558,6 +1558,11 @@ pub const DEFERRED_ADMIN_ROUTE_POLICIES: &[DeferredAdminRoutePolicy] = &[ "/rustfs/admin/v3/datausageinfo", DeferredRoutePolicyReason::MultipleActions, ), + deferred( + HttpMethod::Get, + "/rustfs/admin/v3/usage/{bucket}", + DeferredRoutePolicyReason::MultipleActions, + ), deferred( HttpMethod::Post, "/rustfs/admin/v3/object-zip-downloads", diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index f829b8c94..e81f13d17 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -172,6 +172,7 @@ fn expected_admin_route_matrix() -> Vec { admin_route(Method::POST, "/v4/inspect/archive"), admin_route(Method::GET, "/v3/storageinfo"), admin_route(Method::GET, "/v3/datausageinfo"), + admin_route_sample(Method::GET, "/v3/usage/{bucket}", "/v3/usage/test-bucket"), admin_route(Method::GET, "/v3/metrics"), admin_route(Method::GET, "/v3/object-data-cache/stats"), admin_route(Method::POST, "/v3/object-data-cache/flush"), From beb6e1383e9982dac6a1b06f8e7a2af735336094 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Tue, 18 Aug 2026 01:21:15 +0800 Subject: [PATCH 06/22] feat(helm): add TLSRoute passthrough support for gateway api (#6169) Add an optional TLS passthrough listener to the Gateway API support. When gatewayApi.listeners.tls.enabled is true, the Gateway gets a TLS listener with tls.mode: Passthrough and a TLSRoute is rendered to the RustFS service so TLS terminates at the backend (end-to-end encryption). Refs rustfs/rustfs#3862. --- helm/README.md | 6 +++++ helm/rustfs/templates/gateway-api/gateway.yml | 10 ++++++++ .../rustfs/templates/gateway-api/tlsroute.yml | 25 +++++++++++++++++++ helm/rustfs/values.yaml | 6 +++++ 4 files changed, 47 insertions(+) create mode 100644 helm/rustfs/templates/gateway-api/tlsroute.yml diff --git a/helm/README.md b/helm/README.md index 8bdd8b531..79b7c6fd6 100644 --- a/helm/README.md +++ b/helm/README.md @@ -273,6 +273,10 @@ uer. `ClusterIssuer` or `Issuer`. | | gatewayApi.listeners.http.port| int | `8000` | Gateway API http listener port. | | gatewayApi.listeners.https.name | string | `websecure` | Gateway API https listener name. | | gatewayApi.listeners.https.port| int | `8443` | Gateway API https listener port. | +| gatewayApi.listeners.tls.enabled | bool | `false` | Enable a TLS passthrough listener and generate a TLSRoute. | +| gatewayApi.listeners.tls.name | string | `tls` | Gateway API TLS passthrough listener name. | +| gatewayApi.listeners.tls.port | int | `443` | Gateway API TLS passthrough listener port. | +| gatewayApi.listeners.tls.backendPort | int | `null` | Backend service port that terminates TLS; defaults to the console port. | | gatewayApi.hostname | string | Hostname to access RustFS via gateway api. | | gatewayApi.secretName | string | Secret tls to via RustFS using HTTPS. | | gatewayApi.existingGateway.name | string | `""` | The existing gateway name, instead of creating a new one. | @@ -447,6 +451,8 @@ rustfs-route ["example.rustfs.com"] 172m Then, via RustFS instance via `https://example.rustfs.com` or `http://example.rustfs.com`. +For end-to-end encryption, set `gatewayApi.listeners.tls.enabled` to `true`. The chart then adds a `TLS` listener with `tls.mode: Passthrough` to the `Gateway` and generates a `TLSRoute` that forwards the encrypted stream to the RustFS service, where TLS is terminated on the backend side. Note that backend TLS termination must be configured on RustFS itself (for example `RUSTFS_TLS_PATH` pointing to server certificates), and the installed Gateway API CRDs must include `TLSRoute`. + # Uninstall Uninstalling the rustfs installation with command, diff --git a/helm/rustfs/templates/gateway-api/gateway.yml b/helm/rustfs/templates/gateway-api/gateway.yml index 207637b3e..8de988beb 100644 --- a/helm/rustfs/templates/gateway-api/gateway.yml +++ b/helm/rustfs/templates/gateway-api/gateway.yml @@ -26,5 +26,15 @@ spec: - name: {{ include "rustfs.fullname" $ }}-tls kind: Secret {{- end }} + {{- if .tls.enabled }} + - name: {{ .tls.name }} + port: {{ .tls.port }} + protocol: TLS + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: Same + {{- end }} {{- end }} {{- end }} diff --git a/helm/rustfs/templates/gateway-api/tlsroute.yml b/helm/rustfs/templates/gateway-api/tlsroute.yml new file mode 100644 index 000000000..e35f86c06 --- /dev/null +++ b/helm/rustfs/templates/gateway-api/tlsroute.yml @@ -0,0 +1,25 @@ +{{- if and .Values.gatewayApi.enabled .Values.gatewayApi.listeners.tls.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: TLSRoute +metadata: + name: {{ include "rustfs.fullname" . }}-tlsroute + namespace: {{ .Release.Namespace }} +spec: + parentRefs: + {{- if .Values.gatewayApi.existingGateway.name }} + - name: {{ .Values.gatewayApi.existingGateway.name }} + {{- if .Values.gatewayApi.existingGateway.namespace }} + namespace: {{ .Values.gatewayApi.existingGateway.namespace }} + {{- end }} + sectionName: {{ .Values.gatewayApi.listeners.tls.name }} + {{- else }} + - name: {{ include "rustfs.fullname" $ }}-gateway + sectionName: {{ .Values.gatewayApi.listeners.tls.name }} + {{- end }} + hostnames: + - {{ .Values.gatewayApi.hostname }} + rules: + - backendRefs: + - name: {{ include "rustfs.fullname" . }}-svc + port: {{ .Values.gatewayApi.listeners.tls.backendPort | default .Values.service.console.port }} +{{- end }} diff --git a/helm/rustfs/values.yaml b/helm/rustfs/values.yaml index 19826df96..d57317243 100644 --- a/helm/rustfs/values.yaml +++ b/helm/rustfs/values.yaml @@ -369,6 +369,12 @@ gatewayApi: https: name: websecure port: 8443 + tls: # Optional TLS passthrough listener; renders a TLSRoute so TLS terminates at the RustFS backend. + enabled: false + name: tls + port: 443 + # Service port that terminates TLS on the backend; defaults to the console port. + backendPort: null hostname: example.rustfs.com httpToHttpsRedirect: true existingGateway: From 7cb91a019084be368bbcaa750306628c1a127dce Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 07:51:56 +0800 Subject: [PATCH 07/22] chore(ecstore): adjudicate 32 bare dead_code allows (#6173) Replace every bare `#[allow(dead_code)]` in ecstore with either a deletion or a per-item allow carrying a `reason`. Blanket allows at module, struct, and impl level silence the lint for future members too, so each is narrowed to the members that are actually dead. Delete the dead cluster in `config/heal.rs` (`Config`, its three methods, `RUSTFS_BITROT_CYCLE_IN_MONTHS`, `parse_bitrot_config`) rather than annotate it: it has no callers and is unreachable outside the crate, and `parse_bitrot_config` would panic on its disabled path via `Duration::from_secs_f64(-1.0)`. `DEFAULT_KVS` stays, since the config registry uses it. Correct two `reason` strings on `Checksum::new` and `PutObjReader::md5_current_hex_string`, which are methods but carried a field-only rationale. Refs backlog#1823 Co-authored-by: houseme --- .../ecstore/src/bucket/bucket_target_sys.rs | 4 +- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 9 ++- .../bucket/lifecycle/tier_last_day_stats.rs | 5 +- .../src/bucket/lifecycle/tier_sweeper.rs | 10 +++- crates/ecstore/src/bucket/quota/mod.rs | 2 - crates/ecstore/src/client/api_get_object.rs | 4 +- crates/ecstore/src/client/api_get_options.rs | 1 - crates/ecstore/src/client/api_list.rs | 1 - crates/ecstore/src/client/api_put_object.rs | 4 +- crates/ecstore/src/client/api_remove.rs | 5 +- crates/ecstore/src/client/checksum.rs | 6 +- crates/ecstore/src/client/object_api_utils.rs | 3 +- crates/ecstore/src/config/audit.rs | 3 - crates/ecstore/src/config/heal.rs | 59 ------------------- crates/ecstore/src/config/mod.rs | 1 - crates/ecstore/src/core/pools.rs | 12 +++- crates/ecstore/src/disk/local.rs | 2 +- .../ecstore/src/services/rebalance/types.rs | 1 - .../ecstore/src/services/tier/tier_config.rs | 17 +++++- crates/ecstore/src/services/tier/tier_gen.rs | 1 - 20 files changed, 59 insertions(+), 91 deletions(-) diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index 0a9b0f41c..00cf124f1 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -1549,8 +1549,8 @@ impl Default for PutObjectOptions { } } -#[allow(dead_code)] impl PutObjectOptions { + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn set_match_etag(&mut self, etag: &str) { if etag == "*" { self.custom_header @@ -1561,6 +1561,7 @@ impl PutObjectOptions { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn set_match_etag_except(&mut self, etag: &str) { if etag == "*" { self.custom_header @@ -1696,6 +1697,7 @@ impl PutObjectOptions { header } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn validate(&self, _c: Arc) -> Result<(), std::io::Error> { //if self.checksum.is_set() { /*if !self.trailing_header_support { diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 0366551e0..ec5aaf74e 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -456,16 +456,23 @@ impl<'a> LifecycleExpiryTrace<'a> { } } -#[allow(dead_code)] impl ExpiryStats { pub fn missed_tasks(&self) -> i64 { self.missed_expiry_tasks.load(Ordering::SeqCst) } + #[allow( + dead_code, + reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)" + )] fn missed_free_vers_tasks(&self) -> i64 { self.missed_freevers_tasks.load(Ordering::SeqCst) } + #[allow( + dead_code, + reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)" + )] fn missed_tier_journal_tasks(&self) -> i64 { self.missed_tier_journal_tasks.load(Ordering::SeqCst) } diff --git a/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs b/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs index b32fb32f5..2d8d6c2ea 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs @@ -80,7 +80,10 @@ impl LastDayTierStats { } } - #[allow(dead_code)] + #[allow( + dead_code, + reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)" + )] fn merge(&self, m: LastDayTierStats) -> LastDayTierStats { let mut cl = self.clone(); let mut cm = m; diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index 2ce68fd03..0bde1a23c 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -177,9 +177,10 @@ fn should_record_remote_delete_failure(err: &std::io::Error) -> bool { } #[derive(Default)] -#[allow(dead_code)] struct ObjSweeper { + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] object: String, + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] bucket: String, version_id: Option, versioned: bool, @@ -191,9 +192,9 @@ struct ObjSweeper { remote_object: String, } -#[allow(dead_code)] impl ObjSweeper { #[allow(clippy::new_ret_no_self)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub async fn new(bucket: &str, object: &str) -> Result { Ok(Self { object: object.into(), @@ -202,17 +203,20 @@ impl ObjSweeper { }) } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn with_version(&mut self, vid: Option) -> &Self { self.version_id = vid.clone(); self } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn with_versioning(&mut self, versioned: bool, suspended: bool) -> &Self { self.versioned = versioned; self.suspended = suspended; self } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn get_opts(&self) -> lifecycle::ObjectOpts { let mut opts = ObjectOpts { version_id: self.version_id.clone(), @@ -226,6 +230,7 @@ impl ObjSweeper { opts } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub fn set_transition_state(&mut self, info: TransitionedObject) { self.transition_tier = info.tier; self.transition_status = info.status; @@ -266,6 +271,7 @@ impl ObjSweeper { None } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] pub async fn sweep(&self, api: Arc) { let Some(je) = self.should_remove_remote_object() else { return; diff --git a/crates/ecstore/src/bucket/quota/mod.rs b/crates/ecstore/src/bucket/quota/mod.rs index 0ce5a3a52..3fed86728 100644 --- a/crates/ecstore/src/bucket/quota/mod.rs +++ b/crates/ecstore/src/bucket/quota/mod.rs @@ -312,9 +312,7 @@ mod tests { } #[derive(Deserialize)] struct LegacyBucketQuota { - #[allow(dead_code)] quota: Option, - #[allow(dead_code)] quota_type: LegacyQuotaType, } let legacy = serde_json::from_slice::(&json) diff --git a/crates/ecstore/src/client/api_get_object.rs b/crates/ecstore/src/client/api_get_object.rs index a9c69aad0..1eb6122ca 100644 --- a/crates/ecstore/src/client/api_get_object.rs +++ b/crates/ecstore/src/client/api_get_object.rs @@ -95,7 +95,6 @@ impl TransitionClient { } #[derive(Default)] -#[allow(dead_code)] pub struct GetRequest { pub buffer: Vec, pub offset: i64, @@ -107,11 +106,12 @@ pub struct GetRequest { pub setting_object_info: bool, } -#[allow(dead_code)] pub struct GetResponse { pub size: i64, //pub error: error, + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] pub did_read: bool, + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] pub object_info: ObjectInfo, } diff --git a/crates/ecstore/src/client/api_get_options.rs b/crates/ecstore/src/client/api_get_options.rs index 622c5a4c2..503b44f3a 100644 --- a/crates/ecstore/src/client/api_get_options.rs +++ b/crates/ecstore/src/client/api_get_options.rs @@ -27,7 +27,6 @@ use tracing::warn; use crate::client::api_error_response::err_invalid_argument; #[derive(Default)] -#[allow(dead_code)] pub struct AdvancedGetOptions { pub replication_delete_marker: bool, pub is_replication_ready_for_delete_marker: bool, diff --git a/crates/ecstore/src/client/api_list.rs b/crates/ecstore/src/client/api_list.rs index 6bd8591c3..a7f894a00 100644 --- a/crates/ecstore/src/client/api_list.rs +++ b/crates/ecstore/src/client/api_list.rs @@ -360,7 +360,6 @@ impl TransitionClient { } #[derive(Default)] -#[allow(dead_code)] pub struct ListObjectsOptions { reverse_versions: bool, with_versions: bool, diff --git a/crates/ecstore/src/client/api_put_object.rs b/crates/ecstore/src/client/api_put_object.rs index 5fbc3fd2c..bf9efe4a2 100644 --- a/crates/ecstore/src/client/api_put_object.rs +++ b/crates/ecstore/src/client/api_put_object.rs @@ -137,8 +137,8 @@ impl Default for PutObjectOptions { } } -#[allow(dead_code)] impl PutObjectOptions { + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn set_match_etag(&mut self, etag: &str) { if etag == "*" { self.custom_header.insert("If-Match", HeaderValue::from_static("*")); @@ -149,6 +149,7 @@ impl PutObjectOptions { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn set_match_etag_except(&mut self, etag: &str) { if etag == "*" { self.custom_header.insert("If-None-Match", HeaderValue::from_static("*")); @@ -259,6 +260,7 @@ impl PutObjectOptions { header } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> { //if self.checksum.is_set() { /*if !self.trailing_header_support { diff --git a/crates/ecstore/src/client/api_remove.rs b/crates/ecstore/src/client/api_remove.rs index 25c71d80a..573a2eed1 100644 --- a/crates/ecstore/src/client/api_remove.rs +++ b/crates/ecstore/src/client/api_remove.rs @@ -55,7 +55,6 @@ pub struct RemoveBucketOptions { const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024; #[derive(Debug)] -#[allow(dead_code)] pub struct AdvancedRemoveOptions { pub replication_delete_marker: bool, pub replication_status: ReplicationStatus, @@ -465,10 +464,10 @@ impl TransitionClient { } #[derive(Debug, Default)] -#[allow(dead_code)] pub struct RemoveObjectError { + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] object_name: String, - #[allow(dead_code)] + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] version_id: String, err: Option, } diff --git a/crates/ecstore/src/client/checksum.rs b/crates/ecstore/src/client/checksum.rs index c71394210..7bb96a6a3 100644 --- a/crates/ecstore/src/client/checksum.rs +++ b/crates/ecstore/src/client/checksum.rs @@ -372,8 +372,8 @@ pub struct Checksum { computed: bool, } -#[allow(dead_code)] impl Checksum { + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn new(t: ChecksumMode, b: &[u8]) -> Checksum { if t.is_set() && b.len() == t.raw_byte_len() { return Checksum { @@ -385,7 +385,7 @@ impl Checksum { Checksum::default() } - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn new_checksum_string(t: ChecksumMode, s: &str) -> Result { let b = match base64_decode(s.as_bytes()) { Ok(b) => b, @@ -412,7 +412,7 @@ impl Checksum { base64_encode(&self.r) } - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn raw(&self) -> Option> { if !self.is_set() { return None; diff --git a/crates/ecstore/src/client/object_api_utils.rs b/crates/ecstore/src/client/object_api_utils.rs index 484233fad..b4bd3e0b5 100644 --- a/crates/ecstore/src/client/object_api_utils.rs +++ b/crates/ecstore/src/client/object_api_utils.rs @@ -37,16 +37,17 @@ pub struct PutObjReader { //pub sealMD5Fn: SealMD5CurrFn, } -#[allow(dead_code)] impl PutObjReader { pub fn new(reader: HashReader) -> Self { Self { reader } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn md5_current_hex_string(&self) -> String { self.reader.checksum().map(|v| v.encoded).unwrap_or_default() } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> { self.reader = enc_reader; diff --git a/crates/ecstore/src/config/audit.rs b/crates/ecstore/src/config/audit.rs index b7ed83323..63c63b700 100644 --- a/crates/ecstore/src/config/audit.rs +++ b/crates/ecstore/src/config/audit.rs @@ -39,7 +39,6 @@ use rustfs_config::{ }; use std::sync::LazyLock; -#[allow(dead_code)] #[allow(clippy::declare_interior_mutable_const)] /// Default KVS for audit webhook settings. pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock = LazyLock::new(|| { @@ -117,7 +116,6 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock = LazyLock::new(|| { ]) }); -#[allow(dead_code)] #[allow(clippy::declare_interior_mutable_const)] /// Default KVS for audit MQTT settings. pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock = LazyLock::new(|| { @@ -375,7 +373,6 @@ pub static DEFAULT_AUDIT_NATS_KVS: LazyLock = LazyLock::new(|| { ]) }); -#[allow(dead_code)] pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock = LazyLock::new(|| { KVS(vec![ KV { diff --git a/crates/ecstore/src/config/heal.rs b/crates/ecstore/src/config/heal.rs index 4505d3ce0..8ef41efa1 100644 --- a/crates/ecstore/src/config/heal.rs +++ b/crates/ecstore/src/config/heal.rs @@ -12,12 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::error::{Error, Result}; use rustfs_config::server_config::{KV, KVS}; use rustfs_config::{DEFAULT_HEAL_BITROT_CYCLE_SECS, HEAL_BITROT_CYCLE}; -use rustfs_utils::string::parse_bool; use std::sync::LazyLock; -use std::time::Duration; pub static DEFAULT_KVS: LazyLock = LazyLock::new(|| { KVS(vec![KV { @@ -26,59 +23,3 @@ pub static DEFAULT_KVS: LazyLock = LazyLock::new(|| { hidden_if_empty: false, }]) }); - -#[derive(Debug, Default)] -pub struct Config { - pub bitrot: String, - pub sleep: Duration, - pub io_count: usize, - pub drive_workers: usize, - pub cache: Duration, -} - -impl Config { - pub fn bitrot_scan_cycle(&self) -> Duration { - self.cache - } - - pub fn get_workers(&self) -> usize { - self.drive_workers - } - - pub fn update(&mut self, nopts: &Config) { - self.bitrot = nopts.bitrot.clone(); - self.io_count = nopts.io_count; - self.sleep = nopts.sleep; - self.drive_workers = nopts.drive_workers; - } -} - -const RUSTFS_BITROT_CYCLE_IN_MONTHS: u64 = 1; - -fn parse_bitrot_config(s: &str) -> Result { - match parse_bool(s) { - Ok(enabled) => { - if enabled { - Ok(Duration::from_secs_f64(0.0)) - } else { - Ok(Duration::from_secs_f64(-1.0)) - } - } - Err(_) => { - if !s.ends_with("m") { - return Err(Error::other("unknown format")); - } - - match s.trim_end_matches('m').parse::() { - Ok(months) => { - if months < RUSTFS_BITROT_CYCLE_IN_MONTHS { - return Err(Error::other(format!("minimum bitrot cycle is {RUSTFS_BITROT_CYCLE_IN_MONTHS} month(s)"))); - } - - Ok(Duration::from_secs(months * 30 * 24 * 60)) - } - Err(err) => Err(Error::other(err)), - } - } - } -} diff --git a/crates/ecstore/src/config/mod.rs b/crates/ecstore/src/config/mod.rs index 5e6e3a3e3..ff97d03e0 100644 --- a/crates/ecstore/src/config/mod.rs +++ b/crates/ecstore/src/config/mod.rs @@ -16,7 +16,6 @@ mod audit; pub mod com; -#[allow(dead_code)] pub mod heal; mod notify; mod oidc; diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index f35017760..ec9da3708 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -1996,11 +1996,11 @@ impl PoolMeta { Ok(false) } - #[allow(dead_code)] pub fn validate(&self, pools: Vec>) -> Result { struct PoolInfo { position: usize, completed: bool, + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] decom_started: bool, } @@ -4958,13 +4958,19 @@ fn is_disk_online_state(state: &str) -> bool { } #[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")] -#[allow(dead_code)] +#[allow( + dead_code, + reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)" +)] fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize { fallback_total_capacity_dedup(disks) } #[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")] -#[allow(dead_code)] +#[allow( + dead_code, + reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)" +)] fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize { fallback_free_capacity_dedup(disks) } diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 2cab189aa..1932744e8 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -6562,7 +6562,7 @@ impl LocalDisk { Ok(f) } - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn get_metrics(&self) -> DiskMetrics { DiskMetrics::default() } diff --git a/crates/ecstore/src/services/rebalance/types.rs b/crates/ecstore/src/services/rebalance/types.rs index b43e075b8..5f79275dc 100644 --- a/crates/ecstore/src/services/rebalance/types.rs +++ b/crates/ecstore/src/services/rebalance/types.rs @@ -132,7 +132,6 @@ impl RebalanceStopPropagationRecord { } } -#[allow(dead_code)] #[derive(Debug, Clone, Default)] pub struct DiskStat { pub total_space: u64, diff --git a/crates/ecstore/src/services/tier/tier_config.rs b/crates/ecstore/src/services/tier/tier_config.rs index a5be98866..149f96ef6 100644 --- a/crates/ecstore/src/services/tier/tier_config.rs +++ b/crates/ecstore/src/services/tier/tier_config.rs @@ -16,8 +16,16 @@ use serde::{Deserialize, Serialize}; use std::{fmt::Display, io}; use tracing::info; +#[allow( + dead_code, + reason = "tier config wire version stamped by the parity constructors below (backlog#1823)" +)] const C_TIER_CONFIG_VER: &str = "v1"; +#[allow( + dead_code, + reason = "tier-name validation message reached only from the parity constructors below (backlog#1823)" +)] const ERR_TIER_NAME_EMPTY: &str = "remote tier name empty"; const WASABI_US_EAST_ENDPOINT: &str = "https://s3.wasabisys.com"; const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[ @@ -264,7 +272,6 @@ impl Clone for TierConfig { } } -#[allow(dead_code)] impl TierConfig { pub(crate) fn clone_with_credentials(&self) -> Self { Self { @@ -284,6 +291,7 @@ impl TierConfig { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn endpoint(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default(), @@ -303,6 +311,7 @@ impl TierConfig { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn bucket(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.bucket.clone()).unwrap_or_default(), @@ -322,6 +331,7 @@ impl TierConfig { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn prefix(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.prefix.clone()).unwrap_or_default(), @@ -341,6 +351,7 @@ impl TierConfig { } } + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn region(&self) -> String { match self.tier_type { TierType::S3 => self.s3.as_ref().map(|s| s.region.clone()).unwrap_or_default(), @@ -457,7 +468,7 @@ impl TierWasabi { } impl TierS3 { - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn create( name: &str, access_key: &str, @@ -528,7 +539,7 @@ pub struct TierMinIO { } impl TierMinIO { - #[allow(dead_code)] + #[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")] fn create( name: &str, endpoint: &str, diff --git a/crates/ecstore/src/services/tier/tier_gen.rs b/crates/ecstore/src/services/tier/tier_gen.rs index 400466b47..63a692b85 100644 --- a/crates/ecstore/src/services/tier/tier_gen.rs +++ b/crates/ecstore/src/services/tier/tier_gen.rs @@ -14,7 +14,6 @@ use crate::services::tier::tier::TierConfigMgr; -#[allow(dead_code)] impl TierConfigMgr { pub fn msg_size(&self) -> usize { 100 From 360bceafced5bd63a39f013f7fc30aef162c50f2 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 08:29:29 +0800 Subject: [PATCH 08/22] feat(heal): add progress and trace observability (#6179) * feat(heal): track erasure set progress baseline Record erasure-set heal byte progress from per-object results and seed progress totals from complete usage-cache snapshots when available. Keep usage-cache failures observational so heal execution continues without a baseline. Co-Authored-By: heihutu * feat(heal): skip filtered erasure set versions Skip erasure-set versions written after the durable heal start time, and queue lifecycle-expired versions for expiry before skipping them. Track new-version and ILM-expired skips separately so progress can explain completed baseline work without treating these skips as retry-blocking failures. Co-Authored-By: heihutu * feat(heal): wire abandoned data-dir cleanup check Connect check_abandoned_parts through ECStore, pool, and set layers so heal can invoke the existing orphan data-dir reclaim path instead of returning NotImplemented. Add dry-run support to the reclaim scan and cover dry-run plus scoped set behavior with regression tests. Co-Authored-By: heihutu * feat(obs): add heal scanner trace bus Introduce an in-process broadcast trace bus with typed heal and scanner events, lazy event construction, and bounded lagged-subscriber behavior. Cover zero-subscriber publishing, subscription delivery, drop accounting, and lagged receivers with focused common-crate tests. Co-Authored-By: heihutu * feat(obs): stream heal trace events from admin API Wire the admin trace endpoint to the common trace bus for heal/scanner events, including kind, regex, and threshold filtering. Co-Authored-By: heihutu * feat(obs): emit heal trace events Publish heal task lifecycle and abandoned-parts cleanup events through the common trace bus so the admin trace stream has live heal diagnostics. Co-Authored-By: heihutu * feat(obs): emit scanner trace events Publish scanner folder, lifecycle action, and heal-candidate events through the common trace bus for live admin scanner diagnostics. Co-Authored-By: heihutu * fix(heal): route data usage loader through storage api Keep ECStore data-usage facade access behind the heal storage_api boundary so architecture migration guards can validate the heal progress path. Co-Authored-By: heihutu * perf(heal): avoid lifecycle snapshots on ordinary heal pages Only request lifecycle object snapshots when the heal pass has lifecycle expiry context. This keeps ordinary listing and disk-walk pages from cloning FileInfo/ObjectInfo payloads while preserving the skip path that queues expired versions. Co-Authored-By: heihutu * test(heal): update bug-fix mocks for lifecycle snapshots Carry the lifecycle snapshot opt-in argument through the remaining heal bug-fix test mocks so all-targets clippy covers the updated storage trait. Co-Authored-By: heihutu * test(rustfs): sync heal storage mock signature Update the rustfs storage RPC test mock for the lifecycle snapshot opt-in argument and cover it with rustfs all-targets clippy. Co-Authored-By: heihutu * test(e2e): allocate smoke ports across nextest processes Serialize E2E port selection with a small /tmp allocator so nextest workers do not reuse the same just-released ephemeral port before RustFS binds it. Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- Cargo.lock | 1 + crates/common/Cargo.toml | 1 + crates/common/src/lib.rs | 1 + crates/common/src/trace_bus.rs | 333 ++++++++++++++ crates/e2e_test/src/common.rs | 83 +++- crates/ecstore/src/api/mod.rs | 1 + crates/ecstore/src/bucket/lifecycle/mod.rs | 2 +- crates/ecstore/src/core/pools.rs | 91 ++++ crates/ecstore/src/core/sets.rs | 29 +- .../src/set_disk/core/io_primitives.rs | 22 + crates/ecstore/src/set_disk/mod.rs | 109 ++++- crates/ecstore/src/set_disk/ops/heal.rs | 61 ++- crates/ecstore/src/set_disk/ops/heal_walk.rs | 50 ++- crates/ecstore/src/store/heal.rs | 42 +- crates/ecstore/src/store/heal_walk.rs | 3 +- crates/heal/src/heal/channel.rs | 1 + crates/heal/src/heal/erasure_healer.rs | 344 ++++++++++++-- crates/heal/src/heal/manager.rs | 30 ++ crates/heal/src/heal/progress.rs | 166 ++++++- crates/heal/src/heal/storage.rs | 176 +++++++- crates/heal/src/heal/storage_api.rs | 13 +- crates/heal/src/heal/task.rs | 238 +++++++++- crates/heal/src/lib.rs | 1 + .../heal_b5_versioned_regression_test.rs | 2 +- .../tests/heal_b920_subquorum_union_test.rs | 6 +- crates/heal/tests/heal_bug_fixes_test.rs | 2 + crates/madmin/src/service_commands.rs | 10 +- crates/scanner/src/scanner_folder.rs | 254 +++++++++++ rustfs/src/admin/handlers/profile_admin.rs | 422 ++++++++++++++++-- rustfs/src/storage/rpc/node_service.rs | 1 + 30 files changed, 2383 insertions(+), 112 deletions(-) create mode 100644 crates/common/src/trace_bus.rs diff --git a/Cargo.lock b/Cargo.lock index b521a731d..85c91d523 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9280,6 +9280,7 @@ dependencies = [ "s3s", "serde", "serde_json", + "smallvec", "tokio", "tonic", "tracing", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 5c9094d61..ddc02cf2e 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -42,6 +42,7 @@ chrono = { workspace = true, features = ["serde"] } jiff = { workspace = true, features = ["serde"] } metrics = { workspace = true } serde = { workspace = true, features = ["derive"] } +smallvec = { workspace = true } rmp-serde = { workspace = true } s3s = { workspace = true, features = ["minio"] } tracing = { workspace = true } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 09240e25b..1ae200f24 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -19,6 +19,7 @@ pub mod last_minute; pub mod metrics; mod readiness; pub mod table_catalog; +pub mod trace_bus; pub use globals::*; pub use readiness::{GlobalReadiness, SystemStage}; diff --git a/crates/common/src/trace_bus.rs b/crates/common/src/trace_bus.rs new file mode 100644 index 000000000..5e6a90d60 --- /dev/null +++ b/crates/common/src/trace_bus.rs @@ -0,0 +1,333 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use smallvec::SmallVec; +use std::{ + sync::{ + Arc, OnceLock, + atomic::{AtomicUsize, Ordering}, + }, + time::{Duration, SystemTime}, +}; +use tokio::sync::broadcast; + +const DEFAULT_TRACE_BUS_CAPACITY: usize = 1024; +const TRACE_ATTR_INLINE_CAPACITY: usize = 8; + +static GLOBAL_TRACE_BUS: OnceLock = OnceLock::new(); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TraceKind { + Heal, + Scanner, +} + +impl TraceKind { + pub const fn as_str(self) -> &'static str { + match self { + Self::Heal => "heal", + Self::Scanner => "scanner", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TraceFunc { + HealTask, + HealBucket, + HealObject, + HealCheckAbandonedParts, + HealErasureSetPage, + ScannerFolder, + ScannerIlmAction, + ScannerHealCandidate, + Dropped, +} + +impl TraceFunc { + pub const fn as_str(self) -> &'static str { + match self { + Self::HealTask => "heal.Task", + Self::HealBucket => "heal.Bucket", + Self::HealObject => "heal.Object", + Self::HealCheckAbandonedParts => "heal.CheckAbandonedParts", + Self::HealErasureSetPage => "heal.ErasureSetPage", + Self::ScannerFolder => "scanner.Folder", + Self::ScannerIlmAction => "scanner.IlmAction", + Self::ScannerHealCandidate => "scanner.HealCandidate", + Self::Dropped => "trace.Dropped", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TraceVal { + Bool(bool), + U64(u64), + I64(i64), + Str(Arc), +} + +impl From for TraceVal { + fn from(value: bool) -> Self { + Self::Bool(value) + } +} + +impl From for TraceVal { + fn from(value: u64) -> Self { + Self::U64(value) + } +} + +impl From for TraceVal { + fn from(value: i64) -> Self { + Self::I64(value) + } +} + +impl From<&str> for TraceVal { + fn from(value: &str) -> Self { + Self::Str(Arc::from(value)) + } +} + +impl From for TraceVal { + fn from(value: String) -> Self { + Self::Str(Arc::from(value)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TraceAttr { + pub key: &'static str, + pub value: TraceVal, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TraceEvent { + pub kind: TraceKind, + pub func: TraceFunc, + pub time: SystemTime, + pub bucket: Option>, + pub object: Option>, + pub duration: Duration, + pub bytes: u64, + pub attrs: SmallVec<[TraceAttr; TRACE_ATTR_INLINE_CAPACITY]>, +} + +impl TraceEvent { + pub fn new(kind: TraceKind, func: TraceFunc) -> Self { + Self { + kind, + func, + time: SystemTime::now(), + bucket: None, + object: None, + duration: Duration::ZERO, + bytes: 0, + attrs: SmallVec::new(), + } + } + + pub fn with_bucket(mut self, bucket: impl Into>) -> Self { + self.bucket = Some(bucket.into()); + self + } + + pub fn with_object(mut self, object: impl Into>) -> Self { + self.object = Some(object.into()); + self + } + + pub fn with_duration(mut self, duration: Duration) -> Self { + self.duration = duration; + self + } + + pub fn with_bytes(mut self, bytes: u64) -> Self { + self.bytes = bytes; + self + } + + pub fn with_attr(mut self, key: &'static str, value: impl Into) -> Self { + self.attrs.push(TraceAttr { + key, + value: value.into(), + }); + self + } +} + +#[derive(Debug)] +pub struct TraceBus { + sender: broadcast::Sender>, + subscriber_count: Arc, +} + +impl TraceBus { + pub fn new(capacity: usize) -> Self { + let capacity = capacity.max(1); + let (sender, _receiver) = broadcast::channel(capacity); + Self { + sender, + subscriber_count: Arc::new(AtomicUsize::new(0)), + } + } + + pub fn subscriber_count(&self) -> usize { + self.subscriber_count.load(Ordering::Acquire) + } + + pub fn subscribe(&self) -> TraceSubscription { + let receiver = self.sender.subscribe(); + self.subscriber_count.fetch_add(1, Ordering::AcqRel); + TraceSubscription { + receiver, + subscriber_count: Arc::clone(&self.subscriber_count), + } + } + + pub fn emit(&self, build: impl FnOnce() -> TraceEvent) -> bool { + if self.subscriber_count() == 0 { + return false; + } + + self.sender.send(Arc::new(build())).is_ok() + } +} + +impl Default for TraceBus { + fn default() -> Self { + Self::new(DEFAULT_TRACE_BUS_CAPACITY) + } +} + +#[derive(Debug)] +pub struct TraceSubscription { + receiver: broadcast::Receiver>, + subscriber_count: Arc, +} + +impl TraceSubscription { + pub async fn recv(&mut self) -> Result, broadcast::error::RecvError> { + self.receiver.recv().await + } + + pub fn try_recv(&mut self) -> Result, broadcast::error::TryRecvError> { + self.receiver.try_recv() + } +} + +impl Drop for TraceSubscription { + fn drop(&mut self) { + self.subscriber_count.fetch_sub(1, Ordering::AcqRel); + } +} + +pub fn global_trace_bus() -> &'static TraceBus { + GLOBAL_TRACE_BUS.get_or_init(TraceBus::default) +} + +pub fn subscribe_trace_events() -> TraceSubscription { + global_trace_bus().subscribe() +} + +pub fn trace_emit(build: impl FnOnce() -> TraceEvent) -> bool { + global_trace_bus().emit(build) +} + +pub fn trace_subscriber_count() -> usize { + global_trace_bus().subscriber_count() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicUsize; + + #[test] + fn trace_emit_skips_builder_without_subscribers() { + let bus = TraceBus::new(4); + let built = AtomicUsize::new(0); + + let sent = bus.emit(|| { + built.fetch_add(1, Ordering::Relaxed); + TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask) + }); + + assert!(!sent); + assert_eq!(built.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn trace_subscriber_receives_event() { + let bus = TraceBus::new(4); + let mut subscription = bus.subscribe(); + + assert!(bus.emit(|| { + TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject) + .with_bucket("bucket") + .with_object("object") + .with_duration(Duration::from_millis(7)) + .with_bytes(11) + .with_attr("dry", true) + })); + + let event = subscription + .recv() + .await + .expect("subscriber should receive emitted trace event"); + + assert_eq!(event.kind, TraceKind::Heal); + assert_eq!(event.func, TraceFunc::HealObject); + assert_eq!(event.bucket.as_deref(), Some("bucket")); + assert_eq!(event.object.as_deref(), Some("object")); + assert_eq!(event.duration, Duration::from_millis(7)); + assert_eq!(event.bytes, 11); + assert_eq!( + event.attrs.as_slice(), + &[TraceAttr { + key: "dry", + value: TraceVal::Bool(true) + }] + ); + } + + #[test] + fn trace_subscription_drop_decrements_count() { + let bus = TraceBus::new(4); + let subscription = bus.subscribe(); + + assert_eq!(bus.subscriber_count(), 1); + drop(subscription); + assert_eq!(bus.subscriber_count(), 0); + } + + #[tokio::test] + async fn lagged_subscriber_drops_events_without_blocking_publishers() { + let bus = TraceBus::new(2); + let mut subscription = bus.subscribe(); + + for index in 0_u64..4 { + assert!(bus.emit(|| { TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder).with_attr("index", index) })); + } + + let err = subscription + .recv() + .await + .expect_err("receiver should observe lag instead of blocking publishers"); + assert!(matches!(err, broadcast::error::RecvError::Lagged(_))); + } +} diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index f1fcaa20a..460c764ed 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -32,6 +32,7 @@ use rustfs_signer::sign_v4; use s3s::Body; use std::ffi::OsStr; use std::fs as stdfs; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; use std::sync::Once; @@ -51,6 +52,11 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] = &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_SCANNER_START_DELAY_SECS", "0")]; pub const TEST_BUCKET: &str = "e2e-test-bucket"; const RUSTFS_FULL_FEATURE: &str = "full"; +const TEST_PORT_MIN: u16 = 20_000; +const TEST_PORT_RANGE: u16 = 40_000; +const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port"; +const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock"; +const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30); fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option { let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy(); @@ -67,6 +73,64 @@ fn configured_capture_log_path(temp_dir: &str) -> Option { capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned()) } +struct PortAllocatorGuard; + +impl PortAllocatorGuard { + async fn acquire() -> Result> { + loop { + match stdfs::create_dir(TEST_PORT_LOCK_DIR) { + Ok(()) => return Ok(Self), + Err(err) if err.kind() == ErrorKind::AlreadyExists => { + remove_stale_port_allocator_lock(); + sleep(Duration::from_millis(10)).await; + } + Err(err) => return Err(err.into()), + } + } + } +} + +impl Drop for PortAllocatorGuard { + fn drop(&mut self) { + let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR); + } +} + +fn advance_test_port(port: u16) -> u16 { + let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE; + TEST_PORT_MIN + offset +} + +fn seeded_test_port() -> u16 { + let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16; + TEST_PORT_MIN + offset +} + +fn read_next_test_port() -> u16 { + stdfs::read_to_string(TEST_PORT_COUNTER_PATH) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port)) + .unwrap_or_else(seeded_test_port) +} + +fn remove_stale_port_allocator_lock() { + let Ok(metadata) = stdfs::metadata(TEST_PORT_LOCK_DIR) else { + return; + }; + let Ok(modified) = metadata.modified() else { + return; + }; + if modified.elapsed().is_ok_and(|elapsed| elapsed > TEST_PORT_LOCK_STALE_AFTER) { + let _ = stdfs::remove_dir(TEST_PORT_LOCK_DIR); + } +} + +fn write_next_test_port(port: u16) -> Result<(), Box> { + stdfs::write(TEST_PORT_COUNTER_PATH, port.to_string())?; + Ok(()) +} + pub(crate) fn capture_command_logs( command: &mut Command, log_path: Option<&str>, @@ -508,10 +572,21 @@ impl RustFSTestEnvironment { /// Find an available port for the test pub async fn find_available_port() -> Result> { use std::net::TcpListener; - let listener = TcpListener::bind("127.0.0.1:0")?; - let port = listener.local_addr()?.port(); - drop(listener); - Ok(port) + let _guard = PortAllocatorGuard::acquire().await?; + let mut next_port = read_next_test_port(); + + for _ in 0..TEST_PORT_RANGE { + let port = next_port; + next_port = advance_test_port(next_port); + write_next_test_port(next_port)?; + + if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) { + drop(listener); + return Ok(port); + } + } + + Err("no available E2E test port found".into()) } /// Kill any existing RustFS processes diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 5526b7d00..4b8bc3249 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -483,6 +483,7 @@ pub mod store_list { } pub mod storage { + pub use crate::core::pools::HealLifecycleExpiryContext; pub use crate::store::HealWalkVersion; pub use crate::store::{ ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks, diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index 823263072..6d8e64f1b 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -19,7 +19,7 @@ pub mod core; pub mod evaluator; pub mod manual_transition_job; mod metadata_boundary; -pub(crate) use metadata_boundary::get_expiry_configs; +pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs}; mod object_lock_boundary; pub use self::core as lifecycle; mod replication_sink; diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index ec9da3708..02fe70f29 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -16,6 +16,7 @@ use crate::bucket::replication::replication_state_from_filemeta; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::bucket::{ lifecycle::{ + LifecycleExpiryConfigs, bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::{ LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle, @@ -2335,6 +2336,10 @@ fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool { ) } +fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool { + action.delete() +} + fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result { if !apply_actions || applied { return Ok(true); @@ -2385,7 +2390,80 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement( } } +pub struct HealLifecycleExpiryContext { + configs: LifecycleExpiryConfigs, +} + impl ECStore { + pub async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result> { + if bucket == RUSTFS_META_BUCKET { + return Ok(None); + } + + let configs = get_expiry_configs(self, bucket).await?; + if configs.lifecycle.is_none() { + return Ok(None); + } + + Ok(Some(HealLifecycleExpiryContext { configs })) + } + + pub async fn enqueue_heal_lifecycle_expiry( + self: &Arc, + context: &HealLifecycleExpiryContext, + bucket: &str, + object: &str, + version_id: Option<&str>, + object_info: Option<&crate::object_api::ObjectInfo>, + ) -> Result { + let Some(lifecycle_config) = context.configs.lifecycle.as_ref() else { + return Ok(false); + }; + + let object_info = if let Some(object_info) = object_info { + if object_info.bucket != bucket || object_info.name != object { + return Ok(false); + } + let snapshot_version_id = object_info + .version_id + .filter(|version_id| !version_id.is_nil()) + .map(|version_id| version_id.to_string()); + if snapshot_version_id.as_deref() != version_id { + return Ok(false); + } + object_info.clone() + } else { + match self + .get_object_info( + bucket, + object, + &ObjectOptions { + version_id: version_id.map(str::to_string), + versioned: version_id.is_some(), + expected_bucket_incarnation_id: Some(context.configs.bucket_incarnation_id), + ..Default::default() + }, + ) + .await + { + Ok(object_info) => object_info, + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => return Ok(false), + Err(err) => return Err(err), + } + }; + + let event = eval_action_from_lifecycle(lifecycle_config, context.configs.object_lock.as_deref(), &object_info).await; + if !lifecycle_action_skips_heal_version(event.action) { + return Ok(false); + } + + if lifecycle_delete_all_versions_blocked_by_replication(self.clone(), bucket, &object_info.name, event.action).await? { + return Ok(false); + } + + Ok(apply_expiry_rule_in(self.clone(), &event, &LcEventSrc::Scanner, &object_info).await) + } + async fn save_current_pool_meta(&self) -> Result<()> { let _save_guard = self.pool_meta_save_gate.lock().await; let snapshot = { @@ -4287,6 +4365,19 @@ mod tests { )); } + #[test] + fn lifecycle_action_skips_heal_version_for_every_delete_action() { + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteVersionAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredVersionAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAllVersionsAction)); + assert!(lifecycle_action_skips_heal_version(IlmAction::DelMarkerDeleteAllVersionsAction)); + assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionAction)); + assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionVersionAction)); + assert!(!lifecycle_action_skips_heal_version(IlmAction::NoneAction)); + } + #[test] fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() { let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false) diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index acb8b53b9..1d1bcedeb 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -1140,11 +1140,11 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets { Err(Error::DiskNotFound) } - #[tracing::instrument(skip(self))] - async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> { - // Multipart orphan reconciliation is intentionally retained above the pool/set layers - // until there is a concrete caller and a stable lower-level contract to implement. - Err(StorageError::NotImplemented) + #[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))] + async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { + self.get_disks_for_heal_object(object, opts)? + .check_abandoned_parts(bucket, object, opts) + .await } } @@ -1996,7 +1996,7 @@ mod tests { } #[tokio::test] - async fn sets_check_abandoned_parts_returns_typed_not_implemented_error() { + async fn sets_check_abandoned_parts_rejects_invalid_set_scope() { let format = FormatV3::new(1, 1); let sets = Sets { id: format.id, @@ -2021,10 +2021,21 @@ mod tests { }; let err = sets - .check_abandoned_parts("bucket", "object", &HealOpts::default()) + .check_abandoned_parts( + "bucket", + "object", + &HealOpts { + set: Some(1), + ..Default::default() + }, + ) .await - .expect_err("abandoned-parts ownership should stay above the pool/set storage layers"); - assert!(matches!(err, StorageError::NotImplemented)); + .expect_err("out-of-range abandoned-parts set scope must fail closed"); + assert!( + matches!(err, StorageError::InvalidArgument(_, ref field, ref reason) + if field == "set" && reason.contains("invalid heal set index 1")), + "unexpected invalid set error: {err:?}" + ); } // Builds a single-set `Sets` over `SET_DRIVE_COUNT` local temp-dir disks, diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 41bd6dac7..7af25a64c 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -4860,6 +4860,14 @@ impl SetDisks { /// is best-effort maintenance: individual delete failures are logged and /// skipped rather than propagated. pub(crate) async fn reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result { + self.reclaim_orphan_data_dirs_inner(bucket, object, false).await + } + + pub(crate) async fn dry_run_reclaim_orphan_data_dirs(&self, bucket: &str, object: &str) -> disk::error::Result { + self.reclaim_orphan_data_dirs_inner(bucket, object, true).await + } + + async fn reclaim_orphan_data_dirs_inner(&self, bucket: &str, object: &str, dry_run: bool) -> disk::error::Result { let disks = self.get_disks_internal().await; // Phase 1 (read-only): build the referenced-data-dir union and record the @@ -4967,6 +4975,20 @@ impl SetDisks { continue; } let stray = format!("{object}/{dir}"); + if dry_run { + removed += 1; + debug!( + target: "rustfs_ecstore::set_disk", + event = "heal_abandoned_parts", + component = "ecstore", + subsystem = "heal", + state = "dry_run_matched", + result = "matched", + bucket, object, data_dir = %dir, + "Heal abandoned parts dry-run matched orphaned data directory" + ); + continue; + } match disk .delete( bucket, diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index a04c5db82..5a56ea36e 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -6998,6 +6998,100 @@ mod tests { assert!(object_dir.join(STORAGE_FORMAT_FILE).exists(), "metadata must be preserved"); } + async fn recv_abandoned_parts_trace( + trace: &mut rustfs_common::trace_bus::TraceSubscription, + bucket: &str, + object: &str, + state: &str, + ) -> rustfs_common::trace_bus::TraceEvent { + for _ in 0..32 { + let event = tokio::time::timeout(std::time::Duration::from_secs(1), trace.recv()) + .await + .expect("abandoned-parts trace event should arrive") + .expect("trace bus should stay open"); + if event.kind == rustfs_common::trace_bus::TraceKind::Heal + && event.func == rustfs_common::trace_bus::TraceFunc::HealCheckAbandonedParts + && event.bucket.as_deref() == Some(bucket) + && event.object.as_deref() == Some(object) + && trace_attr_string(&event, "state").as_deref() == Some(state) + { + return (*event).clone(); + } + } + + panic!("expected abandoned-parts trace state {state} for {bucket}/{object}"); + } + + fn trace_attr_string(event: &rustfs_common::trace_bus::TraceEvent, key: &str) -> Option { + event.attrs.iter().find_map(|attr| { + if attr.key != key { + return None; + } + Some(match &attr.value { + rustfs_common::trace_bus::TraceVal::Bool(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::U64(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::I64(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::Str(value) => value.to_string(), + }) + }) + } + + #[tokio::test] + async fn check_abandoned_parts_dry_run_counts_without_deleting() { + let mut trace = rustfs_common::trace_bus::subscribe_trace_events(); + let (dir, disk) = make_single_local_disk().await; + let live = Uuid::new_v4(); + let orphan = Uuid::new_v4(); + + let object_dir = dir.path().join("bucket").join("obj"); + write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await; + fs::create_dir_all(object_dir.join(live.to_string())) + .await + .expect("live data dir should be created"); + fs::create_dir_all(object_dir.join(orphan.to_string())) + .await + .expect("orphan data dir should be created"); + + let set = make_set_disks_with(vec![Some(disk)]).await; + set.check_abandoned_parts( + "bucket", + "obj", + &HealOpts { + dry_run: true, + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("dry-run abandoned-parts check should succeed"); + let dry_run_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "dry_run_matched").await; + assert_eq!(trace_attr_string(&dry_run_trace, "dry_run").as_deref(), Some("true")); + assert_eq!(trace_attr_string(&dry_run_trace, "data_dirs").as_deref(), Some("1")); + + assert!(object_dir.join(live.to_string()).exists(), "referenced data dir must be preserved"); + assert!(object_dir.join(orphan.to_string()).exists(), "dry-run must not remove orphaned data dir"); + + set.check_abandoned_parts( + "bucket", + "obj", + &HealOpts { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("abandoned-parts check should reclaim stale data dir"); + let reclaim_trace = recv_abandoned_parts_trace(&mut trace, "bucket", "obj", "reclaimed").await; + assert_eq!(trace_attr_string(&reclaim_trace, "dry_run").as_deref(), Some("false")); + assert_eq!(trace_attr_string(&reclaim_trace, "data_dirs").as_deref(), Some("1")); + + assert!( + object_dir.join(live.to_string()).exists(), + "referenced data dir must remain after reclaim" + ); + assert!(!object_dir.join(orphan.to_string()).exists(), "orphaned data dir must be removed"); + } + #[tokio::test] async fn reclaim_orphan_data_dirs_recovers_deferred_cleanup_after_restart() { let (dir, disk) = make_single_local_disk().await; @@ -12233,11 +12327,18 @@ mod tests { .expect_err("unsupported copy_object_part should return a typed error"); assert!(matches!(copy_part_err, StorageError::NotImplemented)); - let abandoned_err = set_disks - .check_abandoned_parts("bucket", "object", &HealOpts::default()) + set_disks + .check_abandoned_parts( + "bucket", + "object", + &HealOpts { + dry_run: true, + no_lock: true, + ..Default::default() + }, + ) .await - .expect_err("abandoned-parts check should stay in the upper reconciliation layer"); - assert!(matches!(abandoned_err, StorageError::NotImplemented)); + .expect("abandoned-parts check should be callable on empty disk sets"); } #[tokio::test] diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index 274b33f43..de688fbdb 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -16,6 +16,7 @@ use super::super::*; use crate::disk::disk_store::DiskStoreRenameDataExt; use crate::io_support::bitrot::object_mmap_read_enabled; use crate::storage_api_contracts::namespace::NamespaceLocking as _; +use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit}; use tracing::trace; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; @@ -2057,11 +2058,61 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks { Err(Error::DiskNotFound) } - #[tracing::instrument(skip(self))] - async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> { - // Multipart orphan reconciliation is intentionally retained above the set layer - // until there is a concrete caller and a stable lower-level contract to implement. - Err(StorageError::NotImplemented) + #[tracing::instrument(level = "debug", skip(self, opts), fields(bucket = %bucket, object = %object, dry_run = opts.dry_run))] + async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { + let started_at = std::time::Instant::now(); + let _write_lock_guard = if !opts.no_lock { + let ns_lock = self.new_ns_lock(bucket, object).await?; + Some( + ns_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?, + ) + } else { + None + }; + + let removed = if opts.dry_run { + self.dry_run_reclaim_orphan_data_dirs(bucket, object).await? + } else { + self.reclaim_orphan_data_dirs(bucket, object).await? + }; + let state = if opts.dry_run && removed > 0 { + "dry_run_matched" + } else if removed > 0 { + "reclaimed" + } else { + "checked" + }; + let data_dirs = u64::try_from(removed).unwrap_or(u64::MAX); + + trace_emit(|| { + TraceEvent::new(TraceKind::Heal, TraceFunc::HealCheckAbandonedParts) + .with_bucket(bucket) + .with_object(object) + .with_duration(started_at.elapsed()) + .with_attr("state", state) + .with_attr("dry_run", opts.dry_run) + .with_attr("data_dirs", data_dirs) + }); + + if removed > 0 { + trace!( + event = "heal_abandoned_parts", + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_HEAL, + state = if opts.dry_run { "dry_run_matched" } else { "reclaimed" }, + result = "ok", + bucket, + object, + dry_run = opts.dry_run, + data_dirs = removed, + "Heal abandoned parts checked object data directories" + ); + } + + Ok(()) } } diff --git a/crates/ecstore/src/set_disk/ops/heal_walk.rs b/crates/ecstore/src/set_disk/ops/heal_walk.rs index a39a9abe7..ea1ea66cf 100644 --- a/crates/ecstore/src/set_disk/ops/heal_walk.rs +++ b/crates/ecstore/src/set_disk/ops/heal_walk.rs @@ -23,6 +23,7 @@ //! per-version `SetDisks::heal_object`. use super::super::*; +use crate::object_api::ObjectInfo; use std::collections::HashSet; use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -39,12 +40,16 @@ const BACKGROUND_WALKDIR_STALL_TIMEOUT: Duration = Duration::from_secs(60); /// it must not gate healing logic — the delete-marker vs data path is chosen /// inside `ops/heal.rs` from the resolved latest metadata. `version_id` is /// normalized (nil/absent UUID => `None`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct HealWalkVersion { /// object key pub name: String, /// normalized version id (`None` when the version is nil/absent) pub version_id: Option, + /// version modification time as Unix nanoseconds + pub mod_time_unix_nanos: Option, + /// object snapshot for lifecycle evaluation + pub lifecycle_object_info: Option, /// whether this version is a delete marker (observability only) pub is_delete_marker: bool, } @@ -63,6 +68,7 @@ struct HealWalkCollector { bucket: String, batch_objects: usize, version_budget: usize, + include_lifecycle_object_info: bool, objects: Mutex>, decode_error: Mutex>, version_total: AtomicUsize, @@ -116,10 +122,25 @@ impl HealWalkCollector { let mut versions = Vec::with_capacity(fiv.versions.len() + fiv.free_versions.len()); for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) { + let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil()); + let lifecycle_object_info = if self.include_lifecycle_object_info { + let mut lifecycle_fi = fi.clone(); + lifecycle_fi.version_id = version_uuid; + Some(ObjectInfo::from_file_info( + &lifecycle_fi, + &self.bucket, + &entry.name, + version_uuid.is_some(), + )) + } else { + None + }; versions.push(HealWalkVersion { name: entry.name.clone(), // Normalize: nil/absent version id => None. - version_id: fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()), + version_id: version_uuid.map(|u| u.to_string()), + mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()), + lifecycle_object_info, is_delete_marker: fi.deleted, }); } @@ -173,11 +194,26 @@ impl HealWalkCollector { } }; for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) { - let vid = fi.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()); + let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil()); + let vid = version_uuid.map(|u| u.to_string()); if seen.insert(vid.clone()) { + let lifecycle_object_info = if self.include_lifecycle_object_info { + let mut lifecycle_fi = fi.clone(); + lifecycle_fi.version_id = version_uuid; + Some(ObjectInfo::from_file_info( + &lifecycle_fi, + &self.bucket, + &entry.name, + version_uuid.is_some(), + )) + } else { + None + }; versions.push(HealWalkVersion { name: entry.name.clone(), version_id: vid, + mod_time_unix_nanos: fi.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()), + lifecycle_object_info, is_delete_marker: fi.deleted, }); } @@ -255,6 +291,7 @@ impl SetDisks { forward_to: Option<&str>, batch_objects: usize, version_budget: usize, + include_lifecycle_object_info: bool, ) -> disk::error::Result<(Vec, Option, bool)> { assert!(batch_objects >= 2, "heal_walk_versions_page requires batch_objects >= 2"); @@ -264,6 +301,7 @@ impl SetDisks { bucket: bucket.to_string(), batch_objects, version_budget: version_budget.max(1), + include_lifecycle_object_info, objects: Mutex::new(Vec::new()), decode_error: Mutex::new(None), version_total: AtomicUsize::new(0), @@ -347,6 +385,7 @@ mod tests { bucket: "bucket".to_string(), batch_objects: 2, version_budget: 2, + include_lifecycle_object_info: false, objects: Mutex::new(Vec::new()), decode_error: Mutex::new(None), version_total: AtomicUsize::new(0), @@ -388,6 +427,8 @@ mod tests { HealWalkVersion { name: name.to_string(), version_id: Some(id.to_string()), + mod_time_unix_nanos: None, + lifecycle_object_info: None, is_delete_marker: dm, } } @@ -491,6 +532,7 @@ mod tests { bucket: "bucket".to_string(), batch_objects: 1000, version_budget: 10_000, + include_lifecycle_object_info: false, objects: Mutex::new(Vec::new()), version_total: AtomicUsize::new(0), decode_error: Mutex::new(None), @@ -567,7 +609,7 @@ mod tests { .expect("corrupt test metadata should be written"); let error = set_disks - .heal_walk_versions_page(bucket, "", None, 2, 2) + .heal_walk_versions_page(bucket, "", None, 2, 2, false) .await .expect_err("semantic metadata corruption must fail the heal disk walk"); diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index 3efa4ed5f..d10abe740 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -18,6 +18,7 @@ use tracing::trace; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_HEAL: &str = "heal"; +const EVENT_HEAL_ABANDONED_PARTS: &str = "heal_abandoned_parts"; const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed"; const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started"; @@ -256,13 +257,40 @@ impl ECStore { #[instrument(skip(self))] pub(super) async fn handle_check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { - let _ = (bucket, object, opts); - // Stale multipart reconciliation is already owned by the lifecycle-driven - // background cleanup path in `bucket_lifecycle_ops.rs`. There is currently - // no stable object-heal contract that should fan this request out through - // pool/set storage layers, so keep the placeholder explicit at the ECStore - // boundary instead of dispatching into lower layers. - Err(StorageError::NotImplemented) + let object = encode_dir_object(object); + let pools = self.get_pools_for_heal_object(opts)?; + + let mut futures = Vec::with_capacity(pools.len()); + for pool in pools.iter() { + futures.push(pool.check_abandoned_parts(bucket, &object, opts)); + } + + let mut first_error = None; + for result in join_all(futures).await { + if let Err(err) = result + && first_error.is_none() + { + first_error = Some(err); + } + } + + if let Some(err) = first_error { + return Err(err); + } + + trace!( + event = EVENT_HEAL_ABANDONED_PARTS, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_HEAL, + state = "completed", + result = "ok", + bucket, + object, + dry_run = opts.dry_run, + "Heal abandoned parts completed" + ); + + Ok(()) } } diff --git a/crates/ecstore/src/store/heal_walk.rs b/crates/ecstore/src/store/heal_walk.rs index 39d50007d..191e87e54 100644 --- a/crates/ecstore/src/store/heal_walk.rs +++ b/crates/ecstore/src/store/heal_walk.rs @@ -34,6 +34,7 @@ impl ECStore { forward_to: Option<&str>, batch_objects: usize, version_budget: usize, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { if pool_idx >= self.pools.len() || set_idx >= self.pools[pool_idx].disk_set.len() { return Err(Error::other(format!( @@ -43,7 +44,7 @@ impl ECStore { } self.pools[pool_idx].disk_set[set_idx] - .heal_walk_versions_page(bucket, prefix, forward_to, batch_objects, version_budget) + .heal_walk_versions_page(bucket, prefix, forward_to, batch_objects, version_budget, include_lifecycle_object_info) .await .map_err(Error::from) } diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index d2be2545d..23cf2f168 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -767,6 +767,7 @@ mod tests { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> crate::Result<(Vec, Option, bool)> { Ok((vec![], None, false)) } diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 8f1810bb6..04075837c 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -23,13 +23,14 @@ use crate::heal::{ }; use crate::{Error, Result}; use futures::{StreamExt, stream::FuturesUnordered}; -use metrics::gauge; +use metrics::{counter, gauge}; use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode}; use rustfs_madmin::heal_commands::HealResultItem; use std::sync::{ Arc, atomic::{AtomicUsize, Ordering}, }; +use std::time::{Duration, UNIX_EPOCH}; use tokio::sync::{RwLock, Semaphore}; use tracing::{debug, error, warn}; @@ -47,6 +48,21 @@ enum HealObjectOutcome { Failed, } +fn result_object_size_u64(result: &HealResultItem) -> u64 { + u64::try_from(result.object_size).unwrap_or(u64::MAX) +} + +const NEW_VERSION_SKIP_GRACE_SECS: u64 = 60; +const NANOS_PER_SECOND: i128 = 1_000_000_000; + +fn should_skip_new_version(mod_time_unix_nanos: Option, started_at_secs: u64) -> bool { + let Some(mod_time_unix_nanos) = mod_time_unix_nanos else { + return false; + }; + let cutoff_secs = started_at_secs.saturating_add(NEW_VERSION_SKIP_GRACE_SECS); + mod_time_unix_nanos > i128::from(cutoff_secs).saturating_mul(NANOS_PER_SECOND) +} + struct PageConcurrencyGuard { in_flight: Arc, set_label: String, @@ -492,6 +508,7 @@ impl ErasureSetHealer { &mut skipped_objects, resume_manager, checkpoint_manager, + state.start_time, ) .await; @@ -658,6 +675,7 @@ impl ErasureSetHealer { skipped_objects: &mut u64, resume_manager: &ResumeManager, checkpoint_manager: &CheckpointManager, + started_at_secs: u64, ) -> Result<()> { debug!( target: "rustfs::heal::erasure_healer", @@ -710,6 +728,7 @@ impl ErasureSetHealer { // The end-of-pass summary reports the full failed/skipped counts. let mut transient_skip_samples_logged = 0_u64; let mut failure_samples_logged = 0_u64; + let mut bytes_processed = self.progress.read().await.bytes_processed; // backlog#920: select the per-erasure-set DISK-WALK union enumerator when // the scan is Deep OR the request came from AutoHeal — these are the paths @@ -718,17 +737,25 @@ impl ErasureSetHealer { // which stays the default. let use_disk_walk = matches!(self.heal_opts.scan_mode, HealScanMode::Deep) || matches!(self.source, HealRequestSource::AutoHeal); + let lifecycle_expiry_context = self.storage.load_heal_lifecycle_expiry_context(bucket).await?; + let include_lifecycle_object_info = lifecycle_expiry_context.is_some(); loop { self.verify_replacement_identity_fence("page scan").await?; // Get one page of object versions let (objects, next_token, is_truncated) = if use_disk_walk { self.storage - .list_versions_for_heal_page_disk_walk(set_disk_id, bucket, "", continuation_token.as_deref()) + .list_versions_for_heal_page_disk_walk( + set_disk_id, + bucket, + "", + continuation_token.as_deref(), + include_lifecycle_object_info, + ) .await? } else { self.storage - .list_objects_for_heal_page(bucket, "", continuation_token.as_deref()) + .list_objects_for_heal_page(bucket, "", continuation_token.as_deref(), include_lifecycle_object_info) .await? }; let page_is_empty = objects.is_empty(); @@ -736,6 +763,7 @@ impl ErasureSetHealer { let page_resume_index = *current_object_index; let semaphore = Arc::new(Semaphore::new(page_concurrency_limit)); let mut page_tasks = FuturesUnordered::new(); + let mut completed_in_page = 0usize; // Capture the last version identity of this page for the anti-loop guard. let page_last = objects.last().map(|item| (item.name.clone(), item.version_id.clone())); @@ -751,6 +779,75 @@ impl ErasureSetHealer { continue; } + if should_skip_new_version(item.mod_time_unix_nanos, started_at_secs) { + checkpoint_manager.add_processed_object(key).await?; + *processed_objects = processed_objects.saturating_add(1); + completed_in_page = completed_in_page.saturating_add(1); + counter!("rustfs_heal_skipped_new_versions_total").increment(1); + { + let mut progress = self.progress.write().await; + progress.record_skipped_new_version(); + progress.set_current_object(Some(format!("skipped_new: {bucket}/{}", item.name))); + progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed); + } + debug!( + target: "rustfs::heal::erasure_healer", + event = EVENT_HEAL_ERASURE_OBJECT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, + set_disk_id, + bucket, + object = %item.name, + version_id = ?item.version_id, + state = "skipped_new_version", + "Erasure set object version skipped because it was written after heal started" + ); + if completed_in_page.is_multiple_of(100) { + checkpoint_manager.update_position(bucket_index, page_resume_index).await?; + } + continue; + } + + if let Some(context) = lifecycle_expiry_context.as_ref() + && self + .storage + .enqueue_heal_lifecycle_expiry( + context, + bucket, + &item.name, + item.version_id.as_deref(), + item.lifecycle_object_info.as_ref(), + ) + .await? + { + checkpoint_manager.add_processed_object(key).await?; + *processed_objects = processed_objects.saturating_add(1); + completed_in_page = completed_in_page.saturating_add(1); + counter!("rustfs_heal_skipped_ilm_expired_total").increment(1); + { + let mut progress = self.progress.write().await; + progress.record_skipped_ilm_expired(); + progress.set_current_object(Some(format!("skipped_ilm: {bucket}/{}", item.name))); + progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed); + } + debug!( + target: "rustfs::heal::erasure_healer", + event = EVENT_HEAL_ERASURE_OBJECT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, + set_disk_id, + bucket, + object = %item.name, + version_id = ?item.version_id, + state = "skipped_ilm_expired", + "Erasure set object version skipped because lifecycle expiry was queued" + ); + if completed_in_page.is_multiple_of(100) { + checkpoint_manager.update_position(bucket_index, page_resume_index).await?; + } + continue; + } + resume_manager .set_current_item(Some(bucket.to_string()), Some(item.name.clone())) .await?; @@ -777,7 +874,7 @@ impl ErasureSetHealer { let _permit = match permit { Ok(permit) => permit, - Err(err) => return (dedup_key, object_name, version_id, Err(err)), + Err(err) => return (dedup_key, object_name, version_id, (0, Err(err))), }; let _in_flight_guard = PageConcurrencyGuard::new(in_flight, set_label); @@ -788,7 +885,7 @@ impl ErasureSetHealer { // recorded as skipped-ok rather than failed. The delete-marker // vs data path is chosen internally in ops/heal.rs. let result = if cancel_token.is_cancelled() { - Err(Error::TaskCancelled) + (0, Err(Error::TaskCancelled)) } else { match storage .heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts) @@ -797,8 +894,9 @@ impl ErasureSetHealer { Ok((result, None)) if target_outcomes_complete(&result, &target_endpoints) => { + let object_size = result_object_size_u64(&result); if !replacement_commit_evidence_required { - Ok(true) + (object_size, Ok(true)) } else { match storage .replacement_targets_have_version( @@ -810,27 +908,42 @@ impl ErasureSetHealer { ) .await { - Ok(true) => Ok(true), - Ok(false) => Err(Error::transient_skip(format!( + Ok(true) => (object_size, Ok(true)), + Ok(false) => (object_size, Err(Error::transient_skip(format!( "Skipped heal for {bucket_name}/{object_name} because replacement target readback did not confirm the committed version" - ))), - Err(err) => Err(Error::transient_skip(format!( + )))), + Err(err) => (object_size, Err(Error::transient_skip(format!( "Skipped heal for {bucket_name}/{object_name} because replacement target readback failed: {err}" - ))), + )))), } } - } - Ok((_result, None)) if !target_endpoints.is_empty() => Err(Error::transient_skip(format!( - "Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed" - ))), - Ok((_result, None)) => Ok(true), - Ok((_, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => Ok(false), - Ok((_, Some(err))) | Err(err) => match Self::classify_heal_object_error(&err) { - HealObjectOutcome::Absent => Ok(false), - HealObjectOutcome::Transient => Err(Error::transient_skip(format!( - "Skipped heal for {bucket_name}/{object_name} due to transient error: {err}" + }, + Ok((result, None)) if !target_endpoints.is_empty() => ( + result_object_size_u64(&result), + Err(Error::transient_skip(format!( + "Skipped heal for {bucket_name}/{object_name} because a replacement target was not committed" ))), - HealObjectOutcome::Failed => Err(err), + ), + Ok((result, None)) => (result_object_size_u64(&result), Ok(true)), + Ok((result, Some(err))) if is_missing_object_dir_heal_result(&object_name, &err) => { + (result_object_size_u64(&result), Ok(false)) + } + Ok((result, Some(err))) => { + let object_size = result_object_size_u64(&result); + match Self::classify_heal_object_error(&err) { + HealObjectOutcome::Absent => (object_size, Ok(false)), + HealObjectOutcome::Transient => (object_size, Err(Error::transient_skip(format!( + "Skipped heal for {bucket_name}/{object_name} due to transient error: {err}" + )))), + HealObjectOutcome::Failed => (object_size, Err(err)), + } + } + Err(err) => match Self::classify_heal_object_error(&err) { + HealObjectOutcome::Absent => (0, Ok(false)), + HealObjectOutcome::Transient => (0, Err(Error::transient_skip(format!( + "Skipped heal for {bucket_name}/{object_name} due to transient error: {err}" + )))), + HealObjectOutcome::Failed => (0, Err(err)), }, } }; @@ -839,11 +952,12 @@ impl ErasureSetHealer { }); } - let mut completed_in_page = 0usize; while let Some((key, object, version_id, result)) = page_tasks.next().await { + let (object_size, result) = result; match result { Ok(true) => { *successful_objects += 1; + bytes_processed = bytes_processed.saturating_add(object_size); checkpoint_manager.add_processed_object(key).await?; debug!( target: "rustfs::heal::erasure_healer", @@ -861,6 +975,7 @@ impl ErasureSetHealer { Ok(false) => { checkpoint_manager.add_processed_object(key).await?; *successful_objects += 1; + bytes_processed = bytes_processed.saturating_add(object_size); debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_OBJECT_STATE, @@ -877,6 +992,7 @@ impl ErasureSetHealer { Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err), Err(Error::TransientSkip { message }) => { *skipped_objects += 1; + bytes_processed = bytes_processed.saturating_add(object_size); checkpoint_manager.add_skipped_object(key).await?; demote_to_debug_when!(!take_failure_log_sample(&mut transient_skip_samples_logged), warn, target: "rustfs::heal::erasure_healer", { event = EVENT_HEAL_ERASURE_OBJECT_STATE, @@ -893,6 +1009,7 @@ impl ErasureSetHealer { } Err(err) => { *failed_objects += 1; + bytes_processed = bytes_processed.saturating_add(object_size); checkpoint_manager.add_failed_object(key).await?; demote_to_debug_when!(!take_failure_log_sample(&mut failure_samples_logged), warn, target: "rustfs::heal::erasure_healer", { event = EVENT_HEAL_ERASURE_OBJECT_STATE, @@ -911,6 +1028,11 @@ impl ErasureSetHealer { *processed_objects += 1; completed_in_page += 1; + { + let mut progress = self.progress.write().await; + progress.set_current_object(Some(format!("{bucket}/{object}"))); + progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed); + } if completed_in_page.is_multiple_of(100) { checkpoint_manager.update_position(bucket_index, page_resume_index).await?; @@ -964,7 +1086,9 @@ impl ErasureSetHealer { progress.objects_scanned = state.total_objects; progress.objects_healed = state.successful_objects; progress.objects_failed = state.failed_objects; - progress.bytes_processed = 0; // set to 0 for now, can be extended later + progress.bytes_processed = 0; // Resume state tracks object counts, not byte counters. + progress.start_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.start_time)); + progress.last_update_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.last_update)); progress.set_current_object(state.current_object.clone()); } } @@ -1135,13 +1259,15 @@ mod resume_loop_tests { //! that emits programmable multi-version pages. These exercise the real loop //! logic (cursor seeding, per-version dedup, anti-loop guard, absence //! handling) — not merely a mock's own output. - use super::{ErasureSetHealer, target_outcomes_complete}; + use super::{ + ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete, + }; use crate::heal::progress::HealProgress; use crate::heal::resume::{ CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils, compose_key, }; - use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo, HealStorageAPI}; + use crate::heal::storage::{DiskStatus, HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI}; use crate::heal::storage_api::status::BucketInfo; use crate::heal::{ BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk, @@ -1149,7 +1275,7 @@ mod resume_loop_tests { use crate::{Error, Result}; use rustfs_common::heal_channel::{HealOpts, HealRequestSource}; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; - use std::collections::{HashMap, VecDeque}; + use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use tempfile::TempDir; @@ -1160,10 +1286,37 @@ mod resume_loop_tests { HealListItem { name: name.to_string(), version_id: version.map(str::to_string), + mod_time_unix_nanos: None, + lifecycle_object_info: None, is_delete_marker: delete_marker, } } + fn item_with_mod_time(name: &str, version: Option<&str>, mod_time_secs: u64) -> HealListItem { + HealListItem { + name: name.to_string(), + version_id: version.map(str::to_string), + mod_time_unix_nanos: Some(i128::from(mod_time_secs).saturating_mul(NANOS_PER_SECOND)), + lifecycle_object_info: None, + is_delete_marker: false, + } + } + + #[test] + fn new_version_filter_respects_grace_boundary() { + let started_at = 1_700_000_000; + + assert!(!should_skip_new_version(None, started_at)); + assert!(!should_skip_new_version( + Some(i128::from(started_at + NEW_VERSION_SKIP_GRACE_SECS).saturating_mul(NANOS_PER_SECOND)), + started_at, + )); + assert!(should_skip_new_version( + Some(i128::from(started_at + NEW_VERSION_SKIP_GRACE_SECS + 1).saturating_mul(NANOS_PER_SECOND)), + started_at, + )); + } + #[test] fn target_outcomes_require_each_requested_endpoint_once_and_ok() { let result = HealResultItem { @@ -1246,8 +1399,10 @@ mod resume_loop_tests { /// Target-specific physical readback evidence per `compose_key`; the /// fake models a healthy backend unless a test explicitly revokes it. replacement_commit_evidence: Mutex>, + lifecycle_expired: Mutex>, /// every heal_object call recorded as (name, version_id) heal_calls: Mutex)>>, + list_include_lifecycle_object_info: Mutex>, replacement_target_identity_sequences: Mutex>>, fail_listing: AtomicBool, } @@ -1274,9 +1429,15 @@ mod resume_loop_tests { .unwrap() .insert(compose_key(name, version), ReplacementCommitEvidence::Error(message.to_string())); } + fn set_lifecycle_expired(&self, name: &str, version: Option<&str>) { + self.lifecycle_expired.lock().unwrap().insert(compose_key(name, version)); + } fn calls(&self) -> Vec<(String, Option)> { self.heal_calls.lock().unwrap().clone() } + fn list_include_lifecycle_object_info_calls(&self) -> Vec { + self.list_include_lifecycle_object_info.lock().unwrap().clone() + } fn fail_listing(&self) { self.fail_listing.store(true, Ordering::SeqCst); } @@ -1330,6 +1491,23 @@ mod resume_loop_tests { async fn get_object_checksum(&self, _b: &str, _o: &str) -> Result> { Ok(None) } + async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result> { + Ok((!self.lifecycle_expired.lock().unwrap().is_empty()).then(HealLifecycleExpiryContext::test)) + } + async fn enqueue_heal_lifecycle_expiry( + &self, + _context: &HealLifecycleExpiryContext, + _bucket: &str, + object: &str, + version_id: Option<&str>, + _object_info: Option<&HealObjectInfo>, + ) -> Result { + Ok(self + .lifecycle_expired + .lock() + .unwrap() + .contains(&compose_key(object, version_id))) + } async fn heal_object( &self, _bucket: &str, @@ -1386,7 +1564,12 @@ mod resume_loop_tests { _bucket: &str, _prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { + self.list_include_lifecycle_object_info + .lock() + .unwrap() + .push(include_lifecycle_object_info); if self.fail_listing.load(Ordering::SeqCst) { return Err(Error::other("injected listing failure")); } @@ -1476,6 +1659,7 @@ mod resume_loop_tests { /// Drive one bucket heal pass; returns (processed, successful, failed, skipped, result). async fn run(env: &Env) -> (u64, u64, u64, u64, Result<()>) { + let state = env.resume.get_state().await; let mut current_object_index = 0usize; let mut processed = 0u64; let mut successful = 0u64; @@ -1494,6 +1678,7 @@ mod resume_loop_tests { &mut skipped, &env.resume, &env.checkpoint, + state.start_time, ) .await; (processed, successful, failed, skipped, result) @@ -1559,6 +1744,7 @@ mod resume_loop_tests { let mut successful = 0; let mut failed = 0; let mut skipped = 0; + let started_at = env.resume.get_state().await.start_time; let error = healer .heal_bucket_with_resume( @@ -1572,6 +1758,7 @@ mod resume_loop_tests { &mut skipped, &env.resume, &env.checkpoint, + started_at, ) .await .expect_err("a remounted target must not begin a new page scan"); @@ -1641,6 +1828,109 @@ mod resume_loop_tests { assert_eq!(skipped, 0); } + #[tokio::test] + async fn erasure_set_progress_accumulates_healed_object_bytes() { + let env = make_env().await; + env.storage.set_page( + None, + Page { + items: vec![item("first", Some("v1"), false), item("second", Some("v2"), false)], + next: None, + truncated: false, + }, + ); + env.storage.set_result( + "first", + Some("v1"), + HealResultItem { + object_size: 1024, + ..Default::default() + }, + ); + env.storage.set_result( + "second", + Some("v2"), + HealResultItem { + object_size: 2048, + ..Default::default() + }, + ); + + let (processed, successful, failed, skipped, result) = run(&env).await; + + result.expect("page heal should succeed"); + assert_eq!(processed, 2); + assert_eq!(successful, 2); + assert_eq!(failed, 0); + assert_eq!(skipped, 0); + let progress = env.healer.progress.read().await; + assert_eq!(progress.objects_scanned, 2); + assert_eq!(progress.objects_healed, 2); + assert_eq!(progress.objects_failed, 0); + assert_eq!(progress.bytes_processed, 3072); + assert!(matches!(progress.current_object.as_deref(), Some("b/first" | "b/second"))); + } + + #[tokio::test] + async fn erasure_set_skips_versions_written_after_heal_started() { + let env = make_env().await; + let started_at = env.resume.get_state().await.start_time; + env.storage.set_page( + None, + Page { + items: vec![ + item_with_mod_time("old", Some("v1"), started_at + NEW_VERSION_SKIP_GRACE_SECS), + item_with_mod_time("new", Some("v2"), started_at + NEW_VERSION_SKIP_GRACE_SECS + 1), + ], + next: None, + truncated: false, + }, + ); + + let (processed, successful, failed, skipped, result) = run(&env).await; + + result.expect("page heal should succeed"); + assert_eq!(processed, 2); + assert_eq!(successful, 1); + assert_eq!(failed, 0); + assert_eq!(skipped, 0); + assert_eq!(env.storage.calls(), vec![("old".to_string(), Some("v1".to_string()))]); + let progress = env.healer.progress.read().await; + assert_eq!(progress.skipped_new_versions, 1); + assert_eq!(progress.objects_scanned, 2); + assert_eq!(progress.objects_healed, 1); + assert_eq!(progress.objects_failed, 0); + } + + #[tokio::test] + async fn erasure_set_skips_versions_queued_for_lifecycle_expiry() { + let env = make_env().await; + env.storage.set_page( + None, + Page { + items: vec![item("expired", Some("v1"), false), item("kept", Some("v2"), false)], + next: None, + truncated: false, + }, + ); + env.storage.set_lifecycle_expired("expired", Some("v1")); + + let (processed, successful, failed, skipped, result) = run(&env).await; + + result.expect("page heal should succeed"); + assert_eq!(processed, 2); + assert_eq!(successful, 1); + assert_eq!(failed, 0); + assert_eq!(skipped, 0); + assert_eq!(env.storage.calls(), vec![("kept".to_string(), Some("v2".to_string()))]); + assert_eq!(env.storage.list_include_lifecycle_object_info_calls(), vec![true]); + let progress = env.healer.progress.read().await; + assert_eq!(progress.skipped_ilm_expired, 1); + assert_eq!(progress.objects_scanned, 2); + assert_eq!(progress.objects_healed, 1); + assert_eq!(progress.objects_failed, 0); + } + #[tokio::test] async fn bucket_listing_failure_does_not_mark_set_completed() { let env = make_env().await; diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 79ba6b169..66b8f637f 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -2385,8 +2385,27 @@ impl HealManager { snapshot.objects_scanned = snapshot.objects_scanned.saturating_add(progress.objects_scanned); snapshot.objects_healed = snapshot.objects_healed.saturating_add(progress.objects_healed); snapshot.objects_failed = snapshot.objects_failed.saturating_add(progress.objects_failed); + snapshot.skipped_new_versions = snapshot.skipped_new_versions.saturating_add(progress.skipped_new_versions); + snapshot.skipped_ilm_expired = snapshot.skipped_ilm_expired.saturating_add(progress.skipped_ilm_expired); + snapshot.objects_total_count = snapshot.objects_total_count.saturating_add(progress.objects_total_count); + snapshot.objects_total_size = snapshot.objects_total_size.saturating_add(progress.objects_total_size); snapshot.bytes_processed = snapshot.bytes_processed.saturating_add(progress.bytes_processed); + snapshot.start_time = match (snapshot.start_time, progress.start_time) { + (Some(current), Some(next)) => Some(current.min(next)), + (None, next) => next, + (current, None) => current, + }; + snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) { + (Some(current), Some(next)) => Some(current.max(next)), + (None, next) => next, + (current, None) => current, + }; + if progress.current_object.is_some() { + snapshot.current_object = progress.current_object; + } } + snapshot.refresh_progress_percentage(); + snapshot.refresh_estimated_completion_time(); Some(snapshot) } @@ -3208,6 +3227,7 @@ impl HealManager { } else { completed_task.get_status().await }; + let completed_progress = completed_task.get_progress().await; let completed_status_entry = CompletedHealStatus { heal_type: completed_task.heal_type.clone(), status: completed_status.clone(), @@ -3223,6 +3243,7 @@ impl HealManager { match completed_status { HealTaskStatus::Completed => { stats.update_task_completion(true); + stats.add_healed_objects(completed_progress.objects_healed, completed_progress.bytes_processed); } HealTaskStatus::Retrying { .. } => {} _ => { @@ -3749,6 +3770,7 @@ mod tests { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { Ok((Vec::new(), None, false)) } @@ -5396,6 +5418,8 @@ mod tests { )); { let mut progress = first.progress.write().await; + progress.start_time = Some(SystemTime::now() - Duration::from_secs(20)); + progress.set_total_baseline(12, 8192); progress.update_progress(7, 3, 1, 4096); } @@ -5405,6 +5429,8 @@ mod tests { )); { let mut progress = second.progress.write().await; + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + progress.set_total_baseline(8, 4096); progress.update_progress(11, 5, 2, 2048); } @@ -5419,7 +5445,11 @@ mod tests { assert_eq!(progress.objects_scanned, 18); assert_eq!(progress.objects_healed, 8); assert_eq!(progress.objects_failed, 3); + assert_eq!(progress.objects_total_count, 20); + assert_eq!(progress.objects_total_size, 12288); assert_eq!(progress.bytes_processed, 6144); + assert!((progress.progress_percentage - 50.0).abs() < 0.001); + assert!(progress.estimated_completion_time.is_some()); } #[tokio::test] diff --git a/crates/heal/src/heal/progress.rs b/crates/heal/src/heal/progress.rs index cb602b5de..981aa01fe 100644 --- a/crates/heal/src/heal/progress.rs +++ b/crates/heal/src/heal/progress.rs @@ -13,7 +13,7 @@ // limitations under the License. use serde::{Deserialize, Serialize}; -use std::time::SystemTime; +use std::time::{Duration, SystemTime}; #[derive(Debug, Default, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -24,6 +24,14 @@ pub struct HealProgress { pub objects_healed: u64, /// Objects failed pub objects_failed: u64, + /// Versions skipped because they were written after this heal started + pub skipped_new_versions: u64, + /// Versions skipped because lifecycle already selected them for expiry + pub skipped_ilm_expired: u64, + /// Baseline object count from the latest complete usage snapshot + pub objects_total_count: u64, + /// Baseline object bytes from the latest complete usage snapshot + pub objects_total_size: u64, /// Bytes processed pub bytes_processed: u64, /// Current object @@ -54,10 +62,56 @@ impl HealProgress { self.bytes_processed = bytes; self.last_update_time = Some(SystemTime::now()); - // calculate progress percentage - let total = scanned + healed + failed; + self.refresh_progress_percentage(); + self.refresh_estimated_completion_time(); + } + + pub fn set_total_baseline(&mut self, objects_total_count: u64, objects_total_size: u64) { + self.objects_total_count = objects_total_count; + self.objects_total_size = objects_total_size; + self.last_update_time = Some(SystemTime::now()); + self.refresh_progress_percentage(); + self.refresh_estimated_completion_time(); + } + + pub fn record_skipped_new_version(&mut self) { + self.skipped_new_versions = self.skipped_new_versions.saturating_add(1); + self.last_update_time = Some(SystemTime::now()); + self.refresh_progress_percentage(); + self.refresh_estimated_completion_time(); + } + + pub fn record_skipped_ilm_expired(&mut self) { + self.skipped_ilm_expired = self.skipped_ilm_expired.saturating_add(1); + self.last_update_time = Some(SystemTime::now()); + self.refresh_progress_percentage(); + self.refresh_estimated_completion_time(); + } + + fn completed_for_baseline(&self) -> u64 { + self.objects_healed + .saturating_add(self.objects_failed) + .saturating_add(self.skipped_new_versions) + .saturating_add(self.skipped_ilm_expired) + } + + pub(crate) fn refresh_progress_percentage(&mut self) { + if self.objects_total_size > 0 { + self.progress_percentage = ((self.bytes_processed as f64 / self.objects_total_size as f64) * 100.0).min(100.0); + return; + } + if self.objects_total_count > 0 { + let completed = self.completed_for_baseline(); + self.progress_percentage = ((completed as f64 / self.objects_total_count as f64) * 100.0).min(100.0); + return; + } + + let total = self + .objects_scanned + .saturating_add(self.objects_healed) + .saturating_add(self.objects_failed); if total > 0 { - self.progress_percentage = (healed as f64 / total as f64) * 100.0; + self.progress_percentage = (self.objects_healed as f64 / total as f64) * 100.0; } } @@ -66,9 +120,36 @@ impl HealProgress { self.last_update_time = Some(SystemTime::now()); } + pub fn refresh_estimated_completion_time(&mut self) { + let Some(start_time) = self.start_time else { + self.estimated_completion_time = None; + return; + }; + if self.is_completed() || !(0.0..100.0).contains(&self.progress_percentage) || self.bytes_processed == 0 { + self.estimated_completion_time = None; + return; + } + + let elapsed = match SystemTime::now().duration_since(start_time) { + Ok(elapsed) if !elapsed.is_zero() => elapsed, + _ => { + self.estimated_completion_time = None; + return; + } + }; + let estimated_total_secs = elapsed.as_secs_f64() * 100.0 / self.progress_percentage; + self.estimated_completion_time = start_time.checked_add(Duration::from_secs_f64(estimated_total_secs)); + } + pub fn is_completed(&self) -> bool { - self.progress_percentage >= 100.0 - || self.objects_scanned > 0 && self.objects_healed + self.objects_failed >= self.objects_scanned + if self.progress_percentage >= 100.0 { + return true; + } + if self.objects_total_count > 0 || self.objects_total_size > 0 { + return false; + } + + self.objects_scanned > 0 && self.objects_healed.saturating_add(self.objects_failed) >= self.objects_scanned } pub fn get_success_rate(&self) -> f64 { @@ -158,6 +239,10 @@ mod tests { assert_eq!(progress.objects_scanned, 0); assert_eq!(progress.objects_healed, 0); assert_eq!(progress.objects_failed, 0); + assert_eq!(progress.skipped_new_versions, 0); + assert_eq!(progress.skipped_ilm_expired, 0); + assert_eq!(progress.objects_total_count, 0); + assert_eq!(progress.objects_total_size, 0); assert_eq!(progress.bytes_processed, 0); assert_eq!(progress.progress_percentage, 0.0); assert!(progress.start_time.is_some()); @@ -181,6 +266,73 @@ mod tests { assert!(progress.last_update_time.is_some()); } + #[test] + fn test_heal_progress_estimates_completion_time_from_progress() { + let mut progress = HealProgress::new(); + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + + progress.update_progress(100, 25, 0, 4096); + + let eta = progress + .estimated_completion_time + .expect("partial byte progress should estimate completion"); + assert!(eta > SystemTime::now()); + } + + #[test] + fn test_heal_progress_uses_byte_baseline_for_percentage() { + let mut progress = HealProgress::new(); + progress.set_total_baseline(10, 8192); + + progress.update_progress(100, 25, 0, 4096); + + assert!((progress.progress_percentage - 50.0).abs() < 0.001); + } + + #[test] + fn test_heal_progress_uses_object_baseline_when_bytes_unknown() { + let mut progress = HealProgress::new(); + progress.set_total_baseline(10, 0); + + progress.update_progress(100, 3, 2, 0); + + assert!((progress.progress_percentage - 50.0).abs() < 0.001); + } + + #[test] + fn test_heal_progress_counts_skipped_versions_for_object_baseline() { + let mut progress = HealProgress::new(); + progress.set_total_baseline(10, 0); + + progress.update_progress(100, 3, 2, 0); + progress.record_skipped_new_version(); + + assert_eq!(progress.skipped_new_versions, 1); + assert!((progress.progress_percentage - 60.0).abs() < 0.001); + } + + #[test] + fn test_heal_progress_does_not_estimate_completion_without_bytes() { + let mut progress = HealProgress::new(); + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + + progress.update_progress(100, 25, 0, 0); + + assert!(progress.estimated_completion_time.is_none()); + } + + #[test] + fn test_heal_progress_with_baseline_is_not_completed_by_processed_count() { + let mut progress = HealProgress::new(); + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + progress.set_total_baseline(10, 8192); + + progress.update_progress(1, 1, 0, 1024); + + assert!(!progress.is_completed()); + assert!(progress.estimated_completion_time.is_some()); + } + #[test] fn test_heal_progress_update_progress_zero_total() { let mut progress = HealProgress::new(); @@ -251,6 +403,8 @@ mod tests { assert_eq!(json["objectsScanned"], 10); assert_eq!(json["objectsHealed"], 8); assert_eq!(json["objectsFailed"], 2); + assert_eq!(json["skippedNewVersions"], 0); + assert_eq!(json["skippedIlmExpired"], 0); assert_eq!(json["bytesProcessed"], 1024); assert_eq!(json["currentObject"], "test-bucket/test-object"); assert!(json["progressPercentage"].is_number()); diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index f5101ac1c..fbe51f050 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use tracing::{debug, error, warn}; +use super::storage_api::owner::{EcstoreHealLifecycleExpiryContext, ecstore_load_admin_data_usage_from_backend_cached}; use super::storage_api::storage::{ BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _, ObjectOperations as _, StorageAdminApi, @@ -29,6 +30,37 @@ use super::storage_api::storage::{ use super::{DiskStore, ECStore, Endpoint, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity}; pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader}; +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct HealBucketUsageBaseline { + pub objects_count: u64, + pub bytes: u64, +} + +pub struct HealLifecycleExpiryContext { + inner: HealLifecycleExpiryContextInner, +} + +enum HealLifecycleExpiryContextInner { + Ecstore(EcstoreHealLifecycleExpiryContext), + #[allow(dead_code)] + Test, +} + +impl HealLifecycleExpiryContext { + fn ecstore(inner: EcstoreHealLifecycleExpiryContext) -> Self { + Self { + inner: HealLifecycleExpiryContextInner::Ecstore(inner), + } + } + + #[cfg(test)] + pub(crate) fn test() -> Self { + Self { + inner: HealLifecycleExpiryContextInner::Test, + } + } +} + const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_STORAGE: &str = "storage"; const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io"; @@ -272,6 +304,10 @@ pub struct HealListItem { pub name: String, /// normalized version id (`None` when the version is nil/absent) pub version_id: Option, + /// version modification time as Unix nanoseconds + pub mod_time_unix_nanos: Option, + /// object snapshot for lifecycle evaluation + pub lifecycle_object_info: Option, /// whether this version is a delete marker (observability only) pub is_delete_marker: bool, } @@ -329,6 +365,28 @@ pub trait HealStorageAPI: Send + Sync { /// Get bucket info async fn get_bucket_info(&self, bucket: &str) -> Result>; + /// Aggregate usage-cache baselines for the requested buckets. + async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result> { + Ok(None) + } + + /// Load per-bucket lifecycle expiry context for heal skips. + async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result> { + Ok(None) + } + + /// Queue lifecycle expiry for a version that heal can skip. + async fn enqueue_heal_lifecycle_expiry( + &self, + _context: &HealLifecycleExpiryContext, + _bucket: &str, + _object: &str, + _version_id: Option<&str>, + _object_info: Option<&HealObjectInfo>, + ) -> Result { + Ok(false) + } + /// Fix bucket metadata async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()>; @@ -409,6 +467,7 @@ pub trait HealStorageAPI: Send + Sync { bucket: &str, prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)>; /// List versions for healing via a per-erasure-set DISK-WALK union enumerator @@ -427,8 +486,10 @@ pub trait HealStorageAPI: Send + Sync { bucket: &str, prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { - self.list_objects_for_heal_page(bucket, prefix, continuation_token).await + self.list_objects_for_heal_page(bucket, prefix, continuation_token, include_lifecycle_object_info) + .await } /// Get disk for resume functionality. @@ -1021,6 +1082,85 @@ impl HealStorageAPI for ECStoreHealStorage { } } + async fn erasure_set_usage_baseline(&self, buckets: &[String]) -> Result> { + if buckets.is_empty() { + return Ok(None); + } + + let info = match ecstore_load_admin_data_usage_from_backend_cached(self.ecstore.clone()).await { + Ok(info) if info.is_complete_bucket_usage_snapshot() => info, + Ok(_) | Err(_) => return Ok(None), + }; + + let mut baseline = HealBucketUsageBaseline::default(); + for bucket in buckets { + if let Some(usage) = info.buckets_usage.get(bucket) { + baseline.objects_count = baseline.objects_count.saturating_add(usage.objects_count); + baseline.bytes = baseline.bytes.saturating_add(usage.size); + } + } + + Ok(Some(baseline)) + } + + async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result> { + match self.ecstore.load_heal_lifecycle_expiry_context(bucket).await { + Ok(Some(context)) => Ok(Some(HealLifecycleExpiryContext::ecstore(context))), + Ok(None) => Ok(None), + Err(err) => { + debug!( + target: "rustfs::heal::storage", + event = EVENT_HEAL_STORAGE_ADMIN_OP, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_STORAGE, + operation = "load_heal_lifecycle_expiry_context", + bucket, + result = "failed", + error = %err, + "Heal storage lifecycle expiry context load failed" + ); + Ok(None) + } + } + } + + async fn enqueue_heal_lifecycle_expiry( + &self, + context: &HealLifecycleExpiryContext, + bucket: &str, + object: &str, + version_id: Option<&str>, + object_info: Option<&HealObjectInfo>, + ) -> Result { + let context = match &context.inner { + HealLifecycleExpiryContextInner::Ecstore(context) => context, + HealLifecycleExpiryContextInner::Test => return Ok(false), + }; + match self + .ecstore + .enqueue_heal_lifecycle_expiry(context, bucket, object, version_id, object_info) + .await + { + Ok(queued) => Ok(queued), + Err(err) => { + debug!( + target: "rustfs::heal::storage", + event = EVENT_HEAL_STORAGE_ADMIN_OP, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_STORAGE, + operation = "enqueue_heal_lifecycle_expiry", + bucket, + object, + version_id = ?version_id, + result = "failed", + error = %err, + "Heal storage lifecycle expiry check failed" + ); + Ok(false) + } + } + } + async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()> { debug!( target: "rustfs::heal::storage", @@ -1436,7 +1576,7 @@ impl HealStorageAPI for ECStoreHealStorage { loop { let (page_objects, next_token, is_truncated) = self - .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref()) + .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false) .await?; all_objects.extend(page_objects); @@ -1471,6 +1611,7 @@ impl HealStorageAPI for ECStoreHealStorage { bucket: &str, prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { debug!( target: "rustfs::heal::storage", @@ -1522,10 +1663,19 @@ impl HealStorageAPI for ECStoreHealStorage { let page_objects: Vec = list_info .objects .into_iter() - .map(|obj| HealListItem { - name: obj.name, - version_id: obj.version_id.filter(|u| !u.is_nil()).map(|u| u.to_string()), - is_delete_marker: obj.delete_marker, + .map(|mut obj| { + obj.version_id = obj.version_id.filter(|u| !u.is_nil()); + let version_id = obj.version_id.map(|u| u.to_string()); + let mod_time_unix_nanos = obj.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos()); + let is_delete_marker = obj.delete_marker; + let lifecycle_object_info = include_lifecycle_object_info.then(|| obj.clone()); + HealListItem { + name: obj.name, + version_id, + mod_time_unix_nanos, + lifecycle_object_info, + is_delete_marker, + } }) .collect(); let page_count = page_objects.len(); @@ -1562,6 +1712,7 @@ impl HealStorageAPI for ECStoreHealStorage { bucket: &str, prefix: &str, continuation_token: Option<&str>, + include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { // Per-page bounds for the disk-walk union enumerator. Objects are atomic // (never split across pages), so version_budget only bounds how many @@ -1590,7 +1741,16 @@ impl HealStorageAPI for ECStoreHealStorage { let (versions, next_forward, is_truncated) = self .ecstore - .heal_walk_versions_page(pool_idx, set_idx, bucket, prefix, forward_to.as_deref(), BATCH_OBJECTS, VERSION_BUDGET) + .heal_walk_versions_page( + pool_idx, + set_idx, + bucket, + prefix, + forward_to.as_deref(), + BATCH_OBJECTS, + VERSION_BUDGET, + include_lifecycle_object_info, + ) .await .map_err(|e| { error!( @@ -1614,6 +1774,8 @@ impl HealStorageAPI for ECStoreHealStorage { .map(|v| HealListItem { name: v.name, version_id: v.version_id, + mod_time_unix_nanos: v.mod_time_unix_nanos, + lifecycle_object_info: v.lifecycle_object_info, is_delete_marker: v.is_delete_marker, }) .collect(); diff --git a/crates/heal/src/heal/storage_api.rs b/crates/heal/src/heal/storage_api.rs index 417998e39..ff722b418 100644 --- a/crates/heal/src/heal/storage_api.rs +++ b/crates/heal/src/heal/storage_api.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub(crate) use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME; +pub(crate) use rustfs_ecstore::api::data_usage::{ + DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME, + load_admin_data_usage_from_backend_cached as ecstore_load_admin_data_usage_from_backend_cached, +}; pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint; pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult}; pub(crate) use rustfs_ecstore::api::disk::{ @@ -25,7 +28,9 @@ pub(crate) use rustfs_ecstore::api::disk::{ pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk}; pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError}; pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read; -pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore; +pub(crate) use rustfs_ecstore::api::storage::{ + ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext, +}; use rustfs_storage_api as storage_contracts; pub(crate) mod owner { @@ -34,8 +39,8 @@ pub(crate) mod owner { pub(crate) use super::{ ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, - EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, - ecstore_local_disk_map_read, + EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreHealLifecycleExpiryContext, + EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read, }; #[cfg(test)] diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index 62123c418..f6472eb65 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -19,11 +19,12 @@ use crate::heal::{ resume::{ CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match, }, - storage::{HealStorageAPI, next_heal_listing_token}, + storage::{HealBucketUsageBaseline, HealStorageAPI, next_heal_listing_token}, }; use crate::{Error, Result}; use metrics::{counter, histogram}; use rustfs_common::heal_channel::{HealOpts, HealRequestSource, HealScanMode}; +use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit}; use rustfs_madmin::heal_commands::HealResultItem; use rustfs_utils::path::SLASH_SEPARATOR; use serde::{Deserialize, Serialize}; @@ -178,6 +179,17 @@ pub enum HealPriority { Urgent = 3, } +impl HealPriority { + fn as_str(self) -> &'static str { + match self { + Self::Low => "low", + Self::Normal => "normal", + Self::High => "high", + Self::Urgent => "urgent", + } + } +} + /// Heal options #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HealOptions { @@ -498,6 +510,61 @@ impl HealTask { } } + fn emit_trace_task_state(&self, state: &'static str, duration: Duration, error: Option<&Error>) { + trace_emit(|| { + let mut event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealTask) + .with_duration(duration) + .with_attr("task_id", self.id.as_str()) + .with_attr("heal_type", self.heal_type.log_kind()) + .with_attr("state", state) + .with_attr("source", self.source.as_str()) + .with_attr("priority", self.priority.as_str()) + .with_attr("retry_attempts", u64::from(self.retry_attempts)) + .with_attr("dry_run", self.options.dry_run); + + event = match &self.heal_type { + HealType::Cluster => event, + HealType::Object { + bucket, + object, + version_id, + } => { + let event = event.with_bucket(bucket.as_str()).with_object(object.as_str()); + match version_id { + Some(version_id) => event.with_attr("version_id", version_id.as_str()), + None => event, + } + } + HealType::Bucket { bucket } => event.with_bucket(bucket.as_str()), + HealType::Prefix { bucket, prefix } => event.with_bucket(bucket.as_str()).with_object(prefix.as_str()), + HealType::ErasureSet { buckets, set_disk_id } => { + let bucket_count = u64::try_from(buckets.len()).unwrap_or(u64::MAX); + event + .with_attr("set_disk_id", set_disk_id.as_str()) + .with_attr("bucket_count", bucket_count) + } + HealType::Metadata { bucket, object } => event.with_bucket(bucket.as_str()).with_object(object.as_str()), + HealType::ECDecode { + bucket, + object, + version_id, + } => { + let event = event.with_bucket(bucket.as_str()).with_object(object.as_str()); + match version_id { + Some(version_id) => event.with_attr("version_id", version_id.as_str()), + None => event, + } + } + HealType::MRF { meta_path } => event.with_object(meta_path.as_str()), + }; + + match error { + Some(error) => event.with_attr("error", error.to_string()), + None => event, + } + }); + } + async fn remaining_timeout(&self) -> Result> { if let Some(total) = self.options.timeout { let start_instant = { *self.task_start_instant.read().await }; @@ -717,6 +784,7 @@ impl HealTask { queue_delay = ?queue_delay, "Heal task started" }); + self.emit_trace_task_state("started", Duration::ZERO, None); let result = match &self.heal_type { HealType::Cluster => self.heal_cluster().await, @@ -805,6 +873,14 @@ impl HealTask { } } + let terminal_state = match &result { + Ok(_) => "completed", + Err(Error::TaskCancelled) => "cancelled", + Err(Error::TaskTimeout) => "timed_out", + Err(_) => "failed", + }; + self.emit_trace_task_state(terminal_state, start_instant.elapsed(), result.as_ref().err()); + result } @@ -1535,7 +1611,7 @@ impl HealTask { let (objects, next_token, is_truncated) = self .await_with_control( self.storage - .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref()), + .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false), ) .await?; @@ -1697,6 +1773,23 @@ impl HealTask { Ok(()) } + async fn apply_erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<()> { + let baseline = match self + .await_with_control(self.storage.erasure_set_usage_baseline(buckets)) + .await + { + Ok(Some(baseline)) => baseline, + Ok(None) => return Ok(()), + Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err), + Err(_) => return Ok(()), + }; + + let HealBucketUsageBaseline { objects_count, bytes } = baseline; + let mut progress = self.progress.write().await; + progress.set_total_baseline(objects_count, bytes); + Ok(()) + } + async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> { debug!( target: "rustfs::heal::task", @@ -2298,6 +2391,8 @@ impl HealTask { None }; + self.apply_erasure_set_usage_baseline(&buckets).await?; + let healing_marker = format!("{set_disk_id}:{}", self.id); if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() { let state = resume_manager.get_state().await; @@ -2602,7 +2697,8 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(4, 4, 0, 0); + let bytes_processed = progress.bytes_processed; + progress.update_progress(4, 4, 0, bytes_processed); } match result { @@ -2658,6 +2754,7 @@ mod tests { use super::super::{DiskOption, DiskStore, Endpoint, HealDiskExt as _, new_disk}; use super::*; use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo}; + use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events}; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; use std::collections::{HashMap, VecDeque}; use std::sync::Mutex; @@ -3203,6 +3300,8 @@ mod tests { block_heal_object: Mutex, resume_disk: Mutex>, replacement_resume_disk: Mutex>, + usage_baseline: Mutex>, + usage_baseline_error: Mutex, } #[test] @@ -3265,11 +3364,69 @@ mod tests { assert_eq!(samples_logged, MAX_BUCKET_FAILURE_LOG_SAMPLES); } + #[tokio::test] + async fn execute_emits_heal_trace_task_state() { + let mut trace = subscribe_trace_events(); + let storage = Arc::new(MockStorage::default()); + let task = HealTask::from_request( + HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())), + storage, + ); + + task.execute().await.expect("mock object heal should complete"); + + let started = recv_trace_task_state(&mut trace, &task.id, "started").await; + assert_eq!(started.kind, TraceKind::Heal); + assert_eq!(started.func, TraceFunc::HealTask); + assert_eq!(started.bucket.as_deref(), Some("bucket-a")); + assert_eq!(started.object.as_deref(), Some("object-a")); + assert_eq!(trace_attr_string(&started, "heal_type").as_deref(), Some("object")); + assert_eq!(trace_attr_string(&started, "source").as_deref(), Some("internal")); + assert_eq!(trace_attr_string(&started, "version_id").as_deref(), Some("version-a")); + + let completed = recv_trace_task_state(&mut trace, &task.id, "completed").await; + assert_eq!(completed.kind, TraceKind::Heal); + assert_eq!(completed.func, TraceFunc::HealTask); + assert_eq!(trace_attr_string(&completed, "state").as_deref(), Some("completed")); + } + + async fn recv_trace_task_state(trace: &mut TraceSubscription, task_id: &str, state: &str) -> TraceEvent { + for _ in 0..32 { + let event = tokio::time::timeout(Duration::from_secs(1), trace.recv()) + .await + .expect("trace event should arrive") + .expect("trace bus should stay open"); + if trace_attr_string(&event, "task_id").as_deref() == Some(task_id) + && trace_attr_string(&event, "state").as_deref() == Some(state) + { + return (*event).clone(); + } + } + + panic!("expected trace state {state} for task {task_id}"); + } + + fn trace_attr_string(event: &TraceEvent, key: &str) -> Option { + event.attrs.iter().find_map(|attr| { + if attr.key != key { + return None; + } + Some(match &attr.value { + TraceVal::Bool(value) => value.to_string(), + TraceVal::U64(value) => value.to_string(), + TraceVal::I64(value) => value.to_string(), + TraceVal::Str(value) => value.to_string(), + }) + }) + } + /// Build a latest, non-delete-marker heal list item with no version id. fn heal_item(name: &str) -> HealListItem { HealListItem { name: name.to_string(), version_id: None, + mod_time_unix_nanos: None, + lifecycle_object_info: None, is_delete_marker: false, } } @@ -3357,6 +3514,13 @@ mod tests { })) } + async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result> { + if *self.usage_baseline_error.lock().unwrap() { + return Err(Error::Other("usage baseline unavailable".to_string())); + } + Ok(*self.usage_baseline.lock().unwrap()) + } + async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> { Ok(()) } @@ -3540,6 +3704,7 @@ mod tests { bucket: &str, prefix: &str, continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { self.listed_prefixes.lock().unwrap().push(prefix.to_string()); if *self.truncate_without_token.lock().unwrap() { @@ -4654,6 +4819,73 @@ mod tests { assert!(storage.object_heal_opts.lock().unwrap().is_empty()); } + #[tokio::test] + async fn erasure_set_heal_applies_usage_baseline_to_progress() { + let temp = TempDir::new().expect("temporary directory should be created"); + let disk = make_resume_disk(&temp).await; + let storage = Arc::new(MockStorage { + resume_disk: Mutex::new(Some(disk)), + usage_baseline: Mutex::new(Some(HealBucketUsageBaseline { + objects_count: 10, + bytes: 8, + })), + ..Default::default() + }); + let request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ); + let task = HealTask::from_request(request, storage); + + task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string()) + .await + .expect("erasure set heal should complete"); + + let progress = task.get_progress().await; + assert_eq!(progress.objects_total_count, 10); + assert_eq!(progress.objects_total_size, 8); + assert_eq!(progress.bytes_processed, 2); + assert!((progress.progress_percentage - 25.0).abs() < 0.001); + } + + #[tokio::test] + async fn erasure_set_heal_ignores_usage_baseline_errors() { + let temp = TempDir::new().expect("temporary directory should be created"); + let disk = make_resume_disk(&temp).await; + let storage = Arc::new(MockStorage { + resume_disk: Mutex::new(Some(disk)), + usage_baseline_error: Mutex::new(true), + ..Default::default() + }); + let request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ); + let task = HealTask::from_request(request, storage); + + task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string()) + .await + .expect("usage baseline failures should not fail erasure set heal"); + + let progress = task.get_progress().await; + assert_eq!(progress.objects_total_count, 0); + assert_eq!(progress.objects_total_size, 0); + } + #[tokio::test] async fn resumable_erasure_set_execution_is_cancelled_while_object_heal_is_pending() { let temp = TempDir::new().expect("temporary directory should be created"); diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index 7c156304d..3dd29b064 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -445,6 +445,7 @@ mod tests { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool), Error> { Ok((Vec::new(), None, false)) } diff --git a/crates/heal/tests/heal_b5_versioned_regression_test.rs b/crates/heal/tests/heal_b5_versioned_regression_test.rs index 61a542955..95e0f0a90 100644 --- a/crates/heal/tests/heal_b5_versioned_regression_test.rs +++ b/crates/heal/tests/heal_b5_versioned_regression_test.rs @@ -176,7 +176,7 @@ async fn enumerate_all_versions(heal_storage: &Arc, bucket: let mut token: Option = None; loop { let (page, next, truncated) = heal_storage - .list_objects_for_heal_page(bucket, "", token.as_deref()) + .list_objects_for_heal_page(bucket, "", token.as_deref(), false) .await .expect("list_objects_for_heal_page failed"); items.extend(page); diff --git a/crates/heal/tests/heal_b920_subquorum_union_test.rs b/crates/heal/tests/heal_b920_subquorum_union_test.rs index 6d188b3f3..8d4ac65f2 100644 --- a/crates/heal/tests/heal_b920_subquorum_union_test.rs +++ b/crates/heal/tests/heal_b920_subquorum_union_test.rs @@ -166,7 +166,7 @@ async fn enumerate_b5(heal_storage: &Arc, bucket: &str) -> V let mut token: Option = None; loop { let (page, next, truncated) = heal_storage - .list_objects_for_heal_page(bucket, "", token.as_deref()) + .list_objects_for_heal_page(bucket, "", token.as_deref(), false) .await .expect("b5 list page failed"); items.extend(page); @@ -187,7 +187,7 @@ async fn enumerate_disk_walk(heal_storage: &Arc, bucket: &st let mut token: Option = None; loop { let (page, next, truncated) = heal_storage - .list_versions_for_heal_page_disk_walk(SET_DISK_ID, bucket, "", token.as_deref()) + .list_versions_for_heal_page_disk_walk(SET_DISK_ID, bucket, "", token.as_deref(), false) .await .expect("disk-walk list page failed"); items.extend(page); @@ -418,7 +418,7 @@ mod serial_tests { let mut pages = 0usize; loop { let (versions, next_forward, truncated) = ecstore - .heal_walk_versions_page(0, 0, bucket, "", forward.as_deref(), 2, 100_000) + .heal_walk_versions_page(0, 0, bucket, "", forward.as_deref(), 2, 100_000, false) .await .expect("heal_walk_versions_page failed"); pages += 1; diff --git a/crates/heal/tests/heal_bug_fixes_test.rs b/crates/heal/tests/heal_bug_fixes_test.rs index 7b8737f19..ba59142fd 100644 --- a/crates/heal/tests/heal_bug_fixes_test.rs +++ b/crates/heal/tests/heal_bug_fixes_test.rs @@ -242,6 +242,7 @@ fn test_heal_task_status_atomic_update() { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> rustfs_heal::Result<(Vec, Option, bool)> { Ok((vec![], None, false)) } @@ -385,6 +386,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> rustfs_heal::Result<(Vec, Option, bool)> { Ok((Vec::new(), None, false)) } diff --git a/crates/madmin/src/service_commands.rs b/crates/madmin/src/service_commands.rs index ddda978be..1b5d0b2a9 100644 --- a/crates/madmin/src/service_commands.rs +++ b/crates/madmin/src/service_commands.rs @@ -43,7 +43,7 @@ pub struct ServiceTraceOpts { #[allow(dead_code)] impl ServiceTraceOpts { - fn trace_types(&self) -> TraceType { + pub fn trace_types(&self) -> TraceType { let mut tt = TraceType::default(); tt.set_if(self.s3, &TraceType::S3); tt.set_if(self.internal, &TraceType::INTERNAL); @@ -72,6 +72,14 @@ impl ServiceTraceOpts { tt } + pub fn only_errors(&self) -> bool { + self.only_errors + } + + pub fn threshold(&self) -> Duration { + self.threshold + } + pub fn parse_params(&mut self, uri: &Uri) -> Result<(), String> { let query_pairs: HashMap<_, _> = uri .query() diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 25f1e758d..820cbb12f 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -41,6 +41,7 @@ use rustfs_common::metrics::{ CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource, UpdateCurrentPathFn, current_path_updater, global_metrics, }; +use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit, trace_subscriber_count}; use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration}; @@ -430,6 +431,113 @@ fn non_negative_i64_to_u64(value: i64) -> u64 { value.max(0) as u64 } +fn trace_start_instant() -> Option { + (trace_subscriber_count() > 0).then(Instant::now) +} + +fn emit_scanner_folder_trace(root: &str, folder: &str, objects: u64, started_at: Option, state: &'static str) { + let Some(started_at) = started_at else { + return; + }; + + trace_emit(|| { + let (bucket, prefix) = path2_bucket_object_with_base_path(root, folder); + TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder) + .with_bucket(bucket) + .with_object(prefix) + .with_duration(started_at.elapsed()) + .with_attr("state", state) + .with_attr("objects", objects) + }); +} + +fn emit_scanner_ilm_action_trace( + bucket: &str, + object: &str, + action: IlmAction, + count: u64, + queued: bool, + started_at: Option, +) { + let Some(started_at) = started_at else { + return; + }; + + let state = if queued { "queued" } else { "not_queued" }; + trace_emit(|| { + TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerIlmAction) + .with_bucket(bucket) + .with_object(object) + .with_duration(started_at.elapsed()) + .with_attr("state", state) + .with_attr("action", action.as_str()) + .with_attr("count", count) + .with_attr("queued", queued) + }); +} + +struct ScannerHealCandidateTraceContext { + bucket: String, + object: Option, + version_id: Option, + scan_mode: Option, + started_at: Instant, +} + +fn scanner_heal_candidate_trace_context(request: &HealChannelRequest) -> Option { + let started_at = trace_start_instant()?; + Some(ScannerHealCandidateTraceContext { + bucket: request.bucket.clone(), + object: request.object_prefix.clone(), + version_id: request.object_version_id.clone(), + scan_mode: request.scan_mode, + started_at, + }) +} + +struct ScannerHealCandidateTrace<'a> { + candidate_type: &'static str, + bucket: &'a str, + object: Option<&'a str>, + version_id: Option<&'a str>, + priority: HealChannelPriority, + scan_mode: Option, + result: Result, + started_at: Instant, +} + +fn emit_scanner_heal_candidate_trace(trace: ScannerHealCandidateTrace<'_>) { + trace_emit(|| { + let (state, admission, error) = match trace.result { + Ok(result) if result.is_admitted() => ("admitted", describe_heal_admission(result), None), + Ok(result) => ("not_admitted", describe_heal_admission(result), None), + Err(error) => ("submit_failed", "channel_error".to_string(), Some(error)), + }; + let mut event = TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerHealCandidate) + .with_bucket(trace.bucket) + .with_duration(trace.started_at.elapsed()) + .with_attr("state", state) + .with_attr("candidate_type", trace.candidate_type) + .with_attr("priority", heal_priority_label(trace.priority)) + .with_attr("admission", admission); + + if let Some(object) = trace.object { + event = event.with_object(object); + } + if let Some(version_id) = trace.version_id { + event = event.with_attr("version_id", version_id); + } + if let Some(scan_mode) = trace.scan_mode { + event = event.with_attr("scan_mode", scan_mode.as_str()); + } + if let Some(error) = error { + event = event.with_attr("error", error); + } + + event + }); +} + fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary) { into.size = into.size.saturating_add(summary.total_size); into.versions = into.versions.saturating_add(summary.versions); @@ -677,9 +785,22 @@ async fn send_scanner_heal_request( request: HealChannelRequest, ) -> Result { let priority = request.priority; + let trace_context = scanner_heal_candidate_trace_context(&request); match send_heal_request_with_admission(request).await { Ok(result) => { record_heal_candidate_admission(candidate_type, priority, result); + if let Some(trace_context) = trace_context.as_ref() { + emit_scanner_heal_candidate_trace(ScannerHealCandidateTrace { + candidate_type, + bucket: &trace_context.bucket, + object: trace_context.object.as_deref(), + version_id: trace_context.version_id.as_deref(), + priority, + scan_mode: trace_context.scan_mode, + result: Ok(result), + started_at: trace_context.started_at, + }); + } Ok(result) } Err(err) => { @@ -690,6 +811,18 @@ async fn send_scanner_heal_request( "result" => "channel_error".to_string() ) .increment(1); + if let Some(trace_context) = trace_context.as_ref() { + emit_scanner_heal_candidate_trace(ScannerHealCandidateTrace { + candidate_type, + bucket: &trace_context.bucket, + object: trace_context.object.as_deref(), + version_id: trace_context.version_id.as_deref(), + priority, + scan_mode: trace_context.scan_mode, + result: Err(err.as_str()), + started_at: trace_context.started_at, + }); + } Err(ScannerError::Other(err)) } } @@ -905,7 +1038,9 @@ impl ScannerItem { "Scanner lifecycle action dispatched" ); let done_ilm = Metrics::time_ilm(event.action); + let trace_started_at = trace_start_instant(); let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; + emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at); if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) { done_ilm(1)(); remaining_versions = 0; @@ -957,7 +1092,9 @@ impl ScannerItem { "Scanner lifecycle action dispatched" ); let done_ilm = Metrics::time_ilm(event.action); + let trace_started_at = trace_start_instant(); let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; + emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at); if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) { done_ilm(1)(); if !versioning_config.prefix_enabled(&self.object_path()) && event.action == IlmAction::DeleteAction { @@ -995,7 +1132,9 @@ impl ScannerItem { "Scanner lifecycle action dispatched" ); let done_ilm = Metrics::time_ilm(event.action); + let trace_started_at = trace_start_instant(); let queued = apply_transition_rule(event, &LcEventSrc::Scanner, oi).await; + emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at); if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) { done_ilm(1)(); } @@ -1019,7 +1158,21 @@ impl ScannerItem { let action = event.action; let count = u64::try_from(to_delete_objs.len()).unwrap_or(u64::MAX); let done_ilm = Metrics::time_ilm(action); + let trace_started_at = trace_start_instant(); let queued = enqueue_runtime_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner).await; + if let Some(trace_started_at) = trace_started_at { + let state = if queued { "queued" } else { "not_queued" }; + trace_emit(|| { + TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerIlmAction) + .with_bucket(self.bucket.as_str()) + .with_object(self.object_path()) + .with_duration(trace_started_at.elapsed()) + .with_attr("state", state) + .with_attr("action", action.as_str()) + .with_attr("count", count) + .with_attr("queued", queued) + }); + } if record_scanner_ilm_action_if_queued(global_metrics(), action, count, queued) { done_ilm(count)(); remaining_versions = remaining_versions.saturating_sub(noncurrent_accounting.len()); @@ -1830,6 +1983,7 @@ impl FolderScanner { into: &mut DataUsageEntry, ) -> Result<(), ScannerError> { let done_folder = Metrics::time(Metric::ScanFolder); + let trace_started_at = trace_start_instant(); if ctx.is_cancelled() { return Err(ScannerError::Other("Operation cancelled".to_string())); @@ -2895,6 +3049,8 @@ impl FolderScanner { } done_folder(); + let scanned_objects = u64::try_from(into.objects).unwrap_or(u64::MAX); + emit_scanner_folder_trace(&self.root, &folder.name, scanned_objects, trace_started_at, "completed"); Ok(()) } @@ -4400,6 +4556,104 @@ mod tests { ); } + #[tokio::test] + async fn scanner_trace_helpers_emit_expected_events() { + let mut trace = rustfs_common::trace_bus::subscribe_trace_events(); + + emit_scanner_folder_trace( + "/tmp/rustfs-scanner-trace", + "/tmp/rustfs-scanner-trace/bucket-a/folder-a", + 7, + Some(Instant::now()), + "completed", + ); + let folder = recv_scanner_trace_event( + &mut trace, + TraceFunc::ScannerFolder, + Some("bucket-a"), + Some("folder-a"), + Some("completed"), + ) + .await; + assert_eq!(trace_attr_string(&folder, "objects").as_deref(), Some("7")); + + emit_scanner_ilm_action_trace("bucket-a", "object-a", IlmAction::DeleteAction, 2, true, Some(Instant::now())); + let ilm = recv_scanner_trace_event( + &mut trace, + TraceFunc::ScannerIlmAction, + Some("bucket-a"), + Some("object-a"), + Some("queued"), + ) + .await; + assert_eq!(trace_attr_string(&ilm, "action").as_deref(), Some("delete")); + assert_eq!(trace_attr_string(&ilm, "count").as_deref(), Some("2")); + assert_eq!(trace_attr_string(&ilm, "queued").as_deref(), Some("true")); + + emit_scanner_heal_candidate_trace(ScannerHealCandidateTrace { + candidate_type: "object", + bucket: "bucket-a", + object: Some("object-a"), + version_id: Some("version-a"), + priority: HealChannelPriority::High, + scan_mode: Some(HealScanMode::Deep), + result: Ok(HealAdmissionResult::Merged), + started_at: Instant::now(), + }); + let heal_candidate = recv_scanner_trace_event( + &mut trace, + TraceFunc::ScannerHealCandidate, + Some("bucket-a"), + Some("object-a"), + Some("admitted"), + ) + .await; + assert_eq!(trace_attr_string(&heal_candidate, "candidate_type").as_deref(), Some("object")); + assert_eq!(trace_attr_string(&heal_candidate, "priority").as_deref(), Some("high")); + assert_eq!(trace_attr_string(&heal_candidate, "scan_mode").as_deref(), Some("deep")); + assert_eq!(trace_attr_string(&heal_candidate, "version_id").as_deref(), Some("version-a")); + assert_eq!(trace_attr_string(&heal_candidate, "admission").as_deref(), Some("merged")); + } + + async fn recv_scanner_trace_event( + trace: &mut rustfs_common::trace_bus::TraceSubscription, + func: TraceFunc, + bucket: Option<&str>, + object: Option<&str>, + state: Option<&str>, + ) -> TraceEvent { + for _ in 0..32 { + let event = tokio::time::timeout(Duration::from_secs(1), trace.recv()) + .await + .expect("scanner trace event should arrive") + .expect("trace bus should stay open"); + if event.kind == TraceKind::Scanner + && event.func == func + && event.bucket.as_deref() == bucket + && event.object.as_deref() == object + && state.is_none_or(|state| trace_attr_string(&event, "state").as_deref() == Some(state)) + { + return (*event).clone(); + } + } + + panic!("expected scanner trace event {func:?} for bucket {bucket:?} object {object:?}"); + } + + fn trace_attr_string(event: &TraceEvent, key: &str) -> Option { + event.attrs.iter().find_map(|attr| { + if attr.key != key { + return None; + } + Some(match &attr.value { + rustfs_common::trace_bus::TraceVal::Bool(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::U64(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::I64(value) => value.to_string(), + rustfs_common::trace_bus::TraceVal::Str(value) => value.to_string(), + }) + }) + } + #[test] fn test_build_high_priority_heal_admission_error_contains_context() { let err = build_high_priority_heal_admission_error( diff --git a/rustfs/src/admin/handlers/profile_admin.rs b/rustfs/src/admin/handlers/profile_admin.rs index d4a85360a..d14af9f3b 100644 --- a/rustfs/src/admin/handlers/profile_admin.rs +++ b/rustfs/src/admin/handlers/profile_admin.rs @@ -23,18 +23,24 @@ use futures::{Stream, StreamExt}; use http::{HeaderMap, HeaderValue}; use hyper::{Method, StatusCode}; use matchit::Params; +use regex::Regex; +use rustfs_common::trace_bus::{TraceEvent, TraceKind, TraceVal, subscribe_trace_events}; use rustfs_madmin::service_commands::ServiceTraceOpts; +use rustfs_madmin::trace::TraceType; use rustfs_policy::policy::action::{Action, AdminAction}; use s3s::header::CONTENT_TYPE; use s3s::stream::{ByteStream, DynByteStream}; use s3s::{Body, S3Request, S3Response, S3Result, StdError, s3_error}; use serde::Serialize; +use std::collections::HashMap; use std::pin::Pin; use std::task::{Context, Poll}; -use std::time::Duration; +use std::time::{Duration, SystemTime}; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tracing::error; +use url::form_urlencoded; #[derive(Serialize)] struct ProfileStatus { @@ -206,16 +212,164 @@ impl Stream for TraceStream { impl ByteStream for TraceStream {} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct TraceKindFilter { + heal: bool, + scanner: bool, +} + +impl TraceKindFilter { + const ALL_SUPPORTED: Self = Self { + heal: true, + scanner: true, + }; + + fn from_request(uri: &hyper::Uri, trace_types: TraceType) -> S3Result { + let mut has_kind = false; + let mut filter = Self { + heal: false, + scanner: false, + }; + + for (key, value) in trace_query_pairs(uri) { + if key != "kind" { + continue; + } + has_kind = true; + for item in value.split(',') { + match item.trim().to_ascii_lowercase().as_str() { + "heal" | "healing" => filter.heal = true, + "scanner" => filter.scanner = true, + "all" => return Ok(Self::ALL_SUPPORTED), + _ => return Err(s3_error!(InvalidRequest, "invalid trace kind")), + } + } + } + + if has_kind { + return Ok(filter); + } + + if trace_types.mask() == 0 || trace_query_flag(uri, "all") { + return Ok(Self::ALL_SUPPORTED); + } + + Ok(Self { + heal: trace_types.overlaps(&TraceType::HEALING), + scanner: trace_types.overlaps(&TraceType::SCANNER), + }) + } + + const fn matches(self, kind: TraceKind) -> bool { + match kind { + TraceKind::Heal => self.heal, + TraceKind::Scanner => self.scanner, + } + } +} + +#[derive(Debug)] +struct TraceStreamFilter { + kinds: TraceKindFilter, + regex: Option, + threshold: Duration, +} + +impl TraceStreamFilter { + fn from_request(uri: &hyper::Uri, opts: &ServiceTraceOpts) -> S3Result { + if opts.only_errors() { + return Err(s3_error!( + InvalidRequest, + "trace error-only filter is not supported for heal/scanner trace" + )); + } + + Ok(Self { + kinds: TraceKindFilter::from_request(uri, opts.trace_types())?, + regex: trace_regex_filter(uri)?, + threshold: opts.threshold(), + }) + } + + fn matches_kind(&self, kind: TraceKind) -> bool { + self.kinds.matches(kind) + } + + fn matches_record(&self, record: &TraceWireRecord) -> bool { + record.duration >= self.threshold && self.regex.as_ref().is_none_or(|regex| record.matches_regex(regex)) + } +} + +#[derive(Serialize)] +struct TraceWireRecord { + #[serde(rename = "type")] + trace_type: u64, + #[serde(rename = "nodename")] + node_name: String, + #[serde(rename = "funcname")] + func_name: String, + #[serde(rename = "time")] + time: String, + #[serde(rename = "path")] + path: String, + #[serde(rename = "dur")] + duration: Duration, + #[serde(rename = "bytes", skip_serializing_if = "Option::is_none")] + bytes: Option, + #[serde(rename = "msg", skip_serializing_if = "Option::is_none")] + message: Option, + #[serde(rename = "custom", skip_serializing_if = "Option::is_none")] + custom: Option>, +} + +impl TraceWireRecord { + fn from_event(node_name: &str, event: &TraceEvent) -> Self { + Self { + trace_type: trace_type_mask(event.kind), + node_name: node_name.to_owned(), + func_name: event.func.as_str().to_owned(), + time: trace_time_string(event.time), + path: trace_path(event), + duration: event.duration, + bytes: trace_bytes(event.bytes), + message: None, + custom: trace_custom_attrs(event), + } + } + + fn dropped(node_name: &str, dropped: u64) -> Self { + let mut custom = HashMap::new(); + custom.insert("dropped_events".to_string(), dropped.to_string()); + + Self { + trace_type: 0, + node_name: node_name.to_owned(), + func_name: "trace.Dropped".to_string(), + time: trace_time_string(SystemTime::now()), + path: String::new(), + duration: Duration::ZERO, + bytes: None, + message: Some("trace subscriber lagged".to_string()), + custom: Some(custom), + } + } + + fn matches_regex(&self, regex: &Regex) -> bool { + regex.is_match(&self.func_name) + || regex.is_match(&self.path) + || self.message.as_ref().is_some_and(|message| regex.is_match(message)) + || self + .custom + .as_ref() + .is_some_and(|custom| custom.iter().any(|(key, value)| regex.is_match(key) || regex.is_match(value))) + } +} + /// `GET /v3/trace` — stream real-time server trace events. /// -/// RustFS emits diagnostics through the `tracing` pipeline but does not expose -/// an in-process subscriber that can fan trace events out to an admin client -/// (there is no request-trace broadcast channel). Rather than return an opaque -/// `501` — which would make `mc admin trace` fail to connect — this honors the -/// streaming NDJSON contract: it validates the requested trace filters, opens -/// the stream, emits a single capability record explaining that live tracing is -/// not wired, then holds the connection open with keep-alives. No fabricated -/// trace records are ever sent. +/// RustFS currently publishes heal and scanner diagnostics through the common +/// trace bus. The admin endpoint exposes those events as MinIO-shaped NDJSON +/// records while keeping unsupported trace classes filtered out. pub struct TraceHandler {} #[async_trait::async_trait] @@ -228,24 +382,13 @@ impl Operation for TraceHandler { let mut opts = ServiceTraceOpts::default(); opts.parse_params(&req.uri) .map_err(|_| s3_error!(InvalidRequest, "invalid trace parameters"))?; + let filter = TraceStreamFilter::from_request(&req.uri, &opts)?; let node_name = sysinfo::System::host_name().unwrap_or_else(|| "rustfs".to_string()); - let (tx, rx) = mpsc::channel::>(8); + let mut subscription = subscribe_trace_events(); + let (tx, rx) = mpsc::channel::>(64); spawn_traced(async move { - let notice = serde_json::json!({ - "nodename": node_name, - "funcname": "admin.Trace", - "msg": "RustFS does not expose an in-process trace-event subscriber; live tracing is not yet available", - "err": "trace_streaming_unsupported", - }); - if let Ok(mut encoded) = serde_json::to_vec(¬ice) { - encoded.push(b'\n'); - if tx.send(Ok(Bytes::from(encoded))).await.is_err() { - return; - } - } - let mut ticker = tokio::time::interval(Duration::from_secs(15)); ticker.tick().await; loop { @@ -256,6 +399,26 @@ impl Operation for TraceHandler { break; } } + received = subscription.recv() => { + match received { + Ok(event) => { + if !filter.matches_kind(event.kind) { + continue; + } + let record = TraceWireRecord::from_event(&node_name, &event); + if filter.matches_record(&record) && send_trace_record(&tx, &record).await.is_err() { + break; + } + } + Err(tokio::sync::broadcast::error::RecvError::Lagged(dropped)) => { + let record = TraceWireRecord::dropped(&node_name, dropped); + if send_trace_record(&tx, &record).await.is_err() { + break; + } + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + } + } } } }); @@ -269,17 +432,115 @@ impl Operation for TraceHandler { } } +async fn send_trace_record(tx: &mpsc::Sender>, record: &TraceWireRecord) -> Result<(), ()> { + let Some(encoded) = encode_ndjson(record) else { + return Ok(()); + }; + tx.send(Ok(encoded)).await.map_err(|_| ()) +} + +fn encode_ndjson(value: &impl Serialize) -> Option { + let mut encoded = serde_json::to_vec(value).ok()?; + encoded.push(b'\n'); + Some(Bytes::from(encoded)) +} + +fn trace_query_pairs(uri: &hyper::Uri) -> impl Iterator + '_ { + uri.query() + .into_iter() + .flat_map(|query| form_urlencoded::parse(query.as_bytes())) + .map(|(key, value)| (key.into_owned(), value.into_owned())) +} + +fn trace_query_flag(uri: &hyper::Uri, flag: &str) -> bool { + trace_query_pairs(uri).any(|(key, value)| key == flag && value == "true") +} + +fn trace_regex_filter(uri: &hyper::Uri) -> S3Result> { + trace_query_pairs(uri) + .find_map(|(key, value)| { + if key == "filter" && !value.is_empty() { + Some(value) + } else { + None + } + }) + .map(|pattern| Regex::new(&pattern).map_err(|_| s3_error!(InvalidRequest, "invalid trace filter"))) + .transpose() +} + +fn trace_type_mask(kind: TraceKind) -> u64 { + match kind { + TraceKind::Heal => TraceType::HEALING.mask(), + TraceKind::Scanner => TraceType::SCANNER.mask(), + } +} + +fn trace_time_string(time: SystemTime) -> String { + match OffsetDateTime::from(time).format(&Rfc3339) { + Ok(value) => value, + Err(_) => "1970-01-01T00:00:00Z".to_string(), + } +} + +fn trace_path(event: &TraceEvent) -> String { + match (event.bucket.as_deref(), event.object.as_deref()) { + (Some(bucket), Some(object)) if !object.is_empty() => format!("{bucket}/{object}"), + (Some(bucket), _) => bucket.to_owned(), + (None, Some(object)) => object.to_owned(), + (None, None) => String::new(), + } +} + +fn trace_bytes(bytes: u64) -> Option { + if bytes == 0 { + return None; + } + + match i64::try_from(bytes) { + Ok(value) => Some(value), + Err(_) => Some(i64::MAX), + } +} + +fn trace_custom_attrs(event: &TraceEvent) -> Option> { + if event.attrs.is_empty() { + return None; + } + + Some( + event + .attrs + .iter() + .map(|attr| (attr.key.to_string(), trace_value_string(&attr.value))) + .collect(), + ) +} + +fn trace_value_string(value: &TraceVal) -> String { + match value { + TraceVal::Bool(value) => value.to_string(), + TraceVal::U64(value) => value.to_string(), + TraceVal::I64(value) => value.to_string(), + TraceVal::Str(value) => value.to_string(), + } +} + #[cfg(test)] mod tests { use super::{ ProfileControlHandler, ProfileHandler, ProfileStatusHandler, ProfilingDownloadHandler, ProfilingStartHandler, - TraceHandler, + TraceHandler, TraceKindFilter, TraceStreamFilter, TraceWireRecord, }; use crate::admin::router::Operation; use http::{Extensions, HeaderMap, Uri}; use hyper::Method; use matchit::Params; - use s3s::{Body, S3ErrorCode, S3Request}; + use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind}; + use rustfs_madmin::service_commands::ServiceTraceOpts; + use rustfs_madmin::trace::TraceType; + use s3s::{Body, S3ErrorCode, S3Request, S3Result}; + use std::time::{Duration, UNIX_EPOCH}; fn build_profile_request(uri: &'static str) -> S3Request { S3Request { @@ -295,6 +556,13 @@ mod tests { } } + fn build_trace_stream_filter(uri: &'static str) -> S3Result { + let uri = Uri::from_static(uri); + let mut opts = ServiceTraceOpts::default(); + opts.parse_params(&uri).expect("test trace params should parse"); + TraceStreamFilter::from_request(&uri, &opts) + } + #[tokio::test] async fn profile_handler_rejects_missing_credentials() { let result = ProfileHandler {} @@ -358,4 +626,108 @@ mod tests { .expect_err("trace must reject anonymous requests"); assert_eq!(err.code(), &S3ErrorCode::AccessDenied); } + + #[test] + fn trace_kind_filter_supports_kind_query() { + let uri = Uri::from_static("/rustfs/admin/v3/trace?kind=heal"); + let filter = TraceKindFilter::from_request(&uri, TraceType::default()).expect("kind filter should parse"); + + assert!(filter.matches(TraceKind::Heal)); + assert!(!filter.matches(TraceKind::Scanner)); + } + + #[test] + fn trace_kind_filter_defaults_to_supported_events_without_type_flags() { + let uri = Uri::from_static("/rustfs/admin/v3/trace"); + let filter = TraceKindFilter::from_request(&uri, TraceType::default()).expect("empty filter should parse"); + + assert!(filter.matches(TraceKind::Heal)); + assert!(filter.matches(TraceKind::Scanner)); + } + + #[test] + fn trace_kind_filter_rejects_unknown_kind() { + let uri = Uri::from_static("/rustfs/admin/v3/trace?kind=s3"); + let err = TraceKindFilter::from_request(&uri, TraceType::default()).expect_err("unknown kind should fail"); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[test] + fn trace_stream_filter_matches_regex_against_path_and_attrs() { + let filter = build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&filter=data/.%2Bxl.meta") + .expect("regex filter should parse"); + let event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject) + .with_bucket("data") + .with_object("dir/xl.meta") + .with_attr("dry_run", true); + let record = TraceWireRecord::from_event("node-a", &event); + + assert!(filter.matches_kind(event.kind)); + assert!(filter.matches_record(&record)); + } + + #[test] + fn trace_stream_filter_rejects_invalid_regex() { + let err = build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&filter=[").expect_err("invalid regex should fail"); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[test] + fn trace_stream_filter_applies_threshold() { + let filter = + build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&threshold=10ms").expect("threshold should parse"); + let short = TraceWireRecord::from_event( + "node-a", + &TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject).with_duration(Duration::from_millis(9)), + ); + let long = TraceWireRecord::from_event( + "node-a", + &TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject).with_duration(Duration::from_millis(10)), + ); + + assert!(!filter.matches_record(&short)); + assert!(filter.matches_record(&long)); + } + + #[test] + fn trace_stream_filter_rejects_error_only_filter() { + let err = build_trace_stream_filter("/rustfs/admin/v3/trace?kind=heal&err=true").expect_err("err filter should fail"); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } + + #[test] + fn trace_wire_record_contains_madmin_trace_fields() { + let event = TraceEvent::new(TraceKind::Heal, TraceFunc::HealObject) + .with_bucket("bucket") + .with_object("object") + .with_duration(Duration::from_millis(3)) + .with_bytes(17) + .with_attr("dry", true); + let mut record = TraceWireRecord::from_event("node-a", &event); + record.time = "1970-01-01T00:00:00Z".to_string(); + + let value = serde_json::to_value(&record).expect("trace record should serialize"); + + assert_eq!(value["type"], TraceType::HEALING.mask()); + assert_eq!(value["nodename"], "node-a"); + assert_eq!(value["funcname"], "heal.Object"); + assert_eq!(value["time"], "1970-01-01T00:00:00Z"); + assert_eq!(value["path"], "bucket/object"); + assert_eq!(value["bytes"], 17); + assert_eq!(value["custom"]["dry"], "true"); + } + + #[test] + fn trace_wire_record_formats_epoch_time() { + let event = TraceEvent { + time: UNIX_EPOCH, + ..TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerFolder) + }; + let record = TraceWireRecord::from_event("node-a", &event); + + assert_eq!(record.time, "1970-01-01T00:00:00Z"); + } } diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index d06e0892f..a39ca6c3a 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -2420,6 +2420,7 @@ mod tests { _bucket: &str, _prefix: &str, _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, ) -> rustfs_heal::Result<(Vec, Option, bool)> { Ok((Vec::new(), None, false)) } From 35a30cd6144f5f37f3b03e2964b73655ed408274 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 08:46:32 +0800 Subject: [PATCH 09/22] feat(scanner): emit excess alerts as S3 notification events (HS-04) (#6176) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(scanner): emit excess alerts as S3 notification events The excess-versions / excess-version-size / excess-folders alerts were metrics-and-logs only; consoles and external auditors had no way to hear them (rustfs/backlog#1868, HS-04). MinIO emits s3:ObjectManyVersions / s3:ObjectLargeVersions / s3:PrefixManyFolders for the same conditions — RustFS carries those as EventName::Scanner* with s3:Scanner:* wire names that already existed unpublished. The three alert sites now also dispatch through the standard event pipeline (send_event via the storage_api owner facade), carrying the actual values and thresholds in req_params and UserAgent "Scanner". Without a cooldown a single over-threshold object would re-emit on every ~60s scan cycle, so emissions are edge-held per (kind, bucket, object) for 24h (RUSTFS_SCANNER_ALERT_COOLDOWN_SECS, 0 = every cycle), backed by a process-global map with a 4096-key hard cap that clears rather than grows. Metrics and structured logs stay level-triggered every cycle; only the notification events are held back. A restart resets the cooldown deliberately: one re-emission per still-hot key buys back visibility after the restarts that accompany incident response. Tests pin the edge-hold semantics (first fires, immediate re-check held, independent keys, cooldown expiry re-fires, zero cooldown always emits, hard bound) in one sequential test for the process-global map, and pin the emitted wire names against EventName's canonical string forms so a subscribed bucket notification can never silently stop matching. docs/operations/scanner-excess-alerts.md documents the three events, the metric-vs-event cadence difference, and the HS-15 threshold deltas (alert_excess_folders 65538 vs MinIO 50000 is deliberate: Proxmox Backup Server chunk layout compatibility). Closes rustfs/backlog#1868. Co-Authored-By: heihutu * docs(operations): split scanner excess alerts into English and Chinese pages The page shipped Chinese-only; keep it as scanner-excess-alerts_zh.md and add a faithful English translation at the original path, cross-linked at the top of both. Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- Cargo.lock | 1 + crates/ecstore/src/api/mod.rs | 2 +- crates/scanner/Cargo.toml | 3 + crates/scanner/src/scanner_folder.rs | 229 +++++++++++++++++++- crates/scanner/src/storage_api.rs | 7 +- docs/operations/scanner-excess-alerts.md | 37 ++++ docs/operations/scanner-excess-alerts_zh.md | 37 ++++ 7 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 docs/operations/scanner-excess-alerts.md create mode 100644 docs/operations/scanner-excess-alerts_zh.md diff --git a/Cargo.lock b/Cargo.lock index 85c91d523..cda581ffb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10252,6 +10252,7 @@ dependencies = [ "rustfs-ecstore", "rustfs-filemeta", "rustfs-lock", + "rustfs-s3-types", "rustfs-storage-api", "rustfs-utils", "s3s", diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 4b8bc3249..3a031d60a 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -380,7 +380,7 @@ pub mod erasure { pub mod event { pub use crate::event::name::EventName; - pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook}; + pub use crate::services::event_notification::{EventArgs, register_event_dispatch_hook, send_event}; } pub mod global { diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 16ff34a8e..0e2d039c6 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -108,6 +108,9 @@ temp-env = { workspace = true } tempfile = { workspace = true } uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] } tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] } +# Test-only: pins the emitted scanner alert wire names against the canonical +# EventName string forms subscribers configure (rustfs/backlog#1868). +rustfs-s3-types.workspace = true # Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via # the ecstore `api::tier::test_util` facade module (rustfs/backlog#1148 ilm-6). rustfs-ecstore = { workspace = true, features = ["test-util"] } diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 820cbb12f..15a01507c 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fs::FileType; use std::io::ErrorKind; -use std::sync::{Arc, Once}; +use std::sync::{Arc, Mutex, Once}; use std::time::{Duration, Instant, SystemTime}; use crate::ReplTargetSizeSummary; @@ -32,6 +32,7 @@ use crate::scanner_io::{ SCANNER_SKIP_FILE_ERROR, ScannerIODisk as _, is_scanner_metadata_corrupt_error, is_scanner_metadata_transient_error, }; use crate::sleeper::DynamicSleeper; +use crate::storage_api::owner::{EcstoreEventArgs, ecstore_send_event}; use metrics::{counter, describe_counter}; use rustfs_common::heal_channel::{ HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest, @@ -98,6 +99,101 @@ const METRIC_SCANNER_EXCESS_FOLDERS_TOTAL: &str = "rustfs_scanner_excess_folders const METRIC_SCANNER_PENDING_HEAL_PRUNE_TOTAL: &str = "rustfs_scanner_pending_heal_prune_total"; const METRIC_SCANNER_PENDING_HEAL_MALFORMED_TOTAL: &str = "rustfs_scanner_pending_heal_malformed_total"; const MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET: usize = 128; + +// --- scanner excess alerts as S3 notification events (rustfs/backlog#1868) -- +// +// The excess-versions / excess-version-size / excess-folders alerts were +// metrics-and-logs only; subscribers (consoles, external auditors) had no way +// to hear them. MinIO emits s3:ObjectManyVersions / s3:ObjectLargeVersions / +// s3:PrefixManyFolders for the same conditions — RustFS carries those as +// EventName::Scanner* with the wire names below. Without a cooldown a single +// over-threshold object would re-emit on every scan cycle (~a minute), so +// emissions are edge-held per (kind, bucket, object) for 24h. + +/// `s3:Scanner:ManyVersions` (MinIO `s3:ObjectManyVersions`). +pub const EVENT_SCANNER_MANY_VERSIONS: &str = "s3:Scanner:ManyVersions"; +/// `s3:Scanner:LargeVersions` (MinIO `s3:ObjectLargeVersions`). +pub const EVENT_SCANNER_LARGE_VERSIONS: &str = "s3:Scanner:LargeVersions"; +/// `s3:Scanner:BigPrefix` (MinIO `s3:PrefixManyFolders`). +pub const EVENT_SCANNER_BIG_PREFIX: &str = "s3:Scanner:BigPrefix"; +const ENV_SCANNER_ALERT_COOLDOWN_SECS: &str = "RUSTFS_SCANNER_ALERT_COOLDOWN_SECS"; +const DEFAULT_SCANNER_ALERT_COOLDOWN_SECS: u64 = 86_400; +/// Hard cap on distinct cooldown keys; a pathological number of over-threshold +/// objects clears the map wholesale instead of growing without bound (the +/// worst case is one re-emission per still-hot key per scan cycle). +const MAX_SCANNER_ALERT_COOLDOWN_KEYS: usize = 4096; + +/// Distinct alert kinds sharing one cooldown map. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum ScannerAlertKind { + ManyVersions, + LargeVersions, + BigPrefix, +} + +type ScannerAlertCooldownKey = (ScannerAlertKind, String, String); +type ScannerAlertCooldownMap = HashMap; + +static SCANNER_ALERT_EMISSION_COOLDOWN: Mutex> = Mutex::new(None); + +fn scanner_alert_cooldown() -> Duration { + let raw = std::env::var(ENV_SCANNER_ALERT_COOLDOWN_SECS) + .ok() + .and_then(|v| v.parse::().ok()); + Duration::from_secs(raw.unwrap_or(DEFAULT_SCANNER_ALERT_COOLDOWN_SECS)) +} + +/// Edge-held emission gate: returns `true` (and records the cooldown) only +/// when this (kind, bucket, object) last fired longer than the cooldown ago — +/// or never. Metrics and logs stay level-triggered every cycle; only the +/// notification events are held back. +fn scanner_alert_emission_allows(kind: ScannerAlertKind, bucket: &str, object: &str, cooldown: Duration) -> bool { + let key = (kind, bucket.to_string(), object.to_string()); + let mut guard = SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + let guard = guard.get_or_insert_with(ScannerAlertCooldownMap::new); + let now = Instant::now(); + // Expired entries leave first; the cap is still exceeded only when live + // keys alone overflow it, in which case a wholesale clear trades one + // extra emission per hot key for a hard memory bound. + if guard.len() >= MAX_SCANNER_ALERT_COOLDOWN_KEYS { + guard.retain(|_, fired_at| now.duration_since(*fired_at) < cooldown); + if guard.len() >= MAX_SCANNER_ALERT_COOLDOWN_KEYS { + guard.clear(); + } + } + match guard.get(&key) { + Some(fired_at) if now.duration_since(*fired_at) < cooldown => false, + _ => { + guard.insert(key, now); + true + } + } +} + +/// Emit a scanner alert as an S3 notification event through the standard +/// dispatch pipeline. Fire-and-forget: the notify layer owns delivery, +/// retry, and target filtering; the scanner never waits on it. +fn emit_scanner_alert_event(event_name: &str, bucket: &str, object: &str, size: i64, details: &[(&str, String)]) { + let mut req_params = HashMap::with_capacity(details.len()); + for (key, value) in details { + req_params.insert((*key).to_string(), value.clone()); + } + ecstore_send_event(EcstoreEventArgs { + event_name: event_name.to_string(), + bucket_name: bucket.to_string(), + object: crate::ScannerObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size, + ..Default::default() + }, + req_params, + user_agent: "Scanner".to_string(), + ..Default::default() + }); +} const MAX_PENDING_SCANNER_HEALS_PER_BUCKET: usize = 10_000; static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new(); @@ -1350,6 +1446,7 @@ impl ScannerItem { fn alert_excessive_versions(&self, remaining_versions: usize, cumulative_size: i64) { ensure_scanner_alert_metrics_registered(); let (too_many_versions, too_large_versions) = should_alert_excessive_versions(remaining_versions, cumulative_size); + let object_path = self.object_path(); if too_many_versions { global_metrics().record_scanner_source_executed(ScannerWorkSource::Alerts, 1); counter!( @@ -1357,13 +1454,26 @@ impl ScannerItem { "bucket" => self.bucket.clone() ) .increment(1); + if scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, &self.bucket, &object_path, scanner_alert_cooldown()) + { + emit_scanner_alert_event( + EVENT_SCANNER_MANY_VERSIONS, + &self.bucket, + &object_path, + cumulative_size, + &[ + ("versions", remaining_versions.to_string()), + ("threshold", scanner_excess_versions_threshold().to_string()), + ], + ); + } warn!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_ALERT_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_FOLDER, bucket = %self.bucket, - object = %self.object_path(), + object = %object_path, versions = remaining_versions, threshold = scanner_excess_versions_threshold(), state = "excess_versions", @@ -1377,13 +1487,31 @@ impl ScannerItem { "bucket" => self.bucket.clone() ) .increment(1); + if scanner_alert_emission_allows( + ScannerAlertKind::LargeVersions, + &self.bucket, + &object_path, + scanner_alert_cooldown(), + ) { + emit_scanner_alert_event( + EVENT_SCANNER_LARGE_VERSIONS, + &self.bucket, + &object_path, + cumulative_size, + &[ + ("versions", remaining_versions.to_string()), + ("cumulativeSize", cumulative_size.to_string()), + ("threshold", scanner_excess_version_size_threshold().to_string()), + ], + ); + } warn!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_ALERT_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_FOLDER, bucket = %self.bucket, - object = %self.object_path(), + object = %object_path, versions = remaining_versions, cumulative_size, threshold = scanner_excess_version_size_threshold(), @@ -1764,6 +1892,15 @@ impl FolderScanner { "root" => self.root.clone() ) .increment(1); + if scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, &self.root, folder, scanner_alert_cooldown()) { + emit_scanner_alert_event( + EVENT_SCANNER_BIG_PREFIX, + &self.root, + folder, + 0, + &[("folders", total_folders.to_string()), ("threshold", threshold.to_string())], + ); + } warn!( target: "rustfs::scanner::folder", event = EVENT_SCANNER_ALERT_STATE, @@ -3232,6 +3369,90 @@ mod tests { #[cfg(unix)] use std::os::unix::fs::{PermissionsExt, symlink}; use std::sync::Mutex; + + /// Reset the process-global alert cooldown map; test-only. + fn reset_alert_cooldowns() { + *SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Some(ScannerAlertCooldownMap::new()); + } + + /// The emitted event-name strings must be exactly what `EventName` + /// serializes, or a bucket notification subscribed to the documented name + /// would silently never match (rustfs/backlog#1868). + #[test] + fn scanner_alert_wire_names_match_canonical_event_names() { + use rustfs_s3_types::EventName; + assert_eq!(EVENT_SCANNER_MANY_VERSIONS, EventName::ScannerManyVersions.to_string()); + assert_eq!(EVENT_SCANNER_LARGE_VERSIONS, EventName::ScannerLargeVersions.to_string()); + assert_eq!(EVENT_SCANNER_BIG_PREFIX, EventName::ScannerBigPrefix.to_string()); + } + + fn cooldown_map_len() -> usize { + SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .as_ref() + .map(|map| map.len()) + .unwrap_or(0) + } + + /// Backdate every recorded cooldown so the next check fires again. + fn expire_all_alert_cooldowns(cooldown: Duration) { + let now = Instant::now(); + let mut guard = SCANNER_ALERT_EMISSION_COOLDOWN + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if let Some(map) = guard.as_mut() { + for fired_at in map.values_mut() { + if let Some(expired) = now.checked_sub(cooldown + Duration::from_secs(1)) { + *fired_at = expired; + } + } + } + } + + /// The emission gate is the only thing standing between an over-threshold + /// object and one S3 event per scan cycle, so its edge semantics get + /// pinned directly. All scenarios share one #[test] because the cooldown + /// map is process-global and parallel tests would read each other's + /// firings. + #[test] + fn scanner_alert_emission_is_edge_held_per_key_and_bounded() { + reset_alert_cooldowns(); + let cooldown = Duration::from_secs(3600); + + // First firing allows, an immediate re-check is held. + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown)); + assert!(!scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown)); + + // Different kind, object, and bucket are independent keys. + assert!(scanner_alert_emission_allows(ScannerAlertKind::LargeVersions, "bkt", "obj", cooldown)); + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "other", cooldown)); + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "other", "obj", cooldown)); + assert_eq!(cooldown_map_len(), 4); + + // After the cooldown elapses the same key fires again. + expire_all_alert_cooldowns(cooldown); + assert!(scanner_alert_emission_allows(ScannerAlertKind::ManyVersions, "bkt", "obj", cooldown)); + + // A zero cooldown degenerates to always-emit (operators may want that). + assert!(scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", "dir", Duration::ZERO)); + assert!(scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", "dir", Duration::ZERO)); + + // Hard bound: overflow the cap with zero-cooldown keys and confirm the + // map clears rather than growing past it. + reset_alert_cooldowns(); + for index in 0..=(MAX_SCANNER_ALERT_COOLDOWN_KEYS + 8) { + let _ = scanner_alert_emission_allows(ScannerAlertKind::BigPrefix, "bkt", &format!("dir-{index}"), Duration::ZERO); + } + assert!( + cooldown_map_len() <= MAX_SCANNER_ALERT_COOLDOWN_KEYS, + "cooldown map must stay bounded, got {}", + cooldown_map_len() + ); + } + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use temp_env::{with_var, with_var_unset}; use tracing_subscriber::fmt::MakeWriter; diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index e77033aa2..cbd7c6486 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -78,6 +78,7 @@ pub(crate) use rustfs_ecstore::api::disk::{ pub(crate) use rustfs_ecstore::api::error::{ Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError, }; +pub(crate) use rustfs_ecstore::api::event::{EventArgs as EcstoreEventArgs, send_event as ecstore_send_event}; #[cfg(test)] pub(crate) use rustfs_ecstore::api::layout::{ EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints, @@ -110,8 +111,8 @@ pub(crate) mod owner { ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS, ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions, - EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc, - EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, + EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreEventArgs, + EcstoreLcEventSrc, EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, EcstoreReplicationConfigurationExt, EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi, ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, @@ -121,7 +122,7 @@ pub(crate) mod owner { ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, - ecstore_save_config, scanner_replication_config_for_lifecycle_eval, + ecstore_save_config, ecstore_send_event, scanner_replication_config_for_lifecycle_eval, }; #[cfg(test)] diff --git a/docs/operations/scanner-excess-alerts.md b/docs/operations/scanner-excess-alerts.md new file mode 100644 index 000000000..8079daba6 --- /dev/null +++ b/docs/operations/scanner-excess-alerts.md @@ -0,0 +1,37 @@ +# Scanner Excess Alerts: Metrics, S3 Events, and Thresholds + +> 中文版：[scanner-excess-alerts_zh.md](scanner-excess-alerts_zh.md) + +Date: 2026-08-18 (rustfs/backlog#1868 / HS-04; includes the HS-15 threshold-delta notes) + +The background scanner detects three classes of "excess" conditions while it walks buckets and surfaces them as alerts. This page documents each alert's trigger condition, the subscribable S3 event, the cooldown semantics, and the threshold differences versus MinIO — for operators debugging alerts and for event consumers wiring up subscriptions. + +## The three alerts + +| Alert | Trigger (per scan cycle) | Metric | S3 event (RustFS wire name) | MinIO event name | +|---|---|---|---|---| +| Excess versions | Retained versions of one object ≥ `scanner:alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` | +| Excess version size | Cumulative bytes of all versions of one object ≥ `scanner:alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` | +| Excess folders | Direct subfolders of one directory > `scanner:alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` | + +Subscribe like any bucket notification: configure a notification on the target bucket with the RustFS wire name above (or the `s3:Scanner:*` wildcard). Events carry `UserAgent: Scanner` as their origin marker, and `req_params` holds the observed value and the threshold (`versions` / `cumulativeSize` / `folders` / `threshold`), so consumers can judge severity directly. + +## Metrics and events fire on different cadences + +- **Metrics and structured logs are level-triggered**: as long as the object stays over the threshold, every scan cycle counts and logs it (default cycle ≈ 60s; see `scanner:speed`). +- **S3 events are edge-triggered with a cooldown**: the same (alert kind, bucket, object) emits at most once per cooldown window — 24 hours by default (`RUSTFS_SCANNER_ALERT_COOLDOWN_SECS`; set it to 0 to emit every cycle). When the window lapses and the object is still over the threshold, the event fires again. The cooldown table lives in process memory with a 4096-entry hard cap; on overflow it is cleared and rebuilt (worst case: one extra emission per still-hot key). +- A process restart resets the cooldown (every still-over-threshold object emits once more after a restart) — deliberately: restarts usually accompany incident response, and the re-emission buys visibility. + +## Threshold defaults and the MinIO deltas (HS-15) + +| Config key | ENV | RustFS default | MinIO default | Notes | +|---|---|---|---|---| +| `scanner:alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 | 100 | Identical | +| `scanner:alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB | 1 TB | Same order of magnitude; different unit basis (TiB vs TB) | +| `scanner:alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 | 50000 | **Deliberate divergence**: 65538 tolerates the Proxmox Backup Server chunk layout (65536 chunks per directory plus the directory's own entries); MinIO's 50000 would fire continuously for PBS users. Set it to 50000 explicitly to match MinIO behavior | + +All three keys accept both env and admin config (`PUT /rustfs/admin/v3/config`, `scanner` subsystem); hot updates take effect immediately. + +## Why the event names are mapped + +RustFS's event enum (`rustfs_s3_types::EventName::ScannerManyVersions/LargeVersions/BigPrefix`) keeps the repo's established `s3:Scanner:*` wire names (literally different from MinIO's `s3:ObjectManyVersions`; the enum comments preserve the mapping). Subscribers should use the RustFS wire names in this page. If you need MinIO-literal compatibility, map the names on the console/consumer side — do not change the published wire names. diff --git a/docs/operations/scanner-excess-alerts_zh.md b/docs/operations/scanner-excess-alerts_zh.md new file mode 100644 index 000000000..d4f4995ac --- /dev/null +++ b/docs/operations/scanner-excess-alerts_zh.md @@ -0,0 +1,37 @@ +# Scanner 超限告警：指标、S3 事件与阈值 + +> English version: [scanner-excess-alerts.md](scanner-excess-alerts.md) + +日期：2026-08-18（rustfs/backlog#1868 / HS-04，含 HS-15 阈值差异说明） + +后台 scanner 在扫描过程中检测三类"超限"状态并对外告警。本文说明每类告警的触发条件、可订阅的 S3 事件、冷却语义，以及与 MinIO 的阈值差异，供运维排障与事件消费方对接。 + +## 三类告警 + +| 告警 | 触发条件（任一扫描周期） | 指标 | S3 事件（RustFS wire 名） | MinIO 对应事件名 | +|---|---|---|---|---| +| 版本数超限 | 单对象保留版本数 ≥ `scanner:alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` | +| 版本总大小超限 | 单对象全部版本累计字节 ≥ `scanner:alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` | +| 子目录数超限 | 单目录直接子目录数 > `scanner:alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` | + +订阅方式与普通桶通知一致：对目标桶配置 notification，事件名填上表 RustFS wire 名（或通配 `s3:Scanner:*`）。事件以 `UserAgent: Scanner` 标记来源，`req_params` 携带实际值与阈值（`versions` / `cumulativeSize` / `folders` / `threshold`），便于消费方直接判断严重程度。 + +## 指标与事件的触发节奏不同 + +- **指标与结构化日志是电平触发**：只要对象仍在阈值之上，每个扫描周期都会计数/打日志（默认周期约 60s，见 `scanner:speed`）。 +- **S3 事件是边沿触发 + 冷却**：同一 (告警类型， 桶, 对象) 在冷却窗口内只发一次，默认 24 小时（`RUSTFS_SCANNER_ALERT_COOLDOWN_SECS`，设 0 表示每周期都发）。窗口过后对象仍超限会再次发出。冷却表在进程内有 4096 条硬顶，超限清空重建（最坏情况是每个仍超限的 key 多发一次）。 +- 进程重启会重置冷却（重启后每个仍超限的对象会再发一次）——这是有意为之：重启常伴随排障，重发提供可见性。 + +## 阈值默认值与 MinIO 差异（HS-15） + +| 配置键 | ENV | RustFS 默认 | MinIO 默认 | 差异说明 | +|---|---|---|---|---| +| `scanner:alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 | 100 | 一致 | +| `scanner:alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB | 1 TB | 语义同量级，单位口径不同（TiB vs TB） | +| `scanner:alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 | 50000 | **有意差异**：65538 兼容 Proxmox Backup Server 的 chunk 布局（每目录 65536 个 chunk + 目录自身条目），按 MinIO 的 50000 会对 PBS 用户持续误报。如需与 MinIO 行为一致可显式配置为 50000 | + +三个键均支持 env 与 admin config（`PUT /rustfs/admin/v3/config` 的 `scanner` 子系统）双通道，热更新即时生效。 + +## 事件名映射的由来 + +RustFS 的事件枚举（`rustfs_s3_types::EventName::ScannerManyVersions/LargeVersions/BigPrefix`）沿用仓库既有 wire 名 `s3:Scanner:*`（与 MinIO 的 `s3:ObjectManyVersions` 字面不同，枚举注释中保留了映射关系）。订阅方应以本文的 RustFS wire 名为准；如需 MinIO 字面兼容，请在 console/消费侧做名称映射，不要修改已发布的 wire 名。 From 00de43528c67cf365a6e45f9f779aa46d16eebe6 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 09:55:29 +0800 Subject: [PATCH 10/22] fix(ecstore): describe peer bucket RPC failures with no details (#6190) heal_bucket, list_bucket, get_bucket_info and delete_bucket returned Error::other("") when a peer answered success=false without an error payload, so operators saw a bare "io error " after quorum reduction. Route all five bucket RPCs through peer_failure_without_details, which names the operation and bucket while staying identical across the peers of one operation so reduce_errs keeps grouping them into a single dominant error. --- .../ecstore/src/cluster/rpc/peer_s3_client.rs | 58 ++++++++++++++++--- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs index de7f0e595..02c9f75f1 100644 --- a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs @@ -214,6 +214,19 @@ fn pool_write_quorum(participant_count: usize) -> usize { (participant_count / 2) + 1 } +/// Error for a peer that reported `success = false` without an error payload. +/// +/// The message must stay identical across the peers of one operation: `reduce_errs` +/// buckets `Error::Io` by kind plus rendered message, so any per-peer detail (address, +/// timing) would split one shared failure into single-count buckets and downgrade a real +/// dominant error into `ErasureWriteQuorum`. +fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error { + match bucket { + Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")), + None => Error::other(format!("{op}: peer returned failure without error details")), + } +} + fn reduce_pool_write_quorum_errs(per_pool_errs: &[Option]) -> Option { if per_pool_errs.is_empty() { return Some(Error::ErasureWriteQuorum); @@ -1078,7 +1091,7 @@ impl PeerS3Client for RemotePeerS3Client { return if let Some(err) = response.error { Err(err.into()) } else { - Err(Error::other("")) + Err(peer_failure_without_details("heal_bucket", Some(bucket))) }; } @@ -1105,7 +1118,7 @@ impl PeerS3Client for RemotePeerS3Client { return if let Some(err) = response.error { Err(err.into()) } else { - Err(Error::other("")) + Err(peer_failure_without_details("list_bucket", None)) }; } let bucket_infos = response @@ -1136,9 +1149,7 @@ impl PeerS3Client for RemotePeerS3Client { return if let Some(err) = response.error { Err(err.into()) } else { - Err(Error::other(format!( - "make_bucket({bucket}): peer returned failure without error details" - ))) + Err(peer_failure_without_details("make_bucket", Some(bucket))) }; } @@ -1162,7 +1173,7 @@ impl PeerS3Client for RemotePeerS3Client { return if let Some(err) = response.error { Err(err.into()) } else { - Err(Error::other("")) + Err(peer_failure_without_details("get_bucket_info", Some(bucket))) }; } let bucket_info = serde_json::from_str::(&response.bucket_info)?; @@ -1190,7 +1201,7 @@ impl PeerS3Client for RemotePeerS3Client { return if let Some(err) = response.error { Err(err.into()) } else { - Err(Error::other("")) + Err(peer_failure_without_details("delete_bucket", Some(bucket))) }; } @@ -2314,4 +2325,37 @@ mod tests { .collect::>(); assert_eq!(calls, vec![1, 1, 0, 0, 0, 0, 0, 0]); } + + #[test] + fn peer_failure_without_details_names_operation_and_bucket() { + for op in ["heal_bucket", "make_bucket", "get_bucket_info", "delete_bucket"] { + let message = peer_failure_without_details(op, Some("ops-bucket")).to_string(); + assert!(message.contains(op), "{op} message must name the operation: {message}"); + assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}"); + } + + let message = peer_failure_without_details("list_bucket", None).to_string(); + assert!(message.contains("list_bucket"), "cluster-wide message must name the operation"); + assert!(!message.trim().is_empty()); + } + + #[test] + fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() { + // reduce_errs groups Io errors by kind plus rendered message: peers failing the + // same operation on the same bucket must still reach quorum as one dominant error. + let per_pool_errs = vec![ + Some(peer_failure_without_details("delete_bucket", Some("shared"))), + Some(peer_failure_without_details("delete_bucket", Some("shared"))), + Some(peer_failure_without_details("delete_bucket", Some("shared"))), + ]; + assert_eq!( + reduce_pool_write_quorum_errs(&per_pool_errs), + Some(peer_failure_without_details("delete_bucket", Some("shared"))) + ); + + assert_ne!( + peer_failure_without_details("delete_bucket", Some("shared")), + peer_failure_without_details("get_bucket_info", Some("shared")) + ); + } } From 9a2d06b370b4665116a93976377d658de65f1459 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 10:01:04 +0800 Subject: [PATCH 11/22] test(heal): lock heal vs delete/overwrite race invariants (HS-12) (#6183) * test(heal): add concurrency invariants for heal vs delete/overwrite races (HS-12) Audit conclusion for backlog#1874: RustFS does not need a persistent object-level healing marker (MinIO x-minio-healing) because every path that can touch the same (bucket, object) commit surface serializes on the same namespace write lock, and the heal lock guard spans the whole rename commit including the HEAL_RENAME_INCOMPLETE partial path. Lock the conclusion in with two race regression tests: - heal_racing_version_delete_never_resurrects_the_deleted_version: shard damage is injected on the doomed version so a Deep heal has real reconstruction work while a versioned DELETE runs concurrently; the deleted version must stay deleted and the survivor intact. - heal_racing_unversioned_overwrites_preserves_the_last_commit: unversioned overwrites (activating the post-commit tail that deletes the replaced data dir without the ns lock) race a Deep heal in a loop; the final current version must be exactly the last commit. Also adds docs/operations/heal-concurrency-safety-notes-zh.md with the full intersection matrix (17 intersections), lock-coverage argument, and the residual-window classification (commit tail races are fail-into-retry safe; bare prefix delete has zero production callers; admin no_lock is an explicit operator opt-in). Co-Authored-By: heihutu * test: remove redundant heal etag clone Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- crates/ecstore/src/set_disk/ops/heal.rs | 219 ++++++++++++++++++ .../heal-concurrency-safety-notes-zh.md | 115 +++++++++ 2 files changed, 334 insertions(+) create mode 100644 docs/operations/heal-concurrency-safety-notes-zh.md diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index de688fbdb..7c228595d 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -3297,4 +3297,223 @@ mod heal_result_report_tests { assert!(result.detail.contains("part 1")); assert!(result.detail.contains("bitrot_failure=true")); } + + // HS-12 (backlog#1874): a versioned DELETE racing an object heal must never + // resurrect the deleted version. The heal has real reconstruction work (a + // shard of the doomed version is removed), so both sides touch the same + // (bucket, object, data_dir); whichever order the ns write lock serializes + // them in, the committed delete must win. + #[tokio::test] + #[serial_test::serial] + async fn heal_racing_version_delete_never_resurrects_the_deleted_version() { + let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "heal-race-delete-no-resurrect"; + let object = "object.bin"; + set.make_bucket( + bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .expect("versioned bucket should be created"); + + let mut first_reader = PutObjReader::from_vec(vec![0x11; 1024 * 1024]); + let first_info = set + .put_object( + bucket, + object, + &mut first_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("first version should be written"); + let first_version = first_info + .version_id + .expect("versioned put should return the first version id") + .to_string(); + + let mut second_reader = PutObjReader::from_vec(vec![0x22; 1024 * 1024]); + let second_info = set + .put_object( + bucket, + object, + &mut second_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("second version should be written"); + let second_version = second_info + .version_id + .expect("versioned put should return the second version id") + .to_string(); + + // Damage one shard of the doomed version so the racing heal performs an + // actual reconstruction over its data dir instead of an early exit. + let doomed_source = disks[0] + .read_version("", bucket, object, &first_version, &ReadOptions::default()) + .await + .expect("doomed version metadata should be readable"); + let doomed_data_dir = doomed_source + .data_dir + .expect("non-inline version should have a data directory"); + tokio::fs::remove_file( + temp_dirs[1] + .path() + .join(bucket) + .join(object) + .join(doomed_data_dir.to_string()) + .join("part.1"), + ) + .await + .expect("shard damage should be injected before the race"); + + let delete_set = set.clone(); + let (delete_res, heal_res) = tokio::join!( + async { + delete_set + .delete_object( + bucket, + object, + ObjectOptions { + versioned: true, + version_id: Some(first_version.clone()), + object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new( + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await + }, + async { + set.heal_object( + bucket, + object, + "", + &HealOpts { + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + }, + ); + delete_res.expect("version delete must succeed under lock serialization"); + // The heal may legitimately report a transient failure when the version + // it was rebuilding disappears mid-flight; only the end state matters. + drop(heal_res); + + let resurrected = set + .get_object_info( + bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(first_version.clone()), + ..Default::default() + }, + ) + .await; + assert!( + matches!(&resurrected, Err(Error::FileVersionNotFound) | Err(Error::ObjectNotFound(..))), + "a racing heal must not resurrect the deleted version: {resurrected:?}" + ); + + let survivor = set + .get_object_info( + bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(second_version.clone()), + ..Default::default() + }, + ) + .await + .expect("surviving version must remain readable after the race"); + assert_eq!(survivor.size, 1024 * 1024, "survivor size must be intact"); + } + + // HS-12 (backlog#1874): unversioned overwrite commits race a Deep heal on + // the same object. The overwrite's post-commit tail deletes the replaced + // data dir without the ns lock (object.rs commit tail), which is exactly + // the intersection the audit flagged: the heal must tolerate the tail race + // (retryable outcome) and every committed overwrite must survive — the + // final current version is exactly the last payload written. + #[tokio::test] + #[serial_test::serial] + async fn heal_racing_unversioned_overwrites_preserves_the_last_commit() { + let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "heal-race-put-overwrite"; + let object = "object.bin"; + set.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + + const ROUNDS: usize = 8; + const PAYLOAD_SIZE: usize = 256 * 1024; + let mut last_etag = String::new(); + for round in 0..ROUNDS { + // Give the heal something to rebuild on alternating rounds: remove a + // shard of the current data dir right before the race. + if round % 2 == 1 { + let current = disks[2] + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("current metadata should be readable"); + if let Some(data_dir) = current.data_dir { + let shard = temp_dirs[3] + .path() + .join(bucket) + .join(object) + .join(data_dir.to_string()) + .join("part.1"); + if shard.exists() { + tokio::fs::remove_file(&shard) + .await + .expect("shard damage should be injectable mid-race"); + } + } + } + + let payload = vec![round as u8; PAYLOAD_SIZE]; + let mut put_reader = PutObjReader::from_vec(payload); + let put_opts = ObjectOptions::default(); + let heal_opts = HealOpts { + scan_mode: HealScanMode::Deep, + ..Default::default() + }; + let (put_res, heal_res) = tokio::join!( + set.put_object(bucket, object, &mut put_reader, &put_opts), + set.heal_object(bucket, object, "", &heal_opts), + ); + let put_info = put_res.expect("overwrite must succeed under lock serialization"); + last_etag = put_info.etag.clone().unwrap_or_default(); + // Heal outcome is unconstrained (may hit the tail race and report a + // retryable error); the invariant is checked on the end state. + drop(heal_res); + } + + let final_info = set + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("object must remain readable after the race loop"); + assert_eq!( + final_info.size, PAYLOAD_SIZE as i64, + "final current version must be the last committed overwrite" + ); + assert_eq!( + final_info.etag.unwrap_or_default(), + last_etag, + "the racing heal loop must never leave a stale or resurrected current version" + ); + } } diff --git a/docs/operations/heal-concurrency-safety-notes-zh.md b/docs/operations/heal-concurrency-safety-notes-zh.md new file mode 100644 index 000000000..ffb1e90c7 --- /dev/null +++ b/docs/operations/heal-concurrency-safety-notes-zh.md @@ -0,0 +1,115 @@ +# Heal 并发安全说明（对象级 healing 标记对标审计结论） + +对应 backlog rustfs/backlog#1874（父 #1862，HS-12）。本文回答一个问题：MinIO 在 heal +期间对对象打 `x-minio-healing:true` 元数据标记以防"heal 提交与并发删除/版本清理互毁" +（cmd/xl-storage.go RenameData 的 healing 分支），RustFS 是否需要同款防御。 + +**结论：不需要。** RustFS 不存在 MinIO 用 healing 标记防御的那类竞争：所有会触达同一 +`(bucket, object)` 提交面的路径都在同一把对象级 namespace 写锁上互斥，且 heal 的锁 +guard 覆盖 rename 提交全程；MinIO 需要标记的根因（RenameData 提交内部与版本清理逻辑 +交错）在 RustFS 的提交模型中不存在。RustFS 已有一个瞬态 healing 旗标用于另一目的 +（见下文 §2），并有并发不变量回归测试锁定本结论（§5）。 + +## 1. 两个防御模型的对照 + +MinIO：heal 时对对象写 `x-minio-healing:true`（持久元数据标记），后续任何 RenameData +提交看到该标记就跳过版本清理/legacy purge 逻辑——防御发生在锁外，靠元数据让路。 + +RustFS：三层防御，全部不依赖持久对象标记： + +1. **锁内互斥**：heal 与一切前台/后台写路径的提交点在同一把 `(bucket, object)` ns 写锁 + 上串行（分布式部署为 quorum 锁 RPC，单机为进程内锁管理器；锁粒度是对象级，version + 恒为 None）。 +2. **提交模型隔离**：rename_data 提交内没有会与 heal 交错的版本清理逻辑；被替换旧版本 + 的 data_dir 物理删除被移出提交临界区（commit tail），且只删已被新提交替换的 unshared + 目录。 +3. **瞬态 healing 旗标**：`FileInfo::set_healing`（crates/filemeta/src/fileinfo.rs）在 + heal 提交的内存 FileInfo 上打 `"healing"` 内部键，rename_data 据此允许先清空 stale + 目标 data_dir 再 rename——解决 heal 复用 data_dir 做 in-place 修复时 rename(2) 无法 + 替换非空目录的文件系统语义冲突（EEXIST/ENOTEMPTY）。该键是瞬态的，不落盘 + （`is_skip_meta_key`），与 MinIO 的持久标记目的不同。非 heal 提交撞上非空目标 + data_dir 会显式失败，有测试锁定两个方向的行为。 + +## 2. 交点矩阵 + +中心路径：`heal_object_with_explicit_version_regen`（crates/ecstore/src/set_disk/ops/heal.rs， +下称 heal.rs）在入口取 `(bucket, object)` ns 写锁，guard 绑定到函数作用域末尾，覆盖 +quorum 元数据读取 → EC 重建 → 逐盘 rename 提交 → tmp 清理 → HEAL_RENAME_INCOMPLETE +部分提交返回 → 孤儿 data_dir 回收的全过程。并发侧逐交点判定： + +| # | 并发路径 | 并发侧锁 | 判定 | 关键证据 | +|---|---|---|---|---| +| 1 | PUT 对象提交 | `put_object_commit` 对象写锁，rename_data 在锁内 | 同锁串行 | ops/object.rs 提交锁段 + rename 调用点 | +| 2 | PUT 旧 data_dir tail 清理 | drop 对象锁后的 `commit_rename_data_dir`，无锁 | 无锁并发，语义安全（见 §3.1） | object.rs drop 后 tail 段；io_primitives.rs | +| 3 | DELETE 单对象/版本 | `delete_object` 对象写锁，delete_version 在锁内 | 同锁串行 | object.rs delete_object 锁段 | +| 4 | DELETE 批量 | 批量逐对象写锁（dist 走批量锁 RPC） | 同锁串行 | object.rs delete_objects 锁段 | +| 5 | CompleteMultipart | 对象写锁 + upload 路径锁双锁，rename 在锁内 | 同锁串行 | ops/multipart.rs 提交锁段 | +| 6 | CompleteMultipart tail 清理 | drop 对象锁后的旧 data_dir 删除 | 无锁并发，语义安全（见 §3.1） | multipart.rs drop 后 tail 段 | +| 7 | AbortMultipart | 仅 multipart bucket 的 upload 路径锁 | 锁 key 不相交，但资源不相交（abort 不触对象 data_dir/xl.meta）→ 无实际交点 | multipart.rs abort 锁段 | +| 8 | ILM expiry（含 DeleteAllVersions） | DeleteAllVersions 走 `delete_prefix_object=true` → 仍取对象锁；FreeVersionTask 显式取锁；noncurrent 批量走批量锁 | 同锁串行 | bucket_lifecycle_ops.rs 消费端链路 | +| 9 | 纯 prefix 删除（绕锁能力面） | `delete_prefix`-only 不取子对象锁 | 无锁并发，但生产调用方为零（见 §3.2） | object.rs delete_object 锁条件 | +| 10 | 孤儿 data_dir 回收 reclaim_orphan_data_dirs | 函数本体无锁；唯一生产调用方在 heal 锁内 | heal 流程内=锁内串行 | heal.rs 收尾调用；io_primitives.rs | +| 11 | 旧清理 receipt 对账 reconcile_old_data_cleanup_receipts | 函数本体无锁；调用点在 heal 锁内 + epoch fence 防误删 | 锁内串行 | object.rs 对账函数 | +| 12 | replication | 数据面为远端 HTTP 写（不落本地盘）；本地元数据回写走对象锁 | 同锁串行 / 无交点 | replication_resyncer.rs 链路 | +| 13 | data_movement / rebalance / decommission 源清理 | 显式取对象锁 + 版本未变复核 + guard 复用（no_lock 只是复用已持锁） | 同锁串行 | data_movement/mod.rs 源清理 | +| 14 | copy_object | 目标对象锁 / 走 put 链锁 | 同锁串行 | object.rs copy_object 锁段 | +| 15 | 另一 heal 任务（跨 HealType/force_start） | dedup key 跨类型不相交 + force_start 跳过去重 → 任务级可并发 | 最终在 ns 写锁上串行 | heal/manager.rs dedup key 构成 | +| 16 | admin `no_lock=true` heal | 客户端可控绕锁 | 无锁并发，明示运维选项（见 §3.3） | admin/handlers/heal.rs 透传 | +| 17 | stale multipart 清理 | multipart bucket 的 upload 路径锁 | 资源不相交 → 无交点 | bucket_lifecycle_ops.rs 清理链路 | + +## 3. 残留窗口定性 + +### 3.1 PUT/CompleteMultipart commit tail（交点 2/6） + +写路径提交成功、释放对象锁之后，才 best-effort 删除被替换的旧 data_dir（注释明示有意 +不阻塞下一操作）。该删除与并发 heal 对同一旧 data_dir 的读取/重建存在竞态窗口，但语义 +安全： + +- 删除目标是已被新提交替换的 unshared data_dir；heal 的 canonical 元数据来自 quorum + 仲裁（ETag/mod_time），此时 quorum 已指向新版本，heal 不会把已替换版本当作 canonical + 复活； +- 竞态最坏后果 = heal 当轮对旧版本的一次 transient 失败/空转，重试轮自然收敛；清理 + residue 会上报并重新入队 heal（`report_old_data_dir_cleanup`）； +- 换盘重建等长 heal 走 per-version 显式版本请求，quorum 元数据在锁内读取，不受 tail + 影响。 + +### 3.2 纯 prefix 删除（交点 9） + +`delete_prefix && !delete_prefix_object` 的路径不取子对象锁（对象名空间锁无法保护前缀 +递归删除），与并发 heal 存在理论复活窗口（heal 在 prefix 删除进行中依据旧 quorum 元 +数据重建某版本）。全仓库核对结论：该路径的**生产调用方为零**——所有生产 `delete_prefix: +true` 调用点均同时设置 `delete_prefix_object: true`（从而取对象锁）或在测试模块内。这 +是 API 能力面的暴露而非行为风险。若未来有调用方需要纯 prefix 删除，须在调用点证明与 +heal/scanner 的隔离（例如 bucket 级停扫围栏）。 + +### 3.3 admin `no_lock=true`（交点 16） + +admin heal 请求可透传客户端 `nolock` 参数绕过 ns 锁（与 MinIO madmin 的同名选项对齐）。 +这是运维明示选项：使用即自负与并发写的竞争责任。文档化即可，不建议收紧。 + +## 4. heal 侧自身的不变量保障 + +- dedup key 跨 HealType 不相交（object/metadata/mrf/ecdecode/prefix 各自键面）+ admin + `force_start` 可跳过去重 → 同对象可能同时存在多个 heal 任务，但它们的执行体全部在 + `heal_object` 入口的 ns 写锁上串行（生产入口均 `no_lock=false`）； +- read-repair 的本地 TTL 预留只去重自身来源，不拦截其他来源的 heal——同样由 ns 锁兜底； +- healing 旗标不落盘，故不存在"标记残留导致后续提交错误让路"的反向风险。 + +## 5. 回归测试 + +以下两个并发不变量测试随本审计加入 `crates/ecstore/src/set_disk/ops/heal.rs` 测试模块： + +- `heal_racing_version_delete_never_resurrects_the_deleted_version`：注入 doomed 版本 + shard 损坏后，版本化 DELETE 与 Deep heal 真并发（同一把锁争用），断言已删除版本不被 + 复活、存活版本完好； +- `heal_racing_unversioned_overwrites_preserves_the_last_commit`：非版本化覆盖提交（激活 + commit tail 旧 data_dir 删除）与 Deep heal 循环竞态，断言最终 current 恰为最后一次 + 提交（etag 级一致）。 + +## 6. 结论 + +MinIO 的 `x-minio-healing` 是锁外元数据防御，前提是其 RenameData 提交内部存在与 heal +交错的版本清理逻辑；RustFS 的提交模型把这类交错从根上消除（提交面锁内互斥 + 清理外 +移到 tail + tail 只删 unshared 旧目录），因此引入持久对象级 healing 标记没有对应的竞争 +可防，反而会引入 FileInfo 落盘格式变更与标记残留清理两类新成本。维持现状，本对标疑点 +关闭。 From 84bd76a3ce788c3c515b721aed689f126f87e7a3 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 12:11:53 +0800 Subject: [PATCH 12/22] chore(deps): refresh cargo dependencies (#6198) Update workspace Cargo dependency requirements and lockfile after cargo update/upgrade, including rumqttc-next 0.34.0 and MQTT API compatibility adjustments. Verification: - cargo update --verbose - cargo upgrade --verbose - cargo update -p rumqttc-next --precise 0.34.0 --verbose - cargo tree --invert rumqttc-next --locked - cargo metadata --locked --no-deps --format-version 1 - cargo fmt --all --check - cargo check -p rustfs-targets --all-targets --locked - cargo test -p rustfs-targets mqtt --locked - make pre-pr Co-authored-by: heihutu --- Cargo.lock | 207 +++++++++++++++--------------- Cargo.toml | 16 +-- crates/targets/src/target/mqtt.rs | 29 +++-- 3 files changed, 131 insertions(+), 121 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cda581ffb..82ffafc81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -964,9 +964,9 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.114.0" +version = "1.115.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b7d906608ee41e7ddea9983577ba82200435644d567d63dc34e822e088b453" +checksum = "d5b034f8b7ceadb873d0bc607c30bb4b0be68e09a84c837174e7c2c6878ff882" dependencies = [ "arc-swap", "aws-credential-types", @@ -990,9 +990,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.141.0" +version = "1.142.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f9420d3a2467eed22ed3635ca653653162c386a0b0f65c78189f9bd3c1379e" +checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2" dependencies = [ "arc-swap", "aws-credential-types", @@ -1027,9 +1027,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.105.0" +version = "1.106.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516" +checksum = "2d0efcee834347b6705eca3eea2defd88242f43774f55d7326604222e3c86260" dependencies = [ "arc-swap", "aws-credential-types", @@ -1053,9 +1053,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.107.0" +version = "1.108.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c" +checksum = "a59312a04cf19c962cfee32b64ecfee758f8786407ff6da5b30fff46ae96f201" dependencies = [ "arc-swap", "aws-credential-types", @@ -1079,9 +1079,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.110.0" +version = "1.111.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13" +checksum = "120e7eb63457a9e547f9986fe3b273f77c43679da4d04f46359fa881c5e19b6e" dependencies = [ "arc-swap", "aws-credential-types", @@ -1598,7 +1598,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1617,7 +1617,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -1858,9 +1858,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" dependencies = [ "find-msvc-tools", "jobserver", @@ -1968,7 +1968,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.6", + "crypto-common 0.1.7", "inout 0.1.4", ] @@ -2428,7 +2428,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "rand_core 0.6.4", "subtle", "zeroize", @@ -2453,11 +2453,11 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "typenum", ] @@ -3664,7 +3664,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.6", + "crypto-common 0.1.7", "subtle", ] @@ -3924,7 +3924,7 @@ dependencies = [ "crypto-bigint 0.5.5", "digest 0.10.7", "ff 0.13.1", - "generic-array 0.14.9", + "generic-array 0.14.7", "group 0.13.0", "hkdf 0.12.4", "pem-rfc7468 0.7.0", @@ -4148,9 +4148,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "findshlibs" @@ -4369,9 +4369,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -4380,11 +4380,11 @@ dependencies = [ [[package]] name = "generic-array" -version = "1.4.4" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b" +checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e" dependencies = [ - "generic-array 0.14.9", + "generic-array 0.14.7", "rustversion", "typenum", ] @@ -4726,9 +4726,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -5028,9 +5028,9 @@ dependencies = [ [[package]] name = "hotpath" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e810bedda5a467ef5c9b5c8a20763fefebc89b63ef36f7ee44a143085204a2" +checksum = "dce755d457a63bdd0c95e4c91511daad1b58b33209543b7f38027b676f387e5e" dependencies = [ "arc-swap", "async-channel", @@ -5062,9 +5062,9 @@ dependencies = [ [[package]] name = "hotpath-macros" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01bdc59bfc1a9984bee2ff5da63b2f6fccbaa57cd9a4119d709524632bddf341" +checksum = "a903af89a8429cb07790c3818bc15270b394f80af1bc254e5ccf9c7de2961770" dependencies = [ "proc-macro2", "quote", @@ -5073,15 +5073,15 @@ dependencies = [ [[package]] name = "hotpath-macros-meta" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9216e8a01abe1e1671c376dc8736fb1bf772d7a889538d25f9e1200120ced38" +checksum = "bcc0ab94ffbb2ee77f4a897df02b5a137a10cf24d69bda936e59aff4dd456e61" [[package]] name = "hotpath-meta" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f22a9d20435fb79511b19dae37b3607224cd98f342a410702d84657cc38fc72f" +checksum = "053481f6cec8f775a3276c7f6e2f21123111d28261e4edc15ea7421c445964bb" dependencies = [ "hotpath-macros-meta", ] @@ -5280,9 +5280,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -5294,9 +5294,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -5307,9 +5307,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -5321,16 +5321,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -5341,15 +5342,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" dependencies = [ "displaydoc", "icu_locale_core", @@ -5417,7 +5418,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ "block-padding 0.3.3", - "generic-array 0.14.9", + "generic-array 0.14.7", ] [[package]] @@ -5968,9 +5969,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" +checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" dependencies = [ "libc", ] @@ -6033,9 +6034,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "local-ip-address" @@ -6482,9 +6483,9 @@ dependencies = [ [[package]] name = "mqttbytes-core-next" -version = "0.33.3" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff7ae19c74aba9e0ed6e4071cd52aa364e020076fa3cc6ef17e43662f756f3c" +checksum = "366b6ba2b4209ca4bc5ac731ccddf570d09831981eed07e5fbd63564cf0cf1aa" dependencies = [ "bytes", "thiserror 2.0.20", @@ -6885,7 +6886,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "chrono", "getrandom 0.2.17", "http 1.5.0", @@ -7344,9 +7345,9 @@ dependencies = [ [[package]] name = "pageant" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3a5ae18f65a85c67a77d18d42d3606c07948e3c17c1e5f74852b26589e88a5" +checksum = "3adadc44070da6f464b0918655a12f5792c156e088d8c4082d13e27d94c3e791" dependencies = [ "base16ct 1.0.0", "byteorder", @@ -7728,9 +7729,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "plotters" @@ -7836,9 +7837,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -8046,7 +8047,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -8066,7 +8067,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "petgraph 0.8.3", @@ -8087,7 +8088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -8100,7 +8101,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -8216,7 +8217,7 @@ dependencies = [ "reqwest", "serde_json", "smallvec", - "spin 0.12.2", + "spin 0.12.3", "symbolic-demangle", "tempfile", "thiserror 2.0.20", @@ -8285,9 +8286,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ "aws-lc-rs", "bytes", @@ -8553,9 +8554,9 @@ dependencies = [ [[package]] name = "redis" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3257df217f7eab0044627a268c9cc6cdb60c0c421c88f83ac41c4e31520b6b84" +checksum = "e37a4ca5c6ca42aa3e6df2fd32b987a65d32a4c2159a6f3fe0fd1df306a2658f" dependencies = [ "arc-swap", "arcstr", @@ -8567,7 +8568,7 @@ dependencies = [ "futures-channel", "futures-util", "itoa", - "num-bigint 0.4.8", + "num-bigint 0.5.1", "percent-encoding", "pin-project-lite", "rustls", @@ -8868,9 +8869,9 @@ dependencies = [ [[package]] name = "rumqttc-core-next" -version = "0.33.3" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7d9205738dd41a2546e82d27a634d07d8b303dcf7558565ff70caf3ceb0f9c" +checksum = "249896ab27ed630590971738264baa8f722f18965d2e387c706c40a3c2a572cc" dependencies = [ "async-tungstenite", "futures-io", @@ -8886,18 +8887,18 @@ dependencies = [ [[package]] name = "rumqttc-next" -version = "0.33.3" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed1bad2180ff539da671da9a996152a921bc5316eb6d8a9cc3bd441653138b08" +checksum = "477c9bbfba8f3aecc7aad31c6de2eacb75822efaa18e7aeecb8d3d8e534fbf07" dependencies = [ "rumqttc-v5-next", ] [[package]] name = "rumqttc-v5-next" -version = "0.33.3" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229576cbedfa9089f90c17c9454e9429ac1e89cdd223bac5cb39d837593f79bc" +checksum = "3dfa6ddcc7a7dd5688f9bf78d8f81cb94f367bce56c055d8d94cf81ecb0518bf" dependencies = [ "async-tungstenite", "bytes", @@ -8920,9 +8921,9 @@ dependencies = [ [[package]] name = "russh" -version = "0.62.6" +version = "0.62.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41043523e0edcbd4e31d00903e26f12994f63b21bae9904f7405c1ed92752a5" +checksum = "9decb68e4e44e1079700e54f17c8f23806ec53d7e0db73ab1c71d9dabc666812" dependencies = [ "aes 0.9.2", "aws-lc-rs", @@ -8945,7 +8946,7 @@ dependencies = [ "enum_dispatch", "flate2", "futures", - "generic-array 1.4.4", + "generic-array 1.4.5", "getrandom 0.4.3", "ghash", "hex-literal", @@ -9491,7 +9492,7 @@ dependencies = [ "parking_lot", "rayon", "smallvec", - "spin 0.12.2", + "spin 0.12.3", ] [[package]] @@ -10833,7 +10834,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct 0.2.0", "der 0.7.10", - "generic-array 0.14.9", + "generic-array 0.14.7", "pkcs8 0.10.2", "subtle", "zeroize", @@ -11397,9 +11398,9 @@ checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spin" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8abadc99fd9c7bbb7d0ca2b31d72a067d0c0dcd7aad25ab8cac71ba91417694b" +checksum = "0134f9043ed38b087ac4f7d4af44c79e2c9e5094421fe3164f435ce585953b10" dependencies = [ "lock_api", ] @@ -11811,7 +11812,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -11975,9 +11976,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -12651,9 +12652,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -13168,9 +13169,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "x509-cert" @@ -13350,9 +13351,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -13361,9 +13362,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" dependencies = [ "yoke", "zerofrom", @@ -13372,13 +13373,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6bbd3acd1..ae9b497e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -228,9 +228,9 @@ atoi = "3.1.0" atomic_enum = "0.3.0" aws-config = { version = "1.10.1" } aws-credential-types = { version = "1.3.0" } -aws-sdk-kms = { default-features = false, version = "1.114.0" } -aws-sdk-s3 = { default-features = false, version = "1.141.0" } -aws-sdk-sts = { default-features = false, version = "1.110.0" } +aws-sdk-kms = { default-features = false, version = "1.115.0" } +aws-sdk-s3 = { default-features = false, version = "1.142.0" } +aws-sdk-sts = { default-features = false, version = "1.111.0" } aws-smithy-http-client = { default-features = false, version = "1.3.0" } aws-smithy-runtime-api = { version = "1.14.0" } aws-smithy-types = { version = "1.6.2" } @@ -284,8 +284,8 @@ rayon = "1.12.0" reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "8.0.2" } reed-solomon-simd = "3.1.0" regex = { version = "1.13.1" } -rumqttc = { package = "rumqttc-next", version = "0.33.3" } -redis = { version = "1.5.0" } +rumqttc = { package = "rumqttc-next", version = "0.34.0" } +redis = { version = "1.6.0" } rustify = { version = "0.7", default-features = false } rustix = { version = "1.1.4" } rust-embed = { version = "8.12.0" } @@ -313,7 +313,7 @@ tracing-subscriber = { version = "0.3.23" } transform-stream = "0.3.1" url = "2.5.8" urlencoding = "2.1.3" -uuid = { version = "1.24.0" } +uuid = { version = "1.24.1" } vaultrs = { version = "0.8.0" } tar = "0.4.46" walkdir = "2.5.0" @@ -341,7 +341,7 @@ libunftp = { version = "0.23.0" } unftp-core = "0.1.0" suppaftp = { version = "10.0.1" } rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] } -russh = { version = "0.62.6" } +russh = { version = "0.62.7" } russh-sftp = "2.4.0" # WebDAV @@ -350,7 +350,7 @@ dav-server = "0.11.0" # Performance Analysis and Memory Profiling mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" } libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] } -hotpath = { version = "0.23.2", default-features = false } +hotpath = { version = "0.23.3", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.48" } diff --git a/crates/targets/src/target/mqtt.rs b/crates/targets/src/target/mqtt.rs index 3120d85b1..0490c3a91 100644 --- a/crates/targets/src/target/mqtt.rs +++ b/crates/targets/src/target/mqtt.rs @@ -32,8 +32,8 @@ use arc_swap::ArcSwap; use async_trait::async_trait; use hyper_rustls::ConfigBuilderExt; use rumqttc::{ - AsyncClient, Broker, ClientError, ConnectionError, EventLoop, Incoming, MqttOptions, Outgoing, PublishNoticeError, QoS, - Transport, mqttbytes::Error as MqttBytesError, + AsyncClient, Broker, ClientError, ConnectionError, EventLoop, Incoming, MqttOptions, Outgoing, ProtocolViolation, + PublishNoticeError, PublishOptions, QoS, Transport, mqttbytes::Error as MqttBytesError, }; use rustfs_config::{ EnableState, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_WS_PATH_ALLOWLIST, @@ -791,7 +791,7 @@ where .as_ref() .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?; let notice = client - .publish_tracked(&self.args.topic, self.args.qos, false, body) + .publish_tracked(&self.args.topic, body, PublishOptions::new(self.args.qos)) .await .map_err(|error| classify_mqtt_client_error(&error))?; drop(client_guard); @@ -1145,7 +1145,7 @@ async fn run_mqtt_event_loop(mut eventloop: EventLoop, connected_status: Arc { + rumqttc::Event::Incoming(Incoming::PingResp) => { trace!(target_id = %target_id, "Received PingResp from broker. Connection is alive."); } rumqttc::Event::Incoming(Incoming::SubAck(suback)) => { @@ -1257,7 +1257,11 @@ async fn run_mqtt_event_loop(mut eventloop: EventLoop, connected_status: Arc TargetError { match err { - ClientError::Request(_) | ClientError::TryRequest(_) | ClientError::TrackingUnavailable => TargetError::NotConnected, + ClientError::RequestChannelFull(_) | ClientError::RequestChannelDisconnected(_) | ClientError::TrackingUnavailable => { + TargetError::NotConnected + } + ClientError::InvalidRequest(_) => TargetError::Request(format!("Invalid MQTT publish request: {err}")), + _ => TargetError::NotConnected, } } @@ -1270,10 +1274,14 @@ fn classify_mqtt_notice_error(err: &PublishNoticeError) -> TargetError { PublishNoticeError::Recv | PublishNoticeError::SessionReset | PublishNoticeError::Qos0NotFlushed + | PublishNoticeError::BrokerOnlySessionResume + | PublishNoticeError::SessionPersistence(_) | PublishNoticeError::TopicAliasReplayUnavailable(_) => TargetError::NotConnected, + PublishNoticeError::RetainNotSupported => TargetError::Request(format!("MQTT broker rejected publish: {err}")), PublishNoticeError::V5PubAck(_) | PublishNoticeError::V5PubRec(_) | PublishNoticeError::V5PubComp(_) => { TargetError::Request(format!("MQTT broker rejected publish: {err}")) } + _ => TargetError::NotConnected, } } @@ -1299,12 +1307,13 @@ fn is_fatal_mqtt_error(err: &ConnectionError) -> bool { | MqttBytesError::MalformedPacket // Package format error | MqttBytesError::PayloadTooLong // Too long load | MqttBytesError::PayloadSizeLimitExceeded { .. } // Load size limit exceeded - | MqttBytesError::TopicNotUtf8 // Topic Non-UTF-8 (Serious Agreement Violation) + | MqttBytesError::TopicNotUtf8 { .. } // Topic Non-UTF-8 (Serious Agreement Violation) ) } // Others that are fatal StateError variants rumqttc::StateError::InvalidState // The internal state machine is in invalid state - | rumqttc::StateError::WrongPacket // Agreement Violation: Unexpected Data Packet Received + | rumqttc::StateError::ProtocolViolation(ProtocolViolation::UnexpectedIncomingPacket(_)) // Agreement Violation: Unexpected Data Packet Received + | rumqttc::StateError::ProtocolViolation(_) // Agreement Violation | rumqttc::StateError::Unsolicited(_) // Agreement Violation: Unsolicited ACK Received | rumqttc::StateError::CollisionTimeout // Agreement Violation (if this stage occurs) | rumqttc::StateError::EmptySubscription // Agreement violation (if this stage occurs) @@ -1727,8 +1736,8 @@ where mod tests { use super::{ AsyncClient, ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTarget, MQTTTlsConfig, - MqttOptions, PublishNoticeError, QoS, QueuedPayloadMeta, classify_mqtt_client_error, classify_mqtt_notice_error, - next_reconnect_backoff, reconnect_supervisor, validate_mqtt_broker_url, + MqttOptions, PublishNoticeError, PublishOptions, QoS, QueuedPayloadMeta, classify_mqtt_client_error, + classify_mqtt_notice_error, next_reconnect_backoff, reconnect_supervisor, validate_mqtt_broker_url, }; use crate::error::TargetError; use crate::target::{REDACTED_SECRET, TargetType}; @@ -1794,7 +1803,7 @@ mod tests { .capacity(1) .build(); client - .publish("fill", QoS::AtLeastOnce, false, b"fill".as_slice()) + .publish("fill", b"fill".as_slice(), PublishOptions::new(QoS::AtLeastOnce)) .await .expect("first publish should fill the local channel"); *target.client.lock().await = Some(client); From de9145e87ab90e5c8308b52fb68d72b3b3350fb8 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 12:15:19 +0800 Subject: [PATCH 13/22] feat(storage): add default-off PUT admission gate (#6197) Add an experimental fixed-count foreground PutObject admission gate for #1882 Phase 0 validation. The gate is default-off, returns SlowDown before body ingest when saturated, and keeps the admission permit with the spawned store commit owner until store PUT returns. Co-authored-by: heihutu --- crates/config/src/constants/object.rs | 25 ++++ rustfs/src/app/object_usecase.rs | 46 +++++-- rustfs/src/app/storage_api.rs | 2 +- rustfs/src/storage/concurrency/manager.rs | 153 +++++++++++++++++++++- rustfs/src/storage/concurrency/mod.rs | 2 +- rustfs/src/storage/storage_api.rs | 2 +- 6 files changed, 211 insertions(+), 19 deletions(-) diff --git a/crates/config/src/constants/object.rs b/crates/config/src/constants/object.rs index 081308754..874933aa6 100644 --- a/crates/config/src/constants/object.rs +++ b/crates/config/src/constants/object.rs @@ -234,6 +234,31 @@ pub const ENV_OBJECT_DISK_WRITE_ABSOLUTE_CAP: &str = "RUSTFS_OBJECT_DISK_WRITE_A /// Default absolute per-object erasure write cap in seconds (`0` = disabled). pub const DEFAULT_OBJECT_DISK_WRITE_ABSOLUTE_CAP: u64 = 0; +/// Enable foreground PutObject request admission. +/// +/// This is an experimental, default-off foreground write backpressure gate for +/// strict commit tail investigations. When disabled, PUTs follow the legacy +/// path and only the existing request counters are updated. +pub const ENV_PUT_FOREGROUND_ADMISSION_ENABLE: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_ENABLE"; +pub const DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE: bool = false; + +/// Maximum foreground PutObject requests admitted concurrently per process. +/// +/// The limit is used only when [`ENV_PUT_FOREGROUND_ADMISSION_ENABLE`] is true. +/// A value of `0` disables the gate even when the enable flag is present, so a +/// partially configured rollout cannot reject every PUT. +pub const ENV_PUT_FOREGROUND_ADMISSION_LIMIT: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_LIMIT"; +pub const DEFAULT_PUT_FOREGROUND_ADMISSION_LIMIT: usize = 0; + +/// Time in milliseconds a foreground PutObject waits for an admission permit. +/// +/// Once this timeout expires the request fails before body ingest/storage +/// mutation with S3 `SlowDown`/503. `0` means fail fast when the limit is full. +pub const ENV_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: &str = "RUSTFS_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS"; +pub const DEFAULT_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS: u64 = 0; + +const _: () = assert!(!DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE); + /// Environment variable for minimum GetObject timeout in seconds. /// /// When dynamic timeout calculation is enabled, this is the minimum timeout diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 643b4fb6e..1bb337c69 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -56,8 +56,8 @@ use super::storage_api::object_usecase::bucket::{ }; use super::storage_api::object_usecase::compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; use super::storage_api::object_usecase::concurrency::{ - self, ConcurrencyManager, DiskReadAdmission, GetObjectGuard, PutObjectGuard, get_concurrency_aware_buffer_size, - get_concurrency_manager, get_put_concurrency_aware_buffer_size, + self, ConcurrencyManager, DiskReadAdmission, GetObjectGuard, PutObjectAdmission, PutObjectGuard, + get_concurrency_aware_buffer_size, get_concurrency_manager, get_put_concurrency_aware_buffer_size, }; #[cfg(test)] use super::storage_api::object_usecase::contract::http::HTTPPreconditions; @@ -5681,6 +5681,35 @@ impl DefaultObjectUsecase { let server_side_encryption_requested = server_side_encryption.is_some() || sse_customer_algorithm.is_some() || ssekms_key_id.is_some(); + // Resolve the store through the request-bound server context + // (backlog#1052 S6), not the process-global handle, so an embedded + // second server never writes into the first server's store. + let Some(store) = self.object_store() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + let bucket_validate_stage_start = put_stage_metrics_enabled.then(Instant::now); + validate_bucket_exists(&store, &bucket).await?; + rustfs_io_metrics::record_put_object_stage_duration_from("app_bucket_validate", bucket_validate_stage_start); + + let put_admission = match get_concurrency_manager() + .admit_put_object() + .await + .map_err(|_| s3_error!(InternalError, "foreground write admission closed"))? + { + PutObjectAdmission::Disabled => None, + PutObjectAdmission::Admitted(permit) => { + counter!("rustfs.put_object.foreground_admission.total", "result" => "admitted").increment(1); + Some(permit) + } + PutObjectAdmission::Rejected => { + counter!("rustfs.put_object.foreground_admission.total", "result" => "rejected").increment(1); + return Err(s3_error!( + SlowDown, + "foreground write concurrency limit reached, please reduce your request rate" + )); + } + }; + let mut put_request_guard = PutObjectGuard::new(); let concurrent_put_requests = PutObjectGuard::concurrent_requests(); @@ -5733,16 +5762,6 @@ impl DefaultObjectUsecase { use_large_put_concurrency_tuning, ); - // Resolve the store through the request-bound server context - // (backlog#1052 S6), not the process-global handle, so an embedded - // second server never writes into the first server's store. - let Some(store) = self.object_store() else { - return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); - }; - let bucket_validate_stage_start = put_stage_metrics_enabled.then(Instant::now); - validate_bucket_exists(&store, &bucket).await?; - rustfs_io_metrics::record_put_object_stage_duration_from("app_bucket_validate", bucket_validate_stage_start); - let sse_config_stage_start = put_stage_metrics_enabled.then(Instant::now); let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); rustfs_io_metrics::record_put_object_stage_duration_from("app_sse_config_lookup", sse_config_stage_start); @@ -6132,7 +6151,9 @@ impl DefaultObjectUsecase { let cache_adapter = cache_adapter.clone(); let request_id = request_id.clone(); let put_path = put_path.to_string(); + let put_admission = put_admission; async move { + let _put_admission = put_admission; let object_traffic_progress = object_traffic_health .as_deref() .and_then(ObjectTrafficHealth::track_write_storage); @@ -6183,6 +6204,7 @@ impl DefaultObjectUsecase { } }; rustfs_io_metrics::record_put_object_stage_duration_from("app_store_put", store_put_stage_start); + drop(_put_admission); drop(object_traffic_progress); #[cfg(test)] wait_for_put_post_store_test_hook(&bucket).await; diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 839b8a38f..c6f03f240 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -936,7 +936,7 @@ pub(crate) mod bucket { pub(crate) mod concurrency { pub(crate) use crate::storage::storage_api::concurrency_consumer::{ - ConcurrencyManager, DiskReadAdmission, GetObjectGuard, IoQueueStatus, IoStrategy, PutObjectGuard, + ConcurrencyManager, DiskReadAdmission, GetObjectGuard, IoQueueStatus, IoStrategy, PutObjectAdmission, PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, get_put_concurrency_aware_buffer_size, }; } diff --git a/rustfs/src/storage/concurrency/manager.rs b/rustfs/src/storage/concurrency/manager.rs index 13b509f93..5bc456148 100644 --- a/rustfs/src/storage/concurrency/manager.rs +++ b/rustfs/src/storage/concurrency/manager.rs @@ -65,6 +65,11 @@ pub struct ConcurrencyManager { bandwidth_monitor: Arc>, /// Metrics collector for I/O latency tracking (P50, P95, P99) metrics_collector: Arc, + /// Experimental fixed-count foreground PutObject admission gate. + put_admission_semaphore: Arc, + put_admission_enabled: bool, + put_admission_limit: usize, + put_admission_wait_timeout: Duration, } impl std::fmt::Debug for ConcurrencyManager { @@ -114,6 +119,18 @@ pub enum DiskReadAdmission { Rejected, } +/// Outcome of foreground PutObject request admission. +#[derive(Debug)] +pub enum PutObjectAdmission { + /// Foreground PUT admission is disabled; proceed on the legacy path. + Disabled, + /// Request is admitted and must hold the permit until the store write + /// returns or the request fails before mutation. + Admitted(tokio::sync::OwnedSemaphorePermit), + /// The fixed-count gate stayed full until the configured wait timeout. + Rejected, +} + #[allow(dead_code)] impl ConcurrencyManager { /// Create a new concurrency manager with default settings @@ -161,6 +178,18 @@ impl ConcurrencyManager { // Initialize metrics collector for I/O latency tracking // Keep 1000 samples for P95/P99 calculation let metrics_collector = Arc::new(MetricsCollector::new(performance_metrics, 1000)); + let put_admission_enabled = rustfs_utils::get_env_bool( + rustfs_config::ENV_PUT_FOREGROUND_ADMISSION_ENABLE, + rustfs_config::DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE, + ); + let put_admission_limit = rustfs_utils::get_env_usize( + rustfs_config::ENV_PUT_FOREGROUND_ADMISSION_LIMIT, + rustfs_config::DEFAULT_PUT_FOREGROUND_ADMISSION_LIMIT, + ); + let put_admission_wait_timeout = Duration::from_millis(rustfs_utils::get_env_u64( + rustfs_config::ENV_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS, + rustfs_config::DEFAULT_PUT_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS, + )); // Build queue config directly from scheduler config. let queue_config = IoPriorityQueueConfig::from_scheduler_config(&scheduler_config); @@ -176,6 +205,10 @@ impl ConcurrencyManager { pattern_detector, bandwidth_monitor, metrics_collector, + put_admission_semaphore: Arc::new(Semaphore::new(if put_admission_enabled { put_admission_limit } else { 0 })), + put_admission_enabled, + put_admission_limit, + put_admission_wait_timeout, } } @@ -199,6 +232,16 @@ impl ConcurrencyManager { self.degraded_read_semaphore.close(); } + #[cfg(test)] + pub(crate) fn with_put_admission_for_test(enabled: bool, limit: usize, wait_timeout: Duration) -> Self { + let mut manager = Self::new(); + manager.put_admission_semaphore = Arc::new(Semaphore::new(if enabled { limit } else { 0 })); + manager.put_admission_enabled = enabled; + manager.put_admission_limit = limit; + manager.put_admission_wait_timeout = wait_timeout; + manager + } + /// Track a GetObject request pub fn track_request() -> GetObjectGuard { GetObjectGuard::new() @@ -284,6 +327,32 @@ impl ConcurrencyManager { } } + /// Admit a foreground PutObject request under the experimental fixed-count gate. + /// + /// The default-off path returns [`PutObjectAdmission::Disabled`] without + /// touching the semaphore, preserving legacy behavior. When enabled, the + /// permit must be acquired before body ingest and held until the store write + /// returns, so saturated foreground writes can fail with `SlowDown` before + /// creating visible side effects. + pub async fn admit_put_object(&self) -> Result { + if !self.put_admission_enabled || self.put_admission_limit == 0 { + return Ok(PutObjectAdmission::Disabled); + } + + if self.put_admission_wait_timeout.is_zero() { + return Ok(match self.put_admission_semaphore.clone().try_acquire_owned() { + Ok(permit) => PutObjectAdmission::Admitted(permit), + Err(tokio::sync::TryAcquireError::NoPermits) => PutObjectAdmission::Rejected, + Err(tokio::sync::TryAcquireError::Closed) => PutObjectAdmission::Rejected, + }); + } + + match tokio::time::timeout(self.put_admission_wait_timeout, self.put_admission_semaphore.clone().acquire_owned()).await { + Ok(permit) => Ok(PutObjectAdmission::Admitted(permit?)), + Err(_) => Ok(PutObjectAdmission::Rejected), + } + } + // ============================================ // Adaptive I/O Strategy Methods // ============================================ @@ -692,8 +761,16 @@ impl ConcurrencyManager { /// Get a read-only workload admission snapshot for foreground writes. pub fn put_object_admission_snapshot(&self) -> WorkloadAdmissionSnapshot { - let active = PutObjectGuard::concurrent_count(); - let limit = self.scheduler_config.max_concurrent_reads; + let (active, limit, hard_gate_enabled) = if self.put_admission_enabled && self.put_admission_limit > 0 { + ( + self.put_admission_limit + .saturating_sub(self.put_admission_semaphore.available_permits()), + self.put_admission_limit, + true, + ) + } else { + (PutObjectGuard::concurrent_count(), self.scheduler_config.max_concurrent_reads, false) + }; let state = if limit == 0 { AdmissionState::Disabled } else if active >= limit { @@ -706,7 +783,10 @@ impl ConcurrencyManager { WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, state).with_counts(Some(active), None, Some(limit)); match state { - AdmissionState::Disabled => admission.with_reason("foreground write pressure tracking disabled"), + AdmissionState::Disabled => admission.with_reason("foreground write admission disabled"), + AdmissionState::Saturated if hard_gate_enabled => { + admission.with_reason("foreground write admission permits exhausted") + } AdmissionState::Saturated => admission.with_reason("foreground write concurrency reached local pressure limit"), _ => admission, } @@ -783,7 +863,7 @@ impl Default for ConcurrencyManager { mod integration_tests { use super::super::io_schedule::{IoLoadLevel, IoPriority}; use super::super::request_guard::GetObjectGuard; - use super::ConcurrencyManager; + use super::{ConcurrencyManager, PutObjectAdmission}; use crate::storage::storage_api::concurrency_consumer::PutObjectGuard; use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshotProvider, WorkloadClass}; use rustfs_io_core::io_profile::{AccessPattern, StorageMedia}; @@ -880,6 +960,71 @@ mod integration_tests { crate::storage::concurrency::reset_active_put_requests(); } + #[tokio::test] + #[serial] + async fn test_concurrency_manager_put_admission_disabled_does_not_touch_gate() { + let manager = ConcurrencyManager::with_put_admission_for_test(false, 1, Duration::ZERO); + + let admission = manager + .admit_put_object() + .await + .expect("disabled put admission must not close"); + + assert!(matches!(admission, PutObjectAdmission::Disabled)); + assert_eq!(manager.put_admission_semaphore.available_permits(), 0); + assert_eq!(manager.put_object_admission_snapshot().state, AdmissionState::Open); + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_put_admission_rejects_when_limit_full() { + let manager = ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO); + + let first = manager.admit_put_object().await.expect("first put admission should acquire"); + assert!(matches!(first, PutObjectAdmission::Admitted(_))); + assert_eq!(manager.put_object_admission_snapshot().state, AdmissionState::Saturated); + + let second = manager + .admit_put_object() + .await + .expect("full put admission gate should reject, not close"); + assert!(matches!(second, PutObjectAdmission::Rejected)); + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_put_admission_reuses_released_permit() { + let manager = ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO); + + let first = manager.admit_put_object().await.expect("first put admission should acquire"); + drop(first); + + let second = manager + .admit_put_object() + .await + .expect("released put admission permit should be reusable"); + assert!(matches!(second, PutObjectAdmission::Admitted(_))); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn test_concurrency_manager_put_admission_wait_timeout_rejects() { + let manager = ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::from_secs(5)); + let held = manager.admit_put_object().await.expect("first put admission should acquire"); + let waiter_manager = manager.clone(); + + let waiter = tokio::spawn(async move { waiter_manager.admit_put_object().await }); + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(5)).await; + + let admission = waiter + .await + .expect("put admission waiter task must not panic") + .expect("put admission gate must stay open"); + assert!(matches!(admission, PutObjectAdmission::Rejected)); + drop(held); + } + #[tokio::test] #[serial] async fn test_concurrency_manager_workload_admission_registry_covers_required_classes() { diff --git a/rustfs/src/storage/concurrency/mod.rs b/rustfs/src/storage/concurrency/mod.rs index 110e14cf6..9b14eebda 100644 --- a/rustfs/src/storage/concurrency/mod.rs +++ b/rustfs/src/storage/concurrency/mod.rs @@ -54,7 +54,7 @@ pub use io_schedule::{ pub use request_guard::{GetObjectGuard, PutObjectGuard}; // Concurrency manager -pub use manager::{ConcurrencyManager, DiskReadAdmission}; +pub use manager::{ConcurrencyManager, DiskReadAdmission, PutObjectAdmission}; // ============================================ // New Module Re-exports (for gradual migration) diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index d1e7c0f3a..be9548619 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -117,7 +117,7 @@ pub(crate) mod access_consumer { pub(crate) mod concurrency_consumer { pub(crate) use super::super::concurrency::{ - ConcurrencyManager, DiskReadAdmission, GetObjectGuard, IoQueueStatus, IoStrategy, PutObjectGuard, + ConcurrencyManager, DiskReadAdmission, GetObjectGuard, IoQueueStatus, IoStrategy, PutObjectAdmission, PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, get_put_concurrency_aware_buffer_size, }; } From b825c548505d8a074c1fa52eea72019d39258dad Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 12:21:49 +0800 Subject: [PATCH 14/22] refactor(admin): route kms management auth through shared gate (#6194) --- rustfs/src/admin/handlers/kms_management.rs | 127 ++++++++++++-------- 1 file changed, 75 insertions(+), 52 deletions(-) diff --git a/rustfs/src/admin/handlers/kms_management.rs b/rustfs/src/admin/handlers/kms_management.rs index b9ed1b4c1..796fc2319 100644 --- a/rustfs/src/admin/handlers/kms_management.rs +++ b/rustfs/src/admin/handlers/kms_management.rs @@ -16,13 +16,12 @@ use super::kms_dynamic::current_kms_config_fingerprint; use super::kms_keys::{CreateKeyHandler, DescribeKeyHandler, GenerateDataKeyHandler, ListKeysHandler}; -use crate::admin::auth::validate_admin_request; +use crate::admin::auth::authorize_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{ current_kms_runtime_service_manager, current_notification_system, current_or_init_kms_runtime_service_manager, }; -use crate::auth::{check_key_valid, get_session_token}; -use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use crate::server::ADMIN_PREFIX; use hyper::{HeaderMap, Method, StatusCode}; use matchit::Params; use rustfs_kms::KmsBackend; @@ -69,6 +68,18 @@ fn kms_clear_cache_actions() -> Vec { vec![Action::KmsAction(KmsAction::ClearCacheAction)] } +/// Admin gate for the KMS management endpoints, none of which act on a key. +/// +/// The pre-check keeps these endpoints' historical missing-credentials message; +/// the shared gate reports "get cred failed". +async fn authorize_kms_management_request(req: &S3Request, actions: Vec) -> S3Result<()> { + if req.credentials.is_none() { + return Err(s3_error!(InvalidRequest, "authentication required")); + } + authorize_admin_request(req, actions).await?; + Ok(()) +} + /// Response of `POST /kms/clear-cache`. /// /// Declared rather than built inline so the shape the console already depends @@ -260,22 +271,7 @@ pub struct KmsStatusHandler {} #[async_trait::async_trait] impl Operation for KmsStatusHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - let Some(cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "authentication required")); - }; - - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - kms_service_control_actions(), - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await?; + authorize_kms_management_request(&req, kms_service_control_actions()).await?; let Some(service) = kms_encryption_service_from_context().await else { return Err(s3_error!(InternalError, "KMS service not initialized")); @@ -326,22 +322,7 @@ pub struct KmsConfigHandler {} #[async_trait::async_trait] impl Operation for KmsConfigHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - let Some(cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "authentication required")); - }; - - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - kms_configure_actions(), - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await?; + authorize_kms_management_request(&req, kms_configure_actions()).await?; let Some(service) = kms_encryption_service_from_context().await else { return Err(s3_error!(InternalError, "KMS service not initialized")); @@ -375,22 +356,7 @@ pub struct KmsClearCacheHandler {} #[async_trait::async_trait] impl Operation for KmsClearCacheHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { - let Some(cred) = req.credentials else { - return Err(s3_error!(InvalidRequest, "authentication required")); - }; - - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - kms_clear_cache_actions(), - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await?; + authorize_kms_management_request(&req, kms_clear_cache_actions()).await?; let Some(service) = kms_encryption_service_from_context().await else { return Err(s3_error!(InternalError, "KMS service not initialized")); @@ -422,9 +388,14 @@ impl Operation for KmsClearCacheHandler { #[cfg(test)] mod tests { - use super::{KmsClearCacheResponse, kms_clear_cache_actions, kms_configure_actions, kms_service_control_actions}; + use super::{ + KmsClearCacheResponse, authorize_kms_management_request, kms_clear_cache_actions, kms_configure_actions, + kms_service_control_actions, + }; use crate::admin::handlers::kms_keys::stable_json_value; + use hyper::HeaderMap; use rustfs_policy::policy::action::{Action, AdminAction, KmsAction}; + use s3s::{Body, S3Request}; fn assert_has_action(actions: &[Action], action: Action) { assert!(actions.contains(&action), "expected action list to contain {action:?}"); @@ -434,6 +405,58 @@ mod tests { assert!(!actions.contains(&action), "expected action list not to contain {action:?}"); } + /// These endpoints authorize through the shared admin gate, which reports + /// "get cred failed" for a credential-less request. The pre-check keeps the + /// message these endpoints have always returned (rustfs/backlog#1829). + #[tokio::test] + async fn kms_management_gate_keeps_its_missing_credentials_message() { + let req = S3Request { + input: Body::from(String::new()), + method: http::Method::GET, + uri: "/rustfs/admin/v3/kms/status".parse().expect("uri should parse"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + let err = authorize_kms_management_request(&req, kms_service_control_actions()) + .await + .expect_err("a request without credentials must be rejected"); + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("authentication required")); + } + + /// Every management endpoint must reach the shared gate, each with its own + /// action set. The action lists are pinned above, but nothing else checks + /// which handler asks for which, and a handler that lost its gate entirely + /// would still serve its response. + #[test] + fn management_handlers_authorize_with_their_dedicated_actions() { + let src = include_str!("kms_management.rs"); + + for (handler, actions) in [ + ("KmsStatusHandler", "kms_service_control_actions()"), + ("KmsConfigHandler", "kms_configure_actions()"), + ("KmsClearCacheHandler", "kms_clear_cache_actions()"), + ] { + let block = src + .split_once(&format!("impl Operation for {handler}")) + .unwrap_or_else(|| panic!("{handler} impl should exist")) + .1; + let end = block + .find("\nimpl Operation for") + .or_else(|| block.find("\n#[cfg(test)]")) + .unwrap_or(block.len()); + assert!( + block[..end].contains(&format!("authorize_kms_management_request(&req, {actions})")), + "{handler} must authorize through the shared gate with {actions}" + ); + } + } + #[test] fn kms_management_auth_actions_use_dedicated_kms_actions() { assert_has_action(&kms_service_control_actions(), Action::KmsAction(KmsAction::ServiceControlAction)); From abffa5cf1b2d1e51f48aa6f59cec5b6e77ef9413 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 12:35:36 +0800 Subject: [PATCH 15/22] chore(storage): drop dead io-schedule metrics and helpers (#6199) --- docs/architecture/global-state-inventory.md | 2 +- rustfs/src/storage/concurrency/io_schedule.rs | 212 +----------------- rustfs/src/storage/concurrency/mod.rs | 70 +----- 3 files changed, 19 insertions(+), 265 deletions(-) diff --git a/docs/architecture/global-state-inventory.md b/docs/architecture/global-state-inventory.md index 6c06b10c8..8407ad864 100644 --- a/docs/architecture/global-state-inventory.md +++ b/docs/architecture/global-state-inventory.md @@ -107,7 +107,7 @@ inventory. Generic function-local names such as `CACHE`, `LOCK`, `INIT`, and | `AUTH_FS` | `rustfs/src/storage/access.rs` | Cache or constant / owner-local cache | Authorization tag-condition lookup keeps its filesystem helper private to the access owner. | | `LOCK_STATS` | `rustfs/src/storage/lock_optimizer.rs` | Process-global owner-local metrics | Lock optimization statistics stay private behind lock optimizer helper APIs. | | `DEADLOCK_DETECTOR` | `rustfs/src/storage/deadlock_detector.rs` | Process-global owner-local state | Deadlock detector lifecycle state stays private to the storage deadlock detector owner. | -| `CONCURRENCY_MANAGER`, `ACTIVE_GET_REQUESTS`, `ACTIVE_PUT_REQUESTS`, `IO_PRIORITY_METRICS` | `rustfs/src/storage/concurrency/*` | Process-global owner-local scheduler state | Storage concurrency manager, counters, and metrics remain inside the storage concurrency owner boundary. | +| `CONCURRENCY_MANAGER`, `ACTIVE_GET_REQUESTS`, `ACTIVE_PUT_REQUESTS` | `rustfs/src/storage/concurrency/*` | Process-global owner-local scheduler state | Storage concurrency manager and request counters remain inside the storage concurrency owner boundary. | | `GET_OBJECT_BUFFER_THRESHOLD_WARNED`, `GET_READER_STREAM_BUFFER_SIZE_OVERRIDE`, function-local `ENABLED`, `OBJECT_SEEK_SUPPORT_THRESHOLD`, `OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS` | `rustfs/src/app/object_usecase.rs` | Cache or constant / owner-local cache | Object GET/seek tuning caches and warning guards stay private to object usecase helpers. | | `SUPPORTED_HEADERS` | `rustfs/src/storage/options.rs` | Cache or constant / owner-local constant | Supported-header lookup state stays private to storage option parsing. | | `AUDIT_TARGET_SPECS`, `NOTIFICATION_TARGET_SPECS` | `rustfs/src/admin/handlers/audit.rs`, `rustfs/src/admin/handlers/event.rs`, `rustfs/src/admin/handlers/plugins_instances.rs` | Cache or constant / owner-local constant | Admin target descriptor tables stay private to their handler owners. | diff --git a/rustfs/src/storage/concurrency/io_schedule.rs b/rustfs/src/storage/concurrency/io_schedule.rs index d611dfeb2..8d202708a 100644 --- a/rustfs/src/storage/concurrency/io_schedule.rs +++ b/rustfs/src/storage/concurrency/io_schedule.rs @@ -14,21 +14,12 @@ //! I/O scheduling types for adaptive buffer sizing and load management. //! -//! # Migration Note -//! -//! This module contains types that are also available in `rustfs_io_core`. -//! For new code, prefer using types from `rustfs_io_core` directly: -//! -//! ```ignore -//! // Recommended: Use io-core types -//! use rustfs_io_core::{ -//! IoLoadLevel, IoPriority, IoSchedulerConfig, -//! calculate_optimal_buffer_size, get_buffer_size_for_media, -//! }; -//! ``` -//! -//! This module remains for backward compatibility and provides additional -//! runtime monitoring features (`IoPriorityMetrics`, `IoStrategyDebugInfo`). +//! This is the live scheduling implementation. `rustfs_io_core` supplies the +//! shared config shapes (`IoSchedulerConfig`, `IoPriorityQueueConfig`) that the +//! types here project into through `to_core_config`, plus the `io_profile` +//! storage-media model; bandwidth samples come from `rustfs_io_metrics`. +//! Same-named io-core types are those config shapes, not a backing +//! implementation this module delegates to. use rustfs_config::{KI_B, MI_B}; use rustfs_io_core::io_profile::{AccessPattern, StorageMedia, StorageProfile}; @@ -1762,169 +1753,6 @@ impl IoPriorityQueue { } } -// ============================================ -// I/O Priority Queue Metrics -// ============================================ - -/// Global metrics for I/O priority queue monitoring. -/// -/// These metrics are emitted through the shared metrics pipeline and provide -/// visibility into the priority queue behavior. -#[allow(dead_code)] -pub struct IoPriorityMetrics { - /// High priority queue depth. - pub high_queue_depth: AtomicU64, - /// Normal priority queue depth. - pub normal_queue_depth: AtomicU64, - /// Low priority queue depth. - pub low_queue_depth: AtomicU64, - /// High priority total wait time in nanoseconds. - pub high_wait_time_ns: AtomicU64, - /// Normal priority total wait time in nanoseconds. - pub normal_wait_time_ns: AtomicU64, - /// Low priority total wait time in nanoseconds. - pub low_wait_time_ns: AtomicU64, - /// Total starvation events count. - pub starvation_events: AtomicU64, - /// High priority requests processed. - pub high_processed: AtomicU64, - /// Normal priority requests processed. - pub normal_processed: AtomicU64, - /// Low priority requests processed. - pub low_processed: AtomicU64, -} - -#[allow(dead_code)] -impl Default for IoPriorityMetrics { - fn default() -> Self { - Self::new() - } -} - -#[allow(dead_code)] -impl IoPriorityMetrics { - /// Create a new metrics instance. - pub const fn new() -> Self { - Self { - high_queue_depth: AtomicU64::new(0), - normal_queue_depth: AtomicU64::new(0), - low_queue_depth: AtomicU64::new(0), - high_wait_time_ns: AtomicU64::new(0), - normal_wait_time_ns: AtomicU64::new(0), - low_wait_time_ns: AtomicU64::new(0), - starvation_events: AtomicU64::new(0), - high_processed: AtomicU64::new(0), - normal_processed: AtomicU64::new(0), - low_processed: AtomicU64::new(0), - } - } - - /// Update queue depths from status. - #[allow(dead_code)] - pub fn update_queue_depths(&self, status: &IoQueueStatus) { - self.high_queue_depth - .store(status.high_priority_waiting as u64, Ordering::Relaxed); - self.normal_queue_depth - .store(status.normal_priority_waiting as u64, Ordering::Relaxed); - self.low_queue_depth - .store(status.low_priority_waiting as u64, Ordering::Relaxed); - } - - /// Record a starvation event. - #[allow(dead_code)] - pub fn record_starvation(&self) { - self.starvation_events.fetch_add(1, Ordering::Relaxed); - } - - /// Record a processed request. - #[allow(dead_code)] - pub fn record_processed(&self, priority: IoPriority) { - match priority { - IoPriority::High => self.high_processed.fetch_add(1, Ordering::Relaxed), - IoPriority::Normal => self.normal_processed.fetch_add(1, Ordering::Relaxed), - IoPriority::Low => self.low_processed.fetch_add(1, Ordering::Relaxed), - }; - } - - /// Record wait time for a priority level. - pub fn record_wait_time(&self, priority: IoPriority, wait_ns: u64) { - match priority { - IoPriority::High => self.high_wait_time_ns.fetch_add(wait_ns, Ordering::Relaxed), - IoPriority::Normal => self.normal_wait_time_ns.fetch_add(wait_ns, Ordering::Relaxed), - IoPriority::Low => self.low_wait_time_ns.fetch_add(wait_ns, Ordering::Relaxed), - }; - } - - /// Get high priority queue depth. - pub fn get_high_queue_depth(&self) -> u64 { - self.high_queue_depth.load(Ordering::Relaxed) - } - - /// Get normal priority queue depth. - pub fn get_normal_queue_depth(&self) -> u64 { - self.normal_queue_depth.load(Ordering::Relaxed) - } - - /// Get low priority queue depth. - pub fn get_low_queue_depth(&self) -> u64 { - self.low_queue_depth.load(Ordering::Relaxed) - } - - /// Get total starvation events. - pub fn get_starvation_events(&self) -> u64 { - self.starvation_events.load(Ordering::Relaxed) - } - - /// Get metrics summary for logging/debugging. - pub fn summary(&self) -> String { - format!( - "high_queue={}, normal_queue={}, low_queue={}, starvation={}, high_proc={}, normal_proc={}, low_proc={}", - self.get_high_queue_depth(), - self.get_normal_queue_depth(), - self.get_low_queue_depth(), - self.get_starvation_events(), - self.high_processed.load(Ordering::Relaxed), - self.normal_processed.load(Ordering::Relaxed), - self.low_processed.load(Ordering::Relaxed) - ) - } -} - -/// Global I/O priority metrics instance. -#[allow(dead_code)] -pub static IO_PRIORITY_METRICS: IoPriorityMetrics = IoPriorityMetrics::new(); - -/// Get optimized buffer size for I/O operations. -/// -/// This function provides adaptive buffer sizing based on: -/// - File size (small files get smaller buffers) -/// - Concurrent request count (high concurrency gets smaller buffers) -/// - Base buffer size from configuration -/// -/// # Arguments -/// -/// * `file_size` - Size of the file being read/written (-1 for unknown) -/// -/// # Returns -/// -/// Optimal buffer size in bytes -/// -/// # Example -/// -/// ```ignore -/// let buffer_size = get_buffer_size_opt_in(1024 * 1024); // 1MB file -/// assert!(buffer_size >= 64 * 1024); // At least 64KB -/// ``` -#[allow(dead_code)] -pub fn get_buffer_size_opt_in(file_size: i64) -> usize { - // Get base buffer size from configuration - let base_buffer_size = - rustfs_utils::get_env_usize(rustfs_config::ENV_OBJECT_IO_BUFFER_SIZE, rustfs_config::DEFAULT_OBJECT_IO_BUFFER_SIZE); - - // Apply concurrency-aware adjustments - get_concurrency_aware_buffer_size(file_size, base_buffer_size) -} - // ============================================ // Unit Tests // ============================================ @@ -1933,13 +1761,12 @@ pub fn get_buffer_size_opt_in(file_size: i64) -> usize { #[allow(unused_imports)] mod tests { use super::{ - IoLoadLevel, IoPriority, IoPriorityMetrics, IoPriorityQueue, IoPriorityQueueConfig, IoSchedulerConfig, - IoSchedulingContext, IoStrategy, get_advanced_buffer_size, get_buffer_size_opt_in, get_concurrency_aware_buffer_size, + IoLoadLevel, IoPriority, IoPriorityQueue, IoPriorityQueueConfig, IoSchedulerConfig, IoSchedulingContext, IoStrategy, + get_advanced_buffer_size, get_concurrency_aware_buffer_size, }; use rustfs_io_core::io_profile::{AccessPattern, StorageMedia}; use rustfs_io_metrics::bandwidth::{BandwidthSnapshot, BandwidthTier}; use serial_test::serial; - use std::sync::atomic::Ordering; use std::time::Duration; #[tokio::test] @@ -2126,29 +1953,6 @@ mod tests { assert_eq!(config.starvation_threshold_secs, 120); } - #[tokio::test] - #[serial] - async fn test_io_priority_metrics() { - let metrics = IoPriorityMetrics::new(); - - // Test initial state - assert_eq!(metrics.get_high_queue_depth(), 0); - assert_eq!(metrics.get_normal_queue_depth(), 0); - assert_eq!(metrics.get_low_queue_depth(), 0); - assert_eq!(metrics.get_starvation_events(), 0); - - // Test recording - metrics.record_starvation(); - assert_eq!(metrics.get_starvation_events(), 1); - - metrics.record_processed(IoPriority::High); - metrics.record_processed(IoPriority::High); - metrics.record_processed(IoPriority::Normal); - - assert_eq!(metrics.high_processed.load(Ordering::Relaxed), 2); - assert_eq!(metrics.normal_processed.load(Ordering::Relaxed), 1); - } - // ============================================ // Multi-Factor Strategy Tests // ============================================ diff --git a/rustfs/src/storage/concurrency/mod.rs b/rustfs/src/storage/concurrency/mod.rs index 9b14eebda..eb47aa4ed 100644 --- a/rustfs/src/storage/concurrency/mod.rs +++ b/rustfs/src/storage/concurrency/mod.rs @@ -24,16 +24,14 @@ //! - **Concurrency Management**: Coordination of concurrent GetObject requests //! - **Request Tracking**: RAII guards for request lifecycle management //! -//! # Migration Note +//! # Relationship to the shared crates //! -//! Core algorithms have been migrated to `rustfs-io-core` and metrics to -//! `rustfs-io-metrics`. This module maintains API compatibility while -//! delegating to the new implementations. +//! The scheduling algorithm lives in [`io_schedule`], not in `rustfs-io-core`: +//! this module does not delegate to it. `rustfs-io-core` owns the shared +//! config shapes and the `io_profile` storage-media model that [`io_schedule`] +//! consumes, and `rustfs-io-metrics` owns bandwidth sampling and metric +//! recording. -// Sub-modules -// pub mod bandwidth_monitor; // Migrated to rustfs-io-metrics -// pub mod global_metrics; // Migrated to rustfs-io-metrics -// pub mod io_profile; // Migrated to rustfs-io-core pub mod io_schedule; pub mod manager; pub mod request_guard; @@ -45,9 +43,8 @@ pub mod request_guard; // I/O scheduling types (from io_schedule.rs for backward compatibility) #[allow(unused_imports)] pub use io_schedule::{ - IO_PRIORITY_METRICS, IoLoadLevel, IoPriority, IoPriorityMetrics, IoPriorityQueue, IoPriorityQueueConfig, IoQueueStatus, - IoSchedulerConfig, IoStrategy, get_advanced_buffer_size, get_buffer_size_opt_in, get_concurrency_aware_buffer_size, - get_put_concurrency_aware_buffer_size, + IoLoadLevel, IoPriority, IoPriorityQueue, IoPriorityQueueConfig, IoQueueStatus, IoSchedulerConfig, IoStrategy, + get_advanced_buffer_size, get_concurrency_aware_buffer_size, get_put_concurrency_aware_buffer_size, }; // Request tracking @@ -56,24 +53,6 @@ pub use request_guard::{GetObjectGuard, PutObjectGuard}; // Concurrency manager pub use manager::{ConcurrencyManager, DiskReadAdmission, PutObjectAdmission}; -// ============================================ -// New Module Re-exports (for gradual migration) -// ============================================ - -// Re-export types from rustfs-io-core for convenience -pub use rustfs_io_core::{ - // Backpressure types - BackpressureMonitor, - // Deadlock detection types - DeadlockDetector, - // Scheduler types - IoScheduler, - // Lock optimization types - LockOptimizer, -}; - -// Re-export types from rustfs-io-metrics for convenience - // ============================================ // Helper Functions // ============================================ @@ -83,37 +62,8 @@ pub fn get_concurrency_manager() -> &'static ConcurrencyManager { ConcurrencyManager::global() } -/// Reset the active get requests counter (for testing). -#[allow(dead_code)] -pub fn reset_active_get_requests() { - io_schedule::ACTIVE_GET_REQUESTS.store(0, std::sync::atomic::Ordering::Relaxed); -} - -#[allow(dead_code)] +/// Reset the active put requests counter (for testing). +#[cfg(test)] pub fn reset_active_put_requests() { io_schedule::ACTIVE_PUT_REQUESTS.store(0, std::sync::atomic::Ordering::Relaxed); } - -/// Create a new I/O scheduler with default configuration. -#[allow(dead_code)] -pub fn create_io_scheduler() -> IoScheduler { - IoScheduler::with_defaults() -} - -/// Create a new backpressure monitor with default configuration. -#[allow(dead_code)] -pub fn create_backpressure_monitor() -> BackpressureMonitor { - BackpressureMonitor::with_defaults() -} - -/// Create a new deadlock detector with default configuration. -#[allow(dead_code)] -pub fn create_deadlock_detector() -> DeadlockDetector { - DeadlockDetector::with_defaults() -} - -/// Create a new lock optimizer with default configuration. -#[allow(dead_code)] -pub fn create_lock_optimizer() -> LockOptimizer { - LockOptimizer::with_defaults() -} From a08de9229b69115dc1c4142d04480ec33cded2d7 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 18 Aug 2026 12:43:27 +0800 Subject: [PATCH 16/22] feat(heal): wire MRF intents with durable repair journal (HS-01) (#6189) * feat(common): add MRF intent channel and Mrf request source (HS-01) Introduce the producer-facing half of the mission repair feed: a global bounded (8192) channel carrying lightweight MrfIntent values from IO error paths, plus the RUSTFS_HEAL_MRF_ENABLE delivery kill-switch and config constants for queue/journal sizing. Delivery is strictly non-blocking (try_send, drop-on-full) so it can sit on decode-failure and partial-write paths without adding latency. HealRequestSource grows a 'mrf' variant so admission accounting can attribute replayed intents. Part of backlog#1865 (option a: wire HealEvent-style intents with a durable retry ledger). Co-Authored-By: heihutu * feat(heal): add MRF queue, durable journal, and intent consumer (HS-01) Consumer half of the mission repair feed: a bounded pending queue (100k intents / 8 MiB dual ceiling, drop-newest on overflow), a durable journal at buckets/.heal/mrf/journal.bin holding the unaccepted pending snapshot, and a consumer task that batches intents off the global channel, translates them into prioritized heal requests (decode failure -> Urgent ECDecode, metadata corruption -> High Metadata, partial write -> Normal object heal), and retries full admissions with a 5s backoff and a 3-attempt ceiling. Durability: every journal record carries its own CRC32 and a format/version header, so a torn tail truncates cleanly at replay; the journal is deleted after a successful replay and when the pending set drains (mirroring MinIO's post-replay list.bin unlink). Losing the last 500 ms flush window is acceptable: replayed duplicates merge via the manager dedup key and read-repair remains the safety net. Metrics: rustfs_heal_mrf_queue_depth/_queue_bytes, _dropped_total {reason}, _replayed_total, _journal_bytes, _journal_fsync_total. The consumer is wired at heal runtime bootstrap right after manager start, honoring RUSTFS_HEAL_MRF_ENABLE (default on, rollback = off). Tests: unit tests for the dual ceiling, record roundtrip, torn-tail truncation, and the priority mapping; integration tests against a real 4-disk ECStore proving channel intents reach the manager queue as Urgent/mrf-attributed requests and journal replay arms intents, drops torn tails, and removes the file. Part of backlog#1865 (option a). Co-Authored-By: heihutu * feat(ecstore,scanner): deliver MRF intents from error paths (HS-01) Wire the three production delivery points, each a single non-blocking try_send next to the existing in-memory heal paths, which stay as the fast path: - read.rs decode-error branch: DecodeFailure intent beside the existing read-repair submit, so an Urgent ECDecode request survives restarts even when the Low-priority read-repair request was dropped or lost. - add_partial: PartialWrite intent, giving partial-write recovery a durable Normal-priority object heal across restarts. - scanner_folder metadata-corruption classification: MetadataCorruption intent beside the existing High-priority scanner heal request. All three are on error paths only: zero cost on healthy IO. Part of backlog#1865 (option a). Co-Authored-By: heihutu * fix: include mrf heal source counts Co-Authored-By: heihutu * fix: keep node heal status wire compatibility Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- crates/common/src/heal_channel.rs | 4 + crates/common/src/lib.rs | 1 + crates/common/src/mrf_channel.rs | 203 ++++++ crates/config/src/constants/heal.rs | 28 + crates/ecstore/src/set_disk/ops/object.rs | 8 + crates/ecstore/src/set_disk/read.rs | 9 + crates/heal/Cargo.toml | 2 + crates/heal/src/heal/channel.rs | 3 +- crates/heal/src/heal/manager.rs | 3 + crates/heal/src/heal/mod.rs | 1 + crates/heal/src/heal/mrf_queue.rs | 682 ++++++++++++++++++++ crates/heal/src/lib.rs | 4 + crates/heal/tests/mrf_pipeline_test.rs | 189 ++++++ crates/scanner/src/scanner_folder.rs | 9 + rustfs/src/admin/handlers/heal.rs | 2 + rustfs/src/storage/rpc/node_service/heal.rs | 1 + 16 files changed, 1148 insertions(+), 1 deletion(-) create mode 100644 crates/common/src/mrf_channel.rs create mode 100644 crates/heal/src/heal/mrf_queue.rs create mode 100644 crates/heal/tests/mrf_pipeline_test.rs diff --git a/crates/common/src/heal_channel.rs b/crates/common/src/heal_channel.rs index f0ea0530e..b5a907e4e 100644 --- a/crates/common/src/heal_channel.rs +++ b/crates/common/src/heal_channel.rs @@ -287,6 +287,9 @@ pub enum HealRequestSource { Scanner, AutoHeal, ReadRepair, + /// Mission Repair Feed: intents delivered by error paths and replayed + /// from the durable MRF journal. + Mrf, } impl HealRequestSource { @@ -297,6 +300,7 @@ impl HealRequestSource { Self::Scanner => "scanner", Self::AutoHeal => "auto_heal", Self::ReadRepair => "read_repair", + Self::Mrf => "mrf", } } } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 1ae200f24..68f5f5c68 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -17,6 +17,7 @@ pub mod globals; pub mod heal_channel; pub mod last_minute; pub mod metrics; +pub mod mrf_channel; mod readiness; pub mod table_catalog; pub mod trace_bus; diff --git a/crates/common/src/mrf_channel.rs b/crates/common/src/mrf_channel.rs new file mode 100644 index 000000000..f0a91a238 --- /dev/null +++ b/crates/common/src/mrf_channel.rs @@ -0,0 +1,203 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Mission Repair Feed (MRF) intent channel. +//! +//! Producers on error paths (read decode failure, scanner metadata +//! corruption, partial-write recovery) hand a lightweight [`MrfIntent`] to the +//! heal crate through a global bounded channel. Delivery is strictly +//! non-blocking: `try_send_mrf_intent` never awaits and drops the intent +//! (counting it) when the channel is full or uninitialized — losing one heal +//! hint is always preferred over stalling an IO path. Durable replay of +//! unconsumed intents is the consumer's job (see `rustfs-heal` +//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`. + +use std::sync::{ + Arc, OnceLock, + atomic::{AtomicBool, Ordering}, +}; +use tokio::sync::mpsc; +use uuid::Uuid; + +/// Bounded capacity of the global MRF channel. Backpressure is resolved by +/// dropping (and counting) intents, never by blocking the producer. +const MRF_CHANNEL_CAPACITY: usize = 8192; + +/// Why an intent was produced. Drives the heal priority mapping on the +/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High, +/// PartialWrite -> Normal). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MrfKind { + /// Erasure decode failed while serving a read (read path). + DecodeFailure, + /// Scanner classified object metadata as corrupt. + MetadataCorruption, + /// A write left the object with fewer committed shards than the set size. + PartialWrite, +} + +impl MrfKind { + pub const fn as_str(self) -> &'static str { + match self { + MrfKind::DecodeFailure => "decode-failure", + MrfKind::MetadataCorruption => "metadata-corruption", + MrfKind::PartialWrite => "partial-write", + } + } +} + +/// One repair intent. Kept deliberately small so the in-memory queue and the +/// journal stay bounded; `bucket`/`object` are `Arc` so re-arming an +/// intent never re-allocates the strings. +#[derive(Clone, Debug)] +pub struct MrfIntent { + pub bucket: Arc, + pub object: Arc, + /// Version the intent targets, as raw UUID bytes. + pub version_id: Option<[u8; 16]>, + pub kind: MrfKind, + pub enqueued_at_ms: u64, + /// Times this intent has already been offered to the heal manager. + /// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`. + pub attempts: u8, +} + +/// Consumer-side retry ceiling before an intent is given up on. +pub const MRF_MAX_ATTEMPTS: u8 = 3; + +impl MrfIntent { + /// Rough in-memory footprint used by the queue's byte budget. + pub fn estimated_bytes(&self) -> usize { + // Struct + strings + version bytes; buckets and objects are usually + // far below this bound, so rounding up keeps the budget conservative. + 64 + self.bucket.len() + self.object.len() + } +} + +static GLOBAL_MRF_SENDER: OnceLock> = OnceLock::new(); + +/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check +/// this before touching the channel so the disabled path stays allocation- and +/// sync-free. +static MRF_DELIVERY_ENABLED: AtomicBool = AtomicBool::new(true); + +/// Override delivery (used at heal-runtime startup from configuration). +pub fn set_mrf_delivery_enabled(enabled: bool) { + MRF_DELIVERY_ENABLED.store(enabled, Ordering::Relaxed); +} + +/// Whether producers currently deliver intents. +pub fn mrf_delivery_enabled() -> bool { + MRF_DELIVERY_ENABLED.load(Ordering::Relaxed) +} + +/// Create the global MRF channel and return the consumer half. Fails if the +/// channel is already initialized (the heal runtime is a singleton). +pub fn init_mrf_channel() -> Result, &'static str> { + let (sender, receiver) = mpsc::channel(MRF_CHANNEL_CAPACITY); + GLOBAL_MRF_SENDER + .set(sender) + .map_err(|_| "MRF channel sender already initialized")?; + Ok(receiver) +} + +/// Best-effort, non-blocking intent delivery from an error path. +/// +/// Returns `true` when the intent was accepted into the channel. `false` +/// means the intent was dropped (feature disabled, channel not yet +/// initialized, or channel full) — callers must not retry or await; the +/// existing read-repair / scanner heal paths remain the safety net. +/// +/// This runs on IO error paths, so it stays synchronous and cheap: one +/// bounded allocation for the two `Arc` handles plus the channel slot. +pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option) -> bool { + if !mrf_delivery_enabled() { + return false; + } + let Some(sender) = GLOBAL_MRF_SENDER.get() else { + return false; + }; + let intent = MrfIntent { + bucket: Arc::from(bucket), + object: Arc::from(object), + version_id: version_id.map(|vid| *vid.as_bytes()), + kind, + enqueued_at_ms: unix_now_ms(), + attempts: 0, + }; + sender.try_send(intent).is_ok() +} + +fn unix_now_ms() -> u64 { + // Kept trivial: the timestamp is diagnostic metadata only; wall-clock + // failure would be a bug rather than something to handle here. + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn intents_estimate_is_conservative() { + let intent = MrfIntent { + bucket: Arc::from("bucket"), + object: Arc::from("object"), + version_id: Some([0u8; 16]), + kind: MrfKind::DecodeFailure, + enqueued_at_ms: 0, + attempts: 0, + }; + assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len()); + } + + #[tokio::test] + async fn try_send_delivers_and_respects_capacity() { + let mut receiver = init_mrf_channel().expect("first initialization should succeed"); + assert!(init_mrf_channel().is_err(), "double initialization must fail"); + + assert!(try_send_mrf_intent(MrfKind::DecodeFailure, "b", "o", Some(Uuid::nil()))); + let intent = receiver.recv().await.expect("intent should arrive"); + assert_eq!(intent.kind, MrfKind::DecodeFailure); + assert_eq!(intent.bucket.as_ref(), "b"); + + // Disable delivery: producers become no-ops. + set_mrf_delivery_enabled(false); + assert!(!try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None)); + set_mrf_delivery_enabled(true); + + // Fill the bounded channel past capacity: excess intents are dropped, + // never blocking. + let mut accepted = 0; + for _ in 0..(MRF_CHANNEL_CAPACITY + 64) { + if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) { + accepted += 1; + } + } + assert_eq!(accepted, MRF_CHANNEL_CAPACITY); + } + + #[test] + fn try_send_without_channel_is_false() { + // This test may run after the tokio test above in the same process; + // the singleton semantics make a clean "uninitialized" case hard, so + // assert the flag-off behavior only. + set_mrf_delivery_enabled(false); + assert!(!try_send_mrf_intent(MrfKind::MetadataCorruption, "b", "o", None)); + set_mrf_delivery_enabled(true); + } +} diff --git a/crates/config/src/constants/heal.rs b/crates/config/src/constants/heal.rs index 647ca8533..b8cf3630a 100644 --- a/crates/config/src/constants/heal.rs +++ b/crates/config/src/constants/heal.rs @@ -177,3 +177,31 @@ pub const DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT: usize = 80; /// Default foreground pressure recheck delay for heal scheduler, in milliseconds. pub const DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS: u64 = 250; + +/// Environment variable that toggles the MRF (mission repair feed) intent +/// pipeline: error paths deliver repair intents to the heal runtime, and +/// unconsumed intents are replayed from the durable journal after a restart. +pub const ENV_HEAL_MRF_ENABLE: &str = "RUSTFS_HEAL_MRF_ENABLE"; + +/// Environment variable for the MRF in-memory queue capacity (intent count). +pub const ENV_HEAL_MRF_QUEUE_SIZE: &str = "RUSTFS_HEAL_MRF_QUEUE_SIZE"; + +/// Environment variable for the MRF journal byte budget. The journal is +/// compacted once its on-disk size crosses this bound. +pub const ENV_HEAL_MRF_JOURNAL_MAX_BYTES: &str = "RUSTFS_HEAL_MRF_JOURNAL_MAX_BYTES"; + +/// Environment variable for the MRF journal replay batch size (intents per +/// replay push round). +pub const ENV_HEAL_MRF_REPLAY_BATCH: &str = "RUSTFS_HEAL_MRF_REPLAY_BATCH"; + +/// Default behavior keeps the MRF intent pipeline enabled. +pub const DEFAULT_HEAL_MRF_ENABLE: bool = true; + +/// Default MRF queue capacity (matches MinIO's 100k MRF list ceiling). +pub const DEFAULT_HEAL_MRF_QUEUE_SIZE: usize = 100_000; + +/// Default MRF journal byte budget (8 MiB), mirroring the channel payload cap. +pub const DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES: usize = 8 * 1024 * 1024; + +/// Default MRF replay batch size. +pub const DEFAULT_HEAL_MRF_REPLAY_BATCH: usize = 256; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index e7498ee9d..929ac4d5c 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -5845,6 +5845,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { #[tracing::instrument(skip(self))] async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { + // MRF journal intent: partial-write recovery must survive a restart + // (HS-01); the heal request below remains the in-memory fast path. + rustfs_common::mrf_channel::try_send_mrf_intent( + rustfs_common::mrf_channel::MrfKind::PartialWrite, + bucket, + object, + uuid::Uuid::try_parse(version_id).ok(), + ); let mut request = rustfs_common::heal_channel::create_heal_request_with_options( bucket.to_string(), Some(object.to_string()), diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index d59fa3e7e..07bbcf5f9 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -1077,6 +1077,15 @@ impl SetDisks { "Recoverable decode error triggered read repair" ); let version_id = fi.version_id.as_ref().map(ToString::to_string); + // MRF journal intent: keeps a durable Urgent ECDecode + // request alive across restarts even when the in-memory + // read-repair request is dropped or lost (HS-01). + rustfs_common::mrf_channel::try_send_mrf_intent( + rustfs_common::mrf_channel::MrfKind::DecodeFailure, + bucket, + object, + fi.version_id, + ); submit_read_repair_heal( bucket, object, diff --git a/crates/heal/Cargo.toml b/crates/heal/Cargo.toml index 668c4af24..4d4fb355b 100644 --- a/crates/heal/Cargo.toml +++ b/crates/heal/Cargo.toml @@ -89,6 +89,8 @@ async-trait = { workspace = true } futures = { workspace = true } metrics = { workspace = true } base64 = { workspace = true } +bytes = { workspace = true } +crc-fast = { workspace = true } [dev-dependencies] serde_json = { workspace = true, features = ["raw_value"] } diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index 23cf2f168..1e31056b2 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -612,7 +612,8 @@ impl HealChannelProcessor { HealRequestSource::Admin | HealRequestSource::AutoHeal | HealRequestSource::Internal - | HealRequestSource::ReadRepair => true, + | HealRequestSource::ReadRepair + | HealRequestSource::Mrf => true, }); // Build HealOptions with all available fields diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 66b8f637f..216e17068 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -270,6 +270,8 @@ pub struct HealSourceCounts { pub auto_heal: u64, pub internal: u64, pub read_repair: u64, + #[serde(default)] + pub mrf: u64, } impl HealSourceCounts { @@ -280,6 +282,7 @@ impl HealSourceCounts { HealRequestSource::AutoHeal => self.auto_heal += 1, HealRequestSource::Internal => self.internal += 1, HealRequestSource::ReadRepair => self.read_repair += 1, + HealRequestSource::Mrf => self.mrf += 1, } } } diff --git a/crates/heal/src/heal/mod.rs b/crates/heal/src/heal/mod.rs index 0881e5853..ff910ed70 100644 --- a/crates/heal/src/heal/mod.rs +++ b/crates/heal/src/heal/mod.rs @@ -16,6 +16,7 @@ pub mod channel; pub mod erasure_healer; pub mod event; pub mod manager; +pub mod mrf_queue; pub mod progress; pub(crate) mod replacement_readiness; pub mod resume; diff --git a/crates/heal/src/heal/mrf_queue.rs b/crates/heal/src/heal/mrf_queue.rs new file mode 100644 index 000000000..c1144435e --- /dev/null +++ b/crates/heal/src/heal/mrf_queue.rs @@ -0,0 +1,682 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Mission Repair Feed (MRF) queue, journal, and consumer. +//! +//! Intents arriving on the global channel (see `rustfs_common::mrf_channel`) +//! are buffered in a bounded in-memory queue, translated into prioritized +//! heal requests, and — while they are not yet accepted by the heal manager — +//! mirrored into a durable journal so a crash or restart can replay them. +//! This is the RustFS counterpart of MinIO's `.heal/mrf/list.bin` replay, +//! layered on top of (not replacing) read-repair and scanner heal. +//! +//! Durability model: the journal is a snapshot of the *unaccepted* pending +//! set, rewritten on a group-commit cadence (every flush interval or flush +//! threshold new intents). A rewrite is atomic at the record level only — a +//! torn tail simply truncates during replay because every record carries its +//! own CRC32. Losing the last flush window (≤500 ms) is acceptable: replayed +//! duplicates are merged by the manager's dedup key, and read-repair remains +//! the safety net. + +use super::{DiskStore, HealDiskExt as _, local_disk_map_read}; +use crate::heal::manager::HealManager; +use metrics::{counter, gauge}; +use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult}; +use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfIntent}; +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc; +use uuid::Uuid; + +use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType}; + +/// Journal location inside the metadata bucket, following the resume-state +/// layout. +pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin"; + +/// Record format tag. +const MRF_JOURNAL_FORMAT: u8 = 1; +/// Record layout version. +const MRF_JOURNAL_VERSION: u8 = 1; + +/// Fixed header size: format, version, kind, attempts, enqueued_at_ms, +/// has_version flag. +const MRF_RECORD_FIXED_HEAD: usize = 1 + 1 + 1 + 1 + 8 + 1; + +#[derive(Debug, Clone)] +pub(crate) struct MrfConsumerConfig { + /// In-memory queue capacity in intents. + pub queue_capacity: usize, + /// Journal byte budget; a pending snapshot above this bound is rejected + /// oldest-first so the journal can never grow unbounded. + pub journal_max_bytes: usize, + /// How many journal intents to re-arm per replay round. + pub replay_batch: usize, + /// Group-commit cadence for the journal snapshot. + pub flush_interval: Duration, + /// New intents between flushes that force an early snapshot. + pub flush_threshold: usize, + /// Backoff after the heal manager reports a full admission. + pub admission_backoff: Duration, +} + +impl Default for MrfConsumerConfig { + fn default() -> Self { + Self { + queue_capacity: rustfs_utils::get_env_usize( + rustfs_config::ENV_HEAL_MRF_QUEUE_SIZE, + rustfs_config::DEFAULT_HEAL_MRF_QUEUE_SIZE, + ), + journal_max_bytes: rustfs_utils::get_env_usize( + rustfs_config::ENV_HEAL_MRF_JOURNAL_MAX_BYTES, + rustfs_config::DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES, + ), + replay_batch: rustfs_utils::get_env_usize( + rustfs_config::ENV_HEAL_MRF_REPLAY_BATCH, + rustfs_config::DEFAULT_HEAL_MRF_REPLAY_BATCH, + ), + flush_interval: Duration::from_millis(500), + flush_threshold: 1000, + admission_backoff: Duration::from_secs(5), + } + } +} + +/// Bounded pending set with count and byte ceilings. Overflow drops the +/// incoming intent (never a resident one) and counts the loss. +pub(crate) struct MrfQueue { + pending: VecDeque, + bytes: usize, + capacity: usize, + byte_budget: usize, +} + +impl MrfQueue { + pub(crate) fn new(capacity: usize, byte_budget: usize) -> Self { + Self { + pending: VecDeque::new(), + bytes: 0, + capacity, + byte_budget, + } + } + + /// Returns `false` (after counting) when either ceiling would be crossed. + pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool { + let cost = intent.estimated_bytes(); + if self.pending.len() >= self.capacity || self.bytes + cost > self.byte_budget { + counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1); + return false; + } + self.bytes += cost; + self.pending.push_back(intent); + true + } + + pub(crate) fn pop_front(&mut self) -> Option { + let intent = self.pending.pop_front()?; + self.bytes = self.bytes.saturating_sub(intent.estimated_bytes()); + Some(intent) + } + + pub(crate) fn push_back(&mut self, intent: MrfIntent) { + self.bytes += intent.estimated_bytes(); + self.pending.push_back(intent); + } + + pub(crate) fn depth(&self) -> usize { + self.pending.len() + } + + pub(crate) fn bytes(&self) -> usize { + self.bytes + } + + pub(crate) fn intents(&self) -> impl Iterator { + self.pending.iter() + } +} + +// --------------------------------------------------------------------------- +// Journal record codec +// --------------------------------------------------------------------------- + +/// Append one encoded record to `out`. +pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec) { + let start = out.len(); + out.push(MRF_JOURNAL_FORMAT); + out.push(MRF_JOURNAL_VERSION); + out.push(match intent.kind { + rustfs_common::mrf_channel::MrfKind::DecodeFailure => 1, + rustfs_common::mrf_channel::MrfKind::MetadataCorruption => 2, + rustfs_common::mrf_channel::MrfKind::PartialWrite => 3, + }); + out.push(intent.attempts); + out.extend_from_slice(&intent.enqueued_at_ms.to_le_bytes()); + match intent.version_id { + Some(bytes) => { + out.push(1); + out.extend_from_slice(&bytes); + } + None => out.push(0), + } + out.extend_from_slice(&(intent.bucket.len() as u32).to_le_bytes()); + out.extend_from_slice(&(intent.object.len() as u32).to_le_bytes()); + out.extend_from_slice(intent.bucket.as_bytes()); + out.extend_from_slice(intent.object.as_bytes()); + let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + hasher.update(&out[start..]); + out.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes()); +} + +fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> { + if data.len() < MRF_RECORD_FIXED_HEAD + 8 { + return None; + } + if data[0] != MRF_JOURNAL_FORMAT || data[1] != MRF_JOURNAL_VERSION { + return None; + } + let kind = match data[2] { + 1 => rustfs_common::mrf_channel::MrfKind::DecodeFailure, + 2 => rustfs_common::mrf_channel::MrfKind::MetadataCorruption, + 3 => rustfs_common::mrf_channel::MrfKind::PartialWrite, + _ => return None, + }; + let attempts = data[3]; + let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().expect("slice length checked")); + let has_version = data[12] != 0; + let mut cursor = MRF_RECORD_FIXED_HEAD; + let version_id = if has_version { + if data.len() < cursor + 16 { + return None; + } + let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().expect("slice length checked"); + cursor += 16; + Some(bytes) + } else { + None + }; + if data.len() < cursor + 8 { + return None; + } + let bucket_len = u32::from_le_bytes(data[cursor..cursor + 4].try_into().expect("slice length checked")) as usize; + let object_len = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().expect("slice length checked")) as usize; + cursor += 8; + let body_end = cursor.checked_add(bucket_len)?.checked_add(object_len)?; + let record_end = body_end.checked_add(4)?; + if data.len() < record_end { + return None; + } + let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + hasher.update(&data[..body_end]); + if (hasher.finalize() as u32) != u32::from_le_bytes(data[body_end..record_end].try_into().expect("slice length checked")) { + return None; + } + let bucket = std::sync::Arc::from(std::str::from_utf8(&data[cursor..cursor + bucket_len]).ok()?); + let object = std::sync::Arc::from(std::str::from_utf8(&data[cursor + bucket_len..body_end]).ok()?); + Some(( + MrfIntent { + bucket, + object, + version_id, + kind, + enqueued_at_ms, + attempts, + }, + record_end, + )) +} + +/// Decode a whole journal, stopping at the first torn or corrupt record. +/// Returns the decoded intents and the number of trailing bytes discarded. +pub(crate) fn decode_journal(data: &[u8]) -> (Vec, usize) { + let mut intents = Vec::new(); + let mut cursor = 0usize; + while cursor < data.len() { + match decode_one(&data[cursor..]) { + Some((intent, consumed)) => { + intents.push(intent); + cursor += consumed; + } + None => break, + } + } + let truncated = data.len() - cursor; + (intents, truncated) +} + +// --------------------------------------------------------------------------- +// Journal disk IO (all local disks, first successful read wins) +// --------------------------------------------------------------------------- + +async fn journal_disks() -> Vec { + let map = local_disk_map_read().await; + map.values().flatten().cloned().collect() +} + +async fn read_journal() -> Option> { + for disk in journal_disks().await { + match disk.read_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH).await { + Ok(bytes) => return Some(bytes.to_vec()), + Err(_) => continue, + } + } + None +} + +async fn write_journal(data: &[u8]) { + let payload = bytes::Bytes::copy_from_slice(data); + for disk in journal_disks().await { + if let Err(err) = disk + .write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone()) + .await + { + warn_mrf_journal_write(&err); + } + } + if !data.is_empty() { + counter!("rustfs_heal_mrf_journal_fsync_total").increment(1); + } + gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64); +} + +async fn delete_journal() { + for disk in journal_disks().await { + let _ = disk + .delete( + super::RUSTFS_META_BUCKET, + MRF_JOURNAL_PATH, + crate::heal::storage_api::owner::EcstoreDeleteOptions::default(), + ) + .await; + } +} + +fn warn_mrf_journal_write(err: &super::DiskError) { + tracing::warn!( + target: "rustfs::heal::mrf", + error = %err, + "MRF journal write failed; unconsumed intents may be lost on restart" + ); +} + +// --------------------------------------------------------------------------- +// Consumer +// --------------------------------------------------------------------------- + +/// Translate an intent into the prioritized heal request the issue specifies: +/// decode failures go Urgent ECDecode, metadata corruption goes High +/// Metadata, partial writes go Normal object heal. +pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest { + let bucket = intent.bucket.to_string(); + let object = intent.object.to_string(); + let version_id = intent.version_id.map(|bytes| Uuid::from_bytes(bytes).to_string()); + let (heal_type, priority) = match intent.kind { + rustfs_common::mrf_channel::MrfKind::DecodeFailure => ( + HealType::ECDecode { + bucket, + object, + version_id, + }, + HealPriority::Urgent, + ), + rustfs_common::mrf_channel::MrfKind::MetadataCorruption => (HealType::Metadata { bucket, object }, HealPriority::High), + rustfs_common::mrf_channel::MrfKind::PartialWrite => ( + HealType::Object { + bucket, + object, + version_id, + }, + HealPriority::Normal, + ), + }; + let mut request = HealRequest::new(heal_type, HealOptions::default(), priority); + request.source = rustfs_common::heal_channel::HealRequestSource::Mrf; + request +} + +struct MrfRuntime { + queue: MrfQueue, + config: MrfConsumerConfig, + new_since_flush: usize, + /// True while a journal snapshot exists on disk that no longer reflects + /// an all-consumed pending set; the next idle tick removes it (MinIO + /// deletes its `list.bin` after replay for the same reason). + journal_on_disk: bool, + /// Earliest instant a full-admission retry may proceed. + backoff_until: Option, +} + +impl MrfRuntime { + fn record_accept(&mut self) { + // Accepted intents leave the pending set; the next flush persists the + // smaller snapshot, which is the journal's compaction. + } + + fn snapshot(&self) -> Vec { + let mut buf = Vec::new(); + for intent in self.queue.intents() { + encode_intent(intent, &mut buf); + } + buf + } + + async fn flush(&mut self) { + write_journal(&self.snapshot()).await; + self.new_since_flush = 0; + self.journal_on_disk = true; + } + + /// Drain pending intents into the heal manager until it is full, the + /// queue empties, or attempts are exhausted. + async fn dispatch(&mut self, manager: &HealManager) { + if let Some(until) = self.backoff_until { + if tokio::time::Instant::now() < until { + return; + } + self.backoff_until = None; + } + while let Some(mut intent) = self.queue.pop_front() { + let request = build_heal_request(&intent); + match manager.submit_heal_request(request).await { + Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => self.record_accept(), + Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => { + intent.attempts = intent.attempts.saturating_add(1); + if intent.attempts >= MRF_MAX_ATTEMPTS { + counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1); + continue; + } + self.queue.push_back(intent); + self.backoff_until = Some(tokio::time::Instant::now() + self.config.admission_backoff); + break; + } + Ok(HealAdmissionResult::Dropped(_)) => { + counter!("rustfs_heal_mrf_dropped_total", "reason" => "admission_policy").increment(1); + } + Err(_) => { + intent.attempts = intent.attempts.saturating_add(1); + if intent.attempts >= MRF_MAX_ATTEMPTS { + counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1); + continue; + } + self.queue.push_back(intent); + self.backoff_until = Some(tokio::time::Instant::now() + self.config.admission_backoff); + break; + } + } + } + gauge!("rustfs_heal_mrf_queue_depth").set(self.queue.depth() as f64); + gauge!("rustfs_heal_mrf_queue_bytes").set(self.queue.bytes() as f64); + } +} + +/// Initialize the global MRF channel (honoring `RUSTFS_HEAL_MRF_ENABLE`) and +/// spawn the consumer task. Called once from the heal runtime bootstrap right +/// after the manager started; a disabled feature or a double call is a no-op. +/// Public for integration tests that drive the real consumer loop. +pub fn spawn_mrf_consumer(manager: Arc) { + let enabled = rustfs_utils::get_env_bool(rustfs_config::ENV_HEAL_MRF_ENABLE, rustfs_config::DEFAULT_HEAL_MRF_ENABLE); + rustfs_common::mrf_channel::set_mrf_delivery_enabled(enabled); + if !enabled { + tracing::info!( + target: "rustfs::heal::mrf", + "MRF intent pipeline disabled by configuration; producers will not deliver" + ); + return; + } + let receiver = match rustfs_common::mrf_channel::init_mrf_channel() { + Ok(receiver) => receiver, + Err(err) => { + tracing::warn!( + target: "rustfs::heal::mrf", + error = err, + "MRF channel initialization failed; intents will be dropped at producers" + ); + return; + } + }; + tokio::spawn(async move { + run_mrf_consumer(manager, receiver).await; + }); + tracing::info!(target: "rustfs::heal::mrf", "MRF intent consumer started"); +} + +/// Replay the durable journal into a fresh pending queue and submit whatever +/// it armed. Returns the number of intact intents replayed. Duplicates are +/// merged by the manager's dedup key; the journal file is removed once read +/// (torn tails truncate via the per-record CRC). Public for integration tests; +/// the live consumer invokes this through [`replay_into`] at startup. +pub async fn replay_journal_once(manager: &Arc) -> usize { + let config = MrfConsumerConfig::default(); + let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes); + let mut backoff_until: Option = None; + replay_into(manager, &mut queue, &mut backoff_until).await +} + +/// Shared replay core: read + decode + re-arm + delete, then drain what fits. +async fn replay_into( + manager: &Arc, + queue: &mut MrfQueue, + backoff_until: &mut Option, +) -> usize { + let Some(data) = read_journal().await else { + return 0; + }; + let (intents, truncated) = decode_journal(&data); + if truncated > 0 { + tracing::warn!( + target: "rustfs::heal::mrf", + truncated_bytes = truncated, + "MRF journal had a torn tail; truncated records were discarded" + ); + } + counter!("rustfs_heal_mrf_replayed_total").increment(intents.len() as u64); + let replayed = intents.len(); + for intent in intents { + queue.try_push(intent); + } + delete_journal().await; + + // Drain the replayed intents immediately; whatever the manager refuses + // stays armed in `queue` for the consumer's retry loop. + if backoff_until.is_none() { + while let Some(mut intent) = queue.pop_front() { + let request = build_heal_request(&intent); + match manager.submit_heal_request(request).await { + Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {} + Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => { + intent.attempts = intent.attempts.saturating_add(1); + if intent.attempts < MRF_MAX_ATTEMPTS { + queue.push_back(intent); + *backoff_until = Some(tokio::time::Instant::now()); + } + break; + } + Ok(HealAdmissionResult::Dropped(_)) | Err(_) => {} + } + } + } + replayed +} + +/// Replay the journal, then keep draining the channel into the heal manager +/// while persisting the pending snapshot. +async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receiver) { + let config = MrfConsumerConfig::default(); + let mut runtime = MrfRuntime { + queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes), + config: config.clone(), + new_since_flush: 0, + journal_on_disk: false, + backoff_until: None, + }; + + // Replay: read the journal, re-arm intents (duplicates are merged by the + // manager's dedup key), then drop the file so the next flush starts clean. + replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await; + + let mut flush_tick = tokio::time::interval(runtime.config.flush_interval); + flush_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut batch: Vec = Vec::with_capacity(runtime.config.replay_batch); + + loop { + tokio::select! { + received = receiver.recv_many(&mut batch, runtime.config.replay_batch) => { + if received == 0 { + // Channel closed: flush once more and stop. + runtime.flush().await; + tracing::info!( + target: "rustfs::heal::mrf", + "MRF channel closed; consumer stopped after final flush" + ); + return; + } + for intent in batch.drain(..) { + runtime.queue.try_push(intent); + runtime.new_since_flush += 1; + } + runtime.dispatch(manager.as_ref()).await; + if runtime.new_since_flush >= runtime.config.flush_threshold { + runtime.flush().await; + } + } + _ = flush_tick.tick() => { + if runtime.new_since_flush > 0 || runtime.queue.depth() > 0 { + runtime.flush().await; + runtime.dispatch(manager.as_ref()).await; + } else if runtime.journal_on_disk { + // All intents consumed: remove the journal so a restart + // replays nothing (mirrors MinIO's post-replay unlink). + delete_journal().await; + runtime.journal_on_disk = false; + gauge!("rustfs_heal_mrf_journal_bytes").set(0.0); + } + gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_common::mrf_channel::{MrfIntent, MrfKind}; + use std::sync::Arc as StdArc; + + fn intent(bucket: &str, object: &str, attempts: u8) -> MrfIntent { + MrfIntent { + bucket: StdArc::from(bucket), + object: StdArc::from(object), + version_id: Some([7u8; 16]), + kind: MrfKind::DecodeFailure, + enqueued_at_ms: 1_700_000_000_000, + attempts, + } + } + + #[test] + fn queue_enforces_count_and_byte_ceilings() { + let mut queue = MrfQueue::new(2, usize::MAX); + assert!(queue.try_push(intent("b", "o", 0))); + assert!(queue.try_push(intent("b", "o", 0))); + assert!(!queue.try_push(intent("b", "o", 0)), "count ceiling must drop"); + + let mut tiny = MrfQueue::new(usize::MAX, intent("bucket", "object", 0).estimated_bytes()); + assert!(tiny.try_push(intent("bucket", "object", 0))); + assert!( + !tiny.try_push(intent("bucket", "object", 0)), + "byte budget must drop before the second intent fits" + ); + } + + #[test] + fn journal_roundtrip_preserves_intents() { + let intents = vec![ + intent("bucket-a", "object/a", 0), + intent("bucket-b", "object/b", 2), + MrfIntent { + bucket: StdArc::from("bucket-c"), + object: StdArc::from("object/c"), + version_id: None, + kind: MrfKind::MetadataCorruption, + enqueued_at_ms: 5, + attempts: 1, + }, + ]; + let mut buf = Vec::new(); + for intent in &intents { + encode_intent(intent, &mut buf); + } + let (decoded, truncated) = decode_journal(&buf); + assert_eq!(truncated, 0); + assert_eq!(decoded.len(), intents.len()); + for (left, right) in decoded.iter().zip(intents.iter()) { + assert_eq!(left.bucket, right.bucket); + assert_eq!(left.object, right.object); + assert_eq!(left.version_id, right.version_id); + assert_eq!(left.kind, right.kind); + assert_eq!(left.attempts, right.attempts); + } + } + + #[test] + fn journal_torn_tail_is_truncated() { + let mut buf = Vec::new(); + encode_intent(&intent("b", "o", 0), &mut buf); + let mut torn = buf.clone(); + torn.extend_from_slice(&buf[..buf.len() / 2]); + + let (decoded, truncated) = decode_journal(&torn); + assert_eq!(decoded.len(), 1, "the intact record must survive"); + assert!(truncated > 0, "the partial tail must be discarded"); + + // A corrupted body (CRC mismatch) also truncates from that record on. + let mut corrupt = buf.clone(); + let mid = MRF_RECORD_FIXED_HEAD + 4; + corrupt[mid] ^= 0xff; + let (decoded, truncated) = decode_journal(&corrupt); + assert!(decoded.is_empty()); + assert_eq!(truncated, corrupt.len()); + } + + #[test] + fn heal_request_mapping_follows_priority_matrix() { + let decode = build_heal_request(&intent("b", "o", 0)); + assert!(matches!(decode.heal_type, HealType::ECDecode { .. })); + assert_eq!(decode.priority, HealPriority::Urgent); + + let metadata = build_heal_request(&MrfIntent { + bucket: StdArc::from("b"), + object: StdArc::from("o"), + version_id: None, + kind: MrfKind::MetadataCorruption, + enqueued_at_ms: 0, + attempts: 0, + }); + assert!(matches!(metadata.heal_type, HealType::Metadata { .. })); + assert_eq!(metadata.priority, HealPriority::High); + + let partial = build_heal_request(&MrfIntent { + bucket: StdArc::from("b"), + object: StdArc::from("o"), + version_id: None, + kind: MrfKind::PartialWrite, + enqueued_at_ms: 0, + attempts: 0, + }); + assert!(matches!(partial.heal_type, HealType::Object { .. })); + assert_eq!(partial.priority, HealPriority::Normal); + } +} diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index 3dd29b064..f1ec4cebe 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -158,6 +158,10 @@ pub async fn init_heal_manager_with_workload_provider( return Err(err); } + // Start the MRF intent consumer (error-path repair intents + durable + // journal replay) now that the manager can accept submissions. + heal::mrf_queue::spawn_mrf_consumer(heal_manager.clone()); + #[cfg(test)] test_hook_after_manager_start().await; diff --git a/crates/heal/tests/mrf_pipeline_test.rs b/crates/heal/tests/mrf_pipeline_test.rs new file mode 100644 index 000000000..36f07a5a7 --- /dev/null +++ b/crates/heal/tests/mrf_pipeline_test.rs @@ -0,0 +1,189 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! HS-01 (rustfs/backlog#1865): MRF intent pipeline integration tests. +//! +//! Drives the real consumer loop (`spawn_mrf_consumer`) against a real +//! 4-disk `ECStore` heal storage and a `HealManager` that has not started its +//! scheduler, so submitted intents stay observable in the admission queue. +//! Under `cargo nextest` each test runs in its own process, which keeps the +//! process-global MRF channel singleton safe. + +use rustfs_common::mrf_channel::{self, MrfKind}; +use rustfs_heal::heal::{ + manager::{HealConfig, HealManager}, + mrf_queue, + storage::{ECStoreHealStorage, HealStorageAPI}, +}; +use serial_test::serial; +use std::{path::Path, sync::Arc, time::Duration}; + +mod storage_api; + +use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks}; + +const META_BUCKET: &str = ".rustfs.sys"; +const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin"; + +async fn heal_env() -> (Vec, Arc) { + let env = rustfs_test_utils::TestECStoreEnv::builder() + .prefix("rustfs_heal_mrf_test") + .build() + .await; + let heal_storage: Arc = Arc::new(ECStoreHealStorage::new(env.ecstore.clone())); + (env.disk_paths, heal_storage) +} + +fn make_manager(storage: Arc) -> Arc { + Arc::new(HealManager::new( + storage, + Some(HealConfig { + // Keep the scheduler from draining the queue before assertions. + heal_interval: Duration::from_secs(3600), + enable_auto_heal: false, + ..Default::default() + }), + )) +} + +/// Encode one journal record independently of the implementation, so a format +/// drift between writer and this fixture fails loudly here. +fn journal_record(kind: u8, bucket: &str, object: &str, version: Option<[u8; 16]>, attempts: u8) -> Vec { + let mut body = vec![1u8, 1, kind, attempts]; + body.extend_from_slice(&1_700_000_000_000u64.to_le_bytes()); + match version { + Some(bytes) => { + body.push(1); + body.extend_from_slice(&bytes); + } + None => body.push(0), + } + body.extend_from_slice(&(bucket.len() as u32).to_le_bytes()); + body.extend_from_slice(&(object.len() as u32).to_le_bytes()); + body.extend_from_slice(bucket.as_bytes()); + body.extend_from_slice(object.as_bytes()); + let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + hasher.update(&body); + body.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes()); + body +} + +fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) { + for path in disk_paths { + let journal = path.join(META_BUCKET).join(JOURNAL_REL); + std::fs::create_dir_all(journal.parent().expect("journal parent")).expect("create journal dir"); + std::fs::write(&journal, data).expect("write journal fixture"); + } +} + +async fn wait_until(deadline: Duration, mut probe: F) -> bool +where + F: FnMut() -> Fut, + Fut: std::future::Future, +{ + let start = std::time::Instant::now(); + while start.elapsed() < deadline { + if probe().await { + return true; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + false +} + +/// A decode-failure intent delivered on the global channel must surface in the +/// heal manager as an Urgent request attributed to the MRF source. +#[tokio::test] +#[serial] +async fn decode_failure_intent_maps_to_urgent_mrf_heal_request() { + let (_disk_paths, storage) = heal_env().await; + let manager = make_manager(storage); + + mrf_queue::spawn_mrf_consumer(manager.clone()); + + assert!( + mrf_channel::try_send_mrf_intent(MrfKind::DecodeFailure, "mrf-bucket", "mrf-object", None), + "intent should be accepted while the consumer holds the channel" + ); + + let appeared = wait_until(Duration::from_secs(10), || async { + let snapshot = manager.operations_snapshot().await; + snapshot.queued_by_source.mrf >= 1 && snapshot.queued_by_priority.urgent >= 1 + }) + .await; + assert!( + appeared, + "MRF intent must reach the manager queue as an Urgent request (snapshot: {:?})", + manager.operations_snapshot().await + ); +} + +/// A journal left behind by a previous process must be replayed into the +/// manager queue and then removed, and a torn tail must not block replay of +/// the intact records. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn journal_replay_arms_intents_and_deletes_the_file() { + let (disk_paths, storage) = heal_env().await; + + // The journal reader resolves disks through the process-local disk map; + // register the environment's disks the same way server startup does. + let mut endpoints: Vec = disk_paths + .iter() + .map(|p| Endpoint::try_from(p.to_string_lossy().as_ref()).expect("endpoint from disk path")) + .collect(); + for (i, endpoint) in endpoints.iter_mut().enumerate() { + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(i); + } + let pool = PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: endpoints.len(), + endpoints: Endpoints::from(endpoints), + cmd_line: "mrf-test".to_string(), + platform: String::new(), + }; + init_local_disks(EndpointServerPools::from(vec![pool])) + .await + .expect("local disks should register"); + + let mut journal = journal_record(1, "replay-bucket", "replay-object", Some([9u8; 16]), 0); + journal.extend(journal_record(3, "replay-bucket", "partial-object", None, 1)); + // Torn tail: a third record truncated mid-way must not block the two + // intact records above. + journal.extend_from_slice(&journal_record(2, "replay-bucket", "metadata-object", None, 0)[..8]); + write_journal_to_disks(&disk_paths, &journal); + + let manager = make_manager(storage); + // Replay directly (not via the process-global channel consumer, which the + // sibling test already claimed in this process under plain `cargo test`). + let replayed = mrf_queue::replay_journal_once(&manager).await; + assert_eq!(replayed, 2, "the two intact records must be replayed"); + + let snapshot = manager.operations_snapshot().await; + assert_eq!(snapshot.queued_by_source.mrf, 2, "replayed intents must be attributed to the MRF source"); + + assert!( + disk_paths + .iter() + .all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()), + "the journal file must be removed after a successful replay" + ); + + let snapshot = manager.operations_snapshot().await; + assert_eq!(snapshot.queued_by_priority.urgent, 1, "the decode-failure record must replay as Urgent"); + assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal"); +} diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 15a01507c..030c55fc9 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -2478,6 +2478,15 @@ impl FolderScanner { } if let GetSizeFailureAction::HealMetadata { object } = failure_action { + // MRF journal intent: durable High-priority Metadata + // heal across restarts (HS-01); the scanner heal + // request below stays as the immediate path. + rustfs_common::mrf_channel::try_send_mrf_intent( + rustfs_common::mrf_channel::MrfKind::MetadataCorruption, + &item.bucket, + &object, + None, + ); self.send_required_scanner_heal_request( PendingScannerHealKind::Object, item.bucket.clone(), diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index d9e7d1f7e..52e9fc152 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -317,6 +317,7 @@ fn add_source_counts(total: &mut rustfs_heal::HealSourceCounts, next: rustfs_hea total.auto_heal = total.auto_heal.saturating_add(next.auto_heal); total.internal = total.internal.saturating_add(next.internal); total.read_repair = total.read_repair.saturating_add(next.read_repair); + total.mrf = total.mrf.saturating_add(next.mrf); } fn add_operations(total: &mut rustfs_heal::HealOperationsSnapshot, next: rustfs_heal::HealOperationsSnapshot) { @@ -2353,6 +2354,7 @@ mod tests { auto_heal: value, internal: value, read_repair: value, + mrf: value, }; let operations = |value| rustfs_heal::HealOperationsSnapshot { queue_length: value, diff --git a/rustfs/src/storage/rpc/node_service/heal.rs b/rustfs/src/storage/rpc/node_service/heal.rs index f9b990bf3..8f796726c 100644 --- a/rustfs/src/storage/rpc/node_service/heal.rs +++ b/rustfs/src/storage/rpc/node_service/heal.rs @@ -585,6 +585,7 @@ mod tests { let decoded = decode_node_heal_status(&encoded).expect("fixed v1 fixture should decode"); assert_eq!(decoded.info().bitrot_start_cycle, 9); assert_eq!(decoded.operations.queue_length, 2); + assert_eq!(decoded.operations.queued_by_source.mrf, 0); } #[test] From deb0edb7cc6b9d3d8bd6e8f6563fef4039d20b8e Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 12:45:42 +0800 Subject: [PATCH 17/22] chore: adjudicate 26 bare dead_code allows across five crates (#6187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove every bare `#[allow(dead_code)]` in io-core, object-capacity, targets, rio, and scanner. Each allow was stripped first and clippy was then asked which ones the compiler actually missed, so the verdicts rest on the diagnostic rather than on inspection. 23 were inert: they sat on `pub fn`s inside `pub mod`s, where `dead_code` does not apply, or on scanner integration-test helpers that the tests in the same file do call. The remaining 3 are in rio's private `compress_index` module and the code behind them is deleted rather than annotated. `remove_index_headers` is dead and also wrong — after skipping the 4-byte chunk header it matches against `S2_INDEX_TRAILER` where `S2_INDEX_HEADER` sits, so it returns `None` for every well-formed index; rio-v2 carries the correct equivalent that is actually in use. `restore_index_headers` is its unreachable counterpart, likewise duplicated live in rio-v2. `Index::reset` is a private method with no caller. Refs backlog#1823 --- crates/io-core/src/io_profile.rs | 6 --- .../object-capacity/src/capacity_manager.rs | 5 -- crates/rio/src/compress_index.rs | 50 ------------------- .../tests/lifecycle_integration_test.rs | 7 --- crates/targets/src/net.rs | 4 -- 5 files changed, 72 deletions(-) diff --git a/crates/io-core/src/io_profile.rs b/crates/io-core/src/io_profile.rs index 7618eb949..86cd79448 100644 --- a/crates/io-core/src/io_profile.rs +++ b/crates/io-core/src/io_profile.rs @@ -26,7 +26,6 @@ pub enum StorageMedia { } impl StorageMedia { - #[allow(dead_code)] pub fn as_str(&self) -> &'static str { match self { Self::Nvme => "nvme", @@ -60,7 +59,6 @@ pub enum AccessPattern { } impl AccessPattern { - #[allow(dead_code)] pub fn as_str(&self) -> &'static str { match self { Self::Sequential => "sequential", @@ -71,25 +69,21 @@ impl AccessPattern { } /// Check if this is a sequential access pattern. - #[allow(dead_code)] pub fn is_sequential(&self) -> bool { matches!(self, Self::Sequential) } /// Check if this is a random access pattern. - #[allow(dead_code)] pub fn is_random(&self) -> bool { matches!(self, Self::Random) } /// Check if this is a mixed access pattern. - #[allow(dead_code)] pub fn is_mixed(&self) -> bool { matches!(self, Self::Mixed) } /// Check if this pattern is unknown. - #[allow(dead_code)] pub fn is_unknown(&self) -> bool { matches!(self, Self::Unknown) } diff --git a/crates/object-capacity/src/capacity_manager.rs b/crates/object-capacity/src/capacity_manager.rs index 231d3c6a0..2d70f985c 100644 --- a/crates/object-capacity/src/capacity_manager.rs +++ b/crates/object-capacity/src/capacity_manager.rs @@ -427,7 +427,6 @@ pub enum DataSource { /// Write triggered WriteTriggered, /// Fallback value - #[allow(dead_code)] Fallback, } @@ -603,7 +602,6 @@ impl WriteRecord { /// Hybrid strategy configuration #[derive(Debug, Clone)] -#[allow(dead_code)] pub struct HybridStrategyConfig { /// Scheduled update interval pub scheduled_update_interval: Duration, @@ -998,14 +996,12 @@ impl HybridCapacityManager { } /// Get cache age - #[allow(dead_code)] pub async fn get_cache_age(&self) -> Option { let cache = self.cache.read().await; cache.as_ref().map(|c| c.last_update.elapsed()) } /// Get write frequency (writes/minute) - #[allow(dead_code)] pub async fn get_write_frequency(&self) -> usize { let record = &self.write_record; record.recent_write_count(record.monotonic_second()) @@ -1300,7 +1296,6 @@ pub fn get_capacity_manager() -> Arc { /// .update_capacity(CapacityUpdate::exact(1000, 0), DataSource::RealTime) /// .await; /// ``` -#[allow(dead_code)] pub fn create_isolated_manager(config: HybridStrategyConfig) -> Arc { Arc::new(HybridCapacityManager::new(config)) } diff --git a/crates/rio/src/compress_index.rs b/crates/rio/src/compress_index.rs index 75e415e26..c085d70c3 100644 --- a/crates/rio/src/compress_index.rs +++ b/crates/rio/src/compress_index.rs @@ -49,7 +49,6 @@ pub struct IndexInfo { pub uncompressed_offset: i64, } -#[allow(dead_code)] impl Index { pub fn new() -> Self { Self { @@ -60,14 +59,6 @@ impl Index { } } - #[allow(dead_code)] - fn reset(&mut self, max_block: usize) { - self.est_block_uncomp = max_block as i64; - self.total_compressed = -1; - self.total_uncompressed = -1; - self.info.clear(); - } - pub fn len(&self) -> usize { self.info.len() } @@ -511,47 +502,6 @@ fn read_varint(buf: &[u8]) -> io::Result<(i64, usize)> { Err(io::Error::new(io::ErrorKind::UnexpectedEof, "unexpected EOF")) } -// Helper functions for index header manipulation -#[allow(dead_code)] -pub fn remove_index_headers(b: &[u8]) -> Option<&[u8]> { - if b.len() < 4 + S2_INDEX_TRAILER.len() { - return None; - } - - // Skip size - let b = &b[4..]; - - // Check trailer - if !b.starts_with(S2_INDEX_TRAILER) { - return None; - } - - Some(&b[S2_INDEX_TRAILER.len()..]) -} - -#[allow(dead_code)] -pub fn restore_index_headers(in_data: &[u8]) -> Vec { - if in_data.is_empty() { - return Vec::new(); - } - - let mut b = Vec::with_capacity(4 + S2_INDEX_HEADER.len() + in_data.len() + S2_INDEX_TRAILER.len() + 4); - b.extend_from_slice(&[0x50, 0x2A, 0x4D, 0x18]); - b.extend_from_slice(S2_INDEX_HEADER); - b.extend_from_slice(in_data); - - let total_size = (b.len() + 4 + S2_INDEX_TRAILER.len()) as u32; - b.extend_from_slice(&total_size.to_le_bytes()); - b.extend_from_slice(S2_INDEX_TRAILER); - - let chunk_len = b.len() - 4; - b[1] = chunk_len as u8; - b[2] = (chunk_len >> 8) as u8; - b[3] = (chunk_len >> 16) as u8; - - b -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 7b2130316..71fe961d2 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -206,7 +206,6 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec, Arc, bucket_name: &str) { (**ecstore) .make_bucket(bucket_name, &Default::default()) @@ -251,7 +250,6 @@ async fn modeled_versioned_delete_opts(bucket: &str, object: &str) -> ObjectOpti } /// Test helper: Set bucket lifecycle configuration -#[allow(dead_code)] async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box> { // Create a simple lifecycle configuration XML with 0 days expiry for immediate testing let lifecycle_xml = r#" @@ -274,7 +272,6 @@ async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box Result<(), Box> { // Create lifecycle rule that targets delete-marker cleanup only. // Keep Expiration.Days unset to avoid expiring live transitioned object versions. @@ -297,7 +294,6 @@ async fn set_bucket_lifecycle_deletemarker(bucket_name: &str) -> Result<(), Box< Ok(()) } -#[allow(dead_code)] async fn set_bucket_lifecycle_delmarker_expiration(bucket_name: &str, days: i64) -> Result<(), Box> { let lifecycle_xml = format!( r#" @@ -320,7 +316,6 @@ async fn set_bucket_lifecycle_delmarker_expiration(bucket_name: &str, days: i64) Ok(()) } -#[allow(dead_code)] async fn set_bucket_lifecycle_transition_with_tier( bucket_name: &str, storage_class: &str, @@ -368,7 +363,6 @@ async fn object_exists(ecstore: &Arc, bucket: &str, object: &str) -> bo } /// Test helper: Check if object exists -#[allow(dead_code)] async fn object_is_delete_marker(ecstore: &Arc, bucket: &str, object: &str) -> bool { if let Ok(oi) = (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await { println!("oi: {oi:?}"); @@ -379,7 +373,6 @@ async fn object_is_delete_marker(ecstore: &Arc, bucket: &str, object: & } } -#[allow(dead_code)] async fn wait_for_object_absence(ecstore: &Arc, bucket: &str, object: &str, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; diff --git a/crates/targets/src/net.rs b/crates/targets/src/net.rs index 812e78772..6ed2a6e41 100644 --- a/crates/targets/src/net.rs +++ b/crates/targets/src/net.rs @@ -428,7 +428,6 @@ pub fn parse_url(s: &str) -> Result { Ok(ParsedURL(uu)) } -#[allow(dead_code)] pub fn parse_http_url(s: &str) -> Result { let u = parse_url(s)?; match u.0.scheme() { @@ -437,7 +436,6 @@ pub fn parse_http_url(s: &str) -> Result { } } -#[allow(dead_code)] pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> bool { if err.kind() == std::io::ErrorKind::TimedOut { return !expect_timeouts; @@ -449,12 +447,10 @@ pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> b || err_str.contains("use of closed network connection") } -#[allow(dead_code)] pub fn is_conn_reset_err(err: &std::io::Error) -> bool { err.to_string().contains("connection reset by peer") || matches!(err.raw_os_error(), Some(libc::ECONNRESET)) } -#[allow(dead_code)] pub fn is_conn_refused_err(err: &std::io::Error) -> bool { err.to_string().contains("connection refused") || matches!(err.raw_os_error(), Some(libc::ECONNREFUSED)) } From c7a29ec0a7890e4f8217687cf04fc9582ba9b1f4 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 12:51:44 +0800 Subject: [PATCH 18/22] chore(storage): drop dead backpressure and lock-optimizer wrappers (#6195) Neither rustfs/src/storage/backpressure.rs nor rustfs/src/storage/lock_optimizer.rs had a production caller: their only non-self references were the pub mod lines in storage/mod.rs and a cfg(test) module, so the object transfer path never applied this backpressure and never took these lock shortcuts. The six removed tests in concurrent_fix_test.rs duplicated tests that lived inside the deleted files; the shared primitives they shadowed keep their own coverage in rustfs-io-core. --- crates/concurrency/src/backpressure.rs | 5 +- docs/architecture/global-state-inventory.md | 1 - rustfs/src/storage/backpressure.rs | 618 -------------------- rustfs/src/storage/concurrent_fix_test.rs | 82 +-- rustfs/src/storage/lock_optimizer.rs | 458 --------------- rustfs/src/storage/mod.rs | 2 - 6 files changed, 4 insertions(+), 1162 deletions(-) delete mode 100644 rustfs/src/storage/backpressure.rs delete mode 100644 rustfs/src/storage/lock_optimizer.rs diff --git a/crates/concurrency/src/backpressure.rs b/crates/concurrency/src/backpressure.rs index 502d4fb0f..e692d7329 100644 --- a/crates/concurrency/src/backpressure.rs +++ b/crates/concurrency/src/backpressure.rs @@ -14,9 +14,8 @@ //! Shared backpressure policy type. //! -//! The runtime backpressure implementation (byte-watermark pipes and -//! monitors) lives in `rustfs/src/storage/backpressure.rs`; this module only -//! carries the watermark policy type that implementation shares. +//! This module only carries the watermark policy; the admission primitive it +//! projects into lives in `rustfs-io-core`. use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig; diff --git a/docs/architecture/global-state-inventory.md b/docs/architecture/global-state-inventory.md index 8407ad864..b110374dd 100644 --- a/docs/architecture/global-state-inventory.md +++ b/docs/architecture/global-state-inventory.md @@ -105,7 +105,6 @@ inventory. Generic function-local names such as `CACHE`, `LOCK`, `INIT`, and | `USE_STARSHARD_CACHE`, `BUCKET_CACHE_SMALL`, `BUCKET_CACHE_LARGE` | `rustfs/src/storage/ecfs_extend.rs` | Cache or constant / owner-local cache | Bucket validation cache backend selection and cache storage stay private to the ECFS extension owner. | | `GLOBAL_SSE_DEK_PROVIDER`, `SSE_TEST_LOCK` | `rustfs/src/storage/sse.rs` | Owner-local cache / test state | SSE DEK provider cache and test serialization lock stay private to the SSE owner. | | `AUTH_FS` | `rustfs/src/storage/access.rs` | Cache or constant / owner-local cache | Authorization tag-condition lookup keeps its filesystem helper private to the access owner. | -| `LOCK_STATS` | `rustfs/src/storage/lock_optimizer.rs` | Process-global owner-local metrics | Lock optimization statistics stay private behind lock optimizer helper APIs. | | `DEADLOCK_DETECTOR` | `rustfs/src/storage/deadlock_detector.rs` | Process-global owner-local state | Deadlock detector lifecycle state stays private to the storage deadlock detector owner. | | `CONCURRENCY_MANAGER`, `ACTIVE_GET_REQUESTS`, `ACTIVE_PUT_REQUESTS` | `rustfs/src/storage/concurrency/*` | Process-global owner-local scheduler state | Storage concurrency manager and request counters remain inside the storage concurrency owner boundary. | | `GET_OBJECT_BUFFER_THRESHOLD_WARNED`, `GET_READER_STREAM_BUFFER_SIZE_OVERRIDE`, function-local `ENABLED`, `OBJECT_SEEK_SUPPORT_THRESHOLD`, `OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS` | `rustfs/src/app/object_usecase.rs` | Cache or constant / owner-local cache | Object GET/seek tuning caches and warning guards stay private to object usecase helpers. | diff --git a/rustfs/src/storage/backpressure.rs b/rustfs/src/storage/backpressure.rs deleted file mode 100644 index 9585ba8a8..000000000 --- a/rustfs/src/storage/backpressure.rs +++ /dev/null @@ -1,618 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Backpressure Management for Object Data Transfer. -//! -//! This module provides backpressure-aware pipes for object data transfer, -//! preventing buffer overflow and memory exhaustion under high concurrency. - -//! # Key Features -//! -//! - Configurable buffer size with high/low watermarks -//! - Backpressure state monitoring and events -//! - Backpressure metrics emitted through the shared metrics pipeline -//! - Graceful handling of slow consumers -//! -//! # Architecture -//! -//! ```text -//! [Disk Reader] --> [BackpressurePipe] --> [HTTP Response] -//! | -//! v -//! [Buffer Monitor] -//! | -//! v -//! [High Watermark?] --> Apply Backpressure -//! ``` - -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::time::{Duration, Instant}; -use tokio::io::{DuplexStream, duplex}; -use tracing::{debug, warn}; - -use metrics::counter; -use rustfs_concurrency::PipeBackpressurePolicy; -use rustfs_io_core::BackpressureConfig as CoreBackpressureConfig; - -/// Object-transfer duplex pipe backpressure policy. -#[derive(Debug, Clone, Copy)] -pub struct ObjectPipeBackpressurePolicy { - /// Buffer size in bytes (default 4MB). - pub buffer_size: usize, - /// High watermark percentage (default 80%). - /// When buffer usage exceeds this, backpressure is applied. - pub high_watermark: u32, - /// Low watermark percentage (default 50%). - /// When buffer usage drops below this after high watermark, backpressure is released. - pub low_watermark: u32, -} - -impl Default for ObjectPipeBackpressurePolicy { - fn default() -> Self { - Self { - buffer_size: rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE, - high_watermark: rustfs_config::DEFAULT_OBJECT_BACKPRESSURE_HIGH_WATERMARK, - low_watermark: rustfs_config::DEFAULT_OBJECT_BACKPRESSURE_LOW_WATERMARK, - } - } -} - -impl ObjectPipeBackpressurePolicy { - /// Load configuration from environment variables. - pub fn from_env() -> Self { - let buffer_size = rustfs_utils::get_env_usize( - rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE, - rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE, - ); - let high_watermark = rustfs_utils::get_env_u32( - rustfs_config::ENV_OBJECT_BACKPRESSURE_HIGH_WATERMARK, - rustfs_config::DEFAULT_OBJECT_BACKPRESSURE_HIGH_WATERMARK, - ); - let low_watermark = rustfs_utils::get_env_u32( - rustfs_config::ENV_OBJECT_BACKPRESSURE_LOW_WATERMARK, - rustfs_config::DEFAULT_OBJECT_BACKPRESSURE_LOW_WATERMARK, - ); - - Self { - buffer_size, - high_watermark, - low_watermark, - } - } - - /// Calculate high watermark threshold in bytes. - pub fn high_watermark_bytes(&self) -> usize { - (self.buffer_size as u64 * self.high_watermark as u64 / 100) as usize - } - - /// Calculate low watermark threshold in bytes. - pub fn low_watermark_bytes(&self) -> usize { - (self.buffer_size as u64 * self.low_watermark as u64 / 100) as usize - } - - /// Project this object-transfer policy into the shared concurrency facade policy. - pub fn to_concurrency_policy(&self) -> PipeBackpressurePolicy { - PipeBackpressurePolicy { - buffer_size: self.buffer_size, - high_watermark: self.high_watermark, - low_watermark: self.low_watermark, - } - } - - /// Project this object-transfer policy into the reusable io-core admission config. - pub fn to_core_config(&self) -> CoreBackpressureConfig { - self.to_concurrency_policy().to_core_config() - } -} - -/// Backpressure state. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum BackpressureState { - /// Normal operation, buffer usage is below high watermark. - Normal, - /// Buffer usage is above high watermark, backpressure should be applied. - HighWatermark, - /// Backpressure is actively being applied to the producer. - BackpressureApplied, -} - -impl std::fmt::Display for BackpressureState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - BackpressureState::Normal => write!(f, "normal"), - BackpressureState::HighWatermark => write!(f, "high_watermark"), - BackpressureState::BackpressureApplied => write!(f, "backpressure_applied"), - } - } -} - -/// Compact metadata snapshot for object-transfer backpressure pipes. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct BackpressurePipeMeta { - /// Buffer capacity in bytes. - pub buffer_capacity: usize, - /// Current backpressure state. - pub state: BackpressureState, - /// Age of the pipe since creation. - pub age: Duration, -} - -/// Compact metadata snapshot for the lightweight backpressure monitor. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct BackpressureMonitorMeta { - /// Buffer capacity in bytes. - pub buffer_capacity: usize, - /// Current buffer usage percentage. - pub usage_percent: f32, - /// Current backpressure state. - pub state: BackpressureState, -} - -fn calculate_usage_percent(usage: usize, capacity: usize) -> f32 { - if capacity > 0 { - (usage as f32 / capacity as f32) * 100.0 - } else { - 0.0 - } -} - -fn apply_watermark_transition( - in_high_watermark: &AtomicBool, - usage: usize, - high: usize, - low: usize, -) -> (BackpressureState, bool) { - let current = in_high_watermark.load(Ordering::Acquire); - let next_state = if usage >= high { - BackpressureState::HighWatermark - } else if usage <= low { - BackpressureState::Normal - } else if current { - BackpressureState::HighWatermark - } else { - BackpressureState::Normal - }; - let next_is_high = matches!(next_state, BackpressureState::HighWatermark); - let changed = in_high_watermark.swap(next_is_high, Ordering::AcqRel) != next_is_high; - (next_state, changed) -} - -fn saturating_sub_atomic(value: &AtomicUsize, delta: usize) { - value - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_sub(delta))) - .ok(); -} - -/// A backpressure-aware pipe wrapping tokio's duplex. -/// -/// This provides monitoring and events for backpressure conditions -/// while maintaining compatibility with the standard duplex interface. -pub struct BackpressurePipe { - /// Reader end of the duplex pipe. - reader: DuplexStream, - /// Writer end of the duplex pipe. - writer: DuplexStream, - /// Configuration. - config: ObjectPipeBackpressurePolicy, - /// Current buffer usage (approximate, updated on write). - buffer_usage: AtomicUsize, - /// Current backpressure state. - state: AtomicBool, // true = in high watermark state - /// Total bytes written. - total_written: AtomicUsize, - /// Total bytes read. - total_read: AtomicUsize, - /// Cached high watermark threshold in bytes. - high_watermark_bytes: usize, - /// Cached low watermark threshold in bytes. - low_watermark_bytes: usize, - /// Pipe creation timestamp. - created_at: Instant, -} - -impl BackpressurePipe { - /// Create a new backpressure-aware pipe with default configuration. - pub fn new() -> Self { - Self::with_config(ObjectPipeBackpressurePolicy::from_env()) - } - - /// Create a new backpressure-aware pipe with custom configuration. - pub fn with_config(config: ObjectPipeBackpressurePolicy) -> Self { - let policy = config.to_concurrency_policy(); - let (reader, writer) = duplex(policy.buffer_size); - let high_watermark_bytes = policy.high_watermark_bytes(); - let low_watermark_bytes = policy.low_watermark_bytes(); - - debug!( - buffer_size = config.buffer_size, - high_watermark = config.high_watermark, - low_watermark = config.low_watermark, - high_watermark_bytes, - low_watermark_bytes, - "Created backpressure pipe" - ); - - Self { - reader, - writer, - config, - buffer_usage: AtomicUsize::new(0), - state: AtomicBool::new(false), - total_written: AtomicUsize::new(0), - total_read: AtomicUsize::new(0), - high_watermark_bytes, - low_watermark_bytes, - created_at: Instant::now(), - } - } - - /// Take the reader end of the pipe (consumes self). - pub fn into_reader(self) -> DuplexStream { - self.reader - } - - /// Take the writer end of the pipe (consumes self). - pub fn into_writer(self) -> DuplexStream { - self.writer - } - - /// Split into reader and writer (consumes self). - pub fn split(self) -> (DuplexStream, DuplexStream) { - (self.reader, self.writer) - } - - /// Get current backpressure state. - pub fn state(&self) -> BackpressureState { - if self.state.load(Ordering::Acquire) { - BackpressureState::BackpressureApplied - } else { - BackpressureState::Normal - } - } - - /// Get a compact metadata snapshot for the pipe. - pub fn meta(&self) -> BackpressurePipeMeta { - BackpressurePipeMeta { - buffer_capacity: self.config.buffer_size, - state: self.state(), - age: self.age(), - } - } - - /// Get the age of this pipe. - pub fn age(&self) -> Duration { - self.created_at.elapsed() - } - - /// Get current buffer usage. - pub fn usage(&self) -> usize { - self.buffer_usage.load(Ordering::Acquire) - } - - /// Record bytes written (call after successful write). - pub fn record_write(&self, bytes: usize) { - self.total_written.fetch_add(bytes, Ordering::Relaxed); - self.buffer_usage.fetch_add(bytes, Ordering::Release); - self.update_watermark_state(); - } - - /// Record bytes read (call after successful read). - pub fn record_read(&self, bytes: usize) { - self.total_read.fetch_add(bytes, Ordering::Relaxed); - saturating_sub_atomic(&self.buffer_usage, bytes); - self.update_watermark_state(); - } - - /// Update watermark state and emit transition signals. - fn update_watermark_state(&self) { - let usage = self.buffer_usage.load(Ordering::Acquire); - let usage_percent = calculate_usage_percent(usage, self.config.buffer_size) as u32; - let (next_state, changed) = - apply_watermark_transition(&self.state, usage, self.high_watermark_bytes, self.low_watermark_bytes); - - if changed { - match next_state { - BackpressureState::HighWatermark => { - counter!("rustfs_backpressure_events_total", "state" => "high_watermark").increment(1); - - warn!( - buffer_usage = usage, - buffer_capacity = self.config.buffer_size, - usage_percent, - high_watermark = self.config.high_watermark, - "Backpressure: high watermark reached" - ); - } - BackpressureState::Normal => { - counter!("rustfs_backpressure_events_total", "state" => "normal").increment(1); - - debug!( - buffer_usage = usage, - buffer_capacity = self.config.buffer_size, - usage_percent, - low_watermark = self.config.low_watermark, - "Backpressure: returned to normal" - ); - } - BackpressureState::BackpressureApplied => {} - } - } - } - - /// Get total bytes written. - pub fn total_written(&self) -> usize { - self.total_written.load(Ordering::Relaxed) - } - - /// Get total bytes read. - pub fn total_read(&self) -> usize { - self.total_read.load(Ordering::Relaxed) - } - - /// Get buffer capacity. - pub fn capacity(&self) -> usize { - self.config.buffer_size - } -} - -impl Default for BackpressurePipe { - fn default() -> Self { - Self::new() - } -} - -/// A simple wrapper that provides backpressure monitoring for duplex streams. -/// -/// This is a lighter-weight alternative to `BackpressurePipe` that doesn't -/// wrap the streams but provides monitoring capabilities. -pub struct BackpressureMonitor { - /// Configuration. - config: ObjectPipeBackpressurePolicy, - /// Current buffer usage. - buffer_usage: AtomicUsize, - /// In high watermark state. - in_high_watermark: AtomicBool, - /// Cached high watermark threshold in bytes. - high_watermark_bytes: usize, - /// Cached low watermark threshold in bytes. - low_watermark_bytes: usize, -} - -impl BackpressureMonitor { - /// Create a new monitor with default configuration. - pub fn new() -> Self { - Self::with_config(ObjectPipeBackpressurePolicy::from_env()) - } - - /// Create a new monitor with custom configuration. - pub fn with_config(config: ObjectPipeBackpressurePolicy) -> Self { - let policy = config.to_concurrency_policy(); - let high_watermark_bytes = policy.high_watermark_bytes(); - let low_watermark_bytes = policy.low_watermark_bytes(); - Self { - config, - buffer_usage: AtomicUsize::new(0), - in_high_watermark: AtomicBool::new(false), - high_watermark_bytes, - low_watermark_bytes, - } - } - - /// Record bytes added to buffer. - pub fn on_write(&self, bytes: usize) -> BackpressureState { - self.buffer_usage.fetch_add(bytes, Ordering::Release); - self.update_state() - } - - /// Record bytes removed from buffer. - pub fn on_read(&self, bytes: usize) -> BackpressureState { - saturating_sub_atomic(&self.buffer_usage, bytes); - self.update_state() - } - - /// Get current state. - pub fn state(&self) -> BackpressureState { - if self.in_high_watermark.load(Ordering::Acquire) { - BackpressureState::HighWatermark - } else { - BackpressureState::Normal - } - } - - /// Get current buffer usage. - pub fn usage(&self) -> usize { - self.buffer_usage.load(Ordering::Acquire) - } - - /// Get usage percentage. - pub fn usage_percent(&self) -> f32 { - let usage = self.buffer_usage.load(Ordering::Acquire); - calculate_usage_percent(usage, self.config.buffer_size) - } - - /// Get a compact metadata snapshot for the monitor. - pub fn meta(&self) -> BackpressureMonitorMeta { - let usage = self.buffer_usage.load(Ordering::Acquire); - BackpressureMonitorMeta { - buffer_capacity: self.config.buffer_size, - usage_percent: calculate_usage_percent(usage, self.config.buffer_size), - state: self.state(), - } - } - - /// Update state based on current usage. - fn update_state(&self) -> BackpressureState { - let usage = self.buffer_usage.load(Ordering::Acquire); - let usage_percent = calculate_usage_percent(usage, self.config.buffer_size) as u32; - let (next_state, changed) = - apply_watermark_transition(&self.in_high_watermark, usage, self.high_watermark_bytes, self.low_watermark_bytes); - - if matches!(next_state, BackpressureState::HighWatermark) { - if changed { - counter!("rustfs_backpressure_events_total", "state" => "high_watermark").increment(1); - - debug!(usage_percent, "Backpressure: entered high watermark"); - } - BackpressureState::HighWatermark - } else { - if changed { - counter!("rustfs_backpressure_events_total", "state" => "normal").increment(1); - - debug!(usage_percent, "Backpressure: returned to normal"); - } - BackpressureState::Normal - } - } -} - -impl Default for BackpressureMonitor { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -#[allow(unused_imports)] -mod tests { - use super::{BackpressureMonitor, BackpressurePipe, BackpressureState, ObjectPipeBackpressurePolicy}; - - #[test] - fn test_backpressure_config_default() { - let config = ObjectPipeBackpressurePolicy::default(); - assert_eq!(config.buffer_size, 4 * 1024 * 1024); - assert_eq!(config.high_watermark, 80); - assert_eq!(config.low_watermark, 50); - } - - #[test] - fn test_backpressure_config_watermarks() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 1000, - high_watermark: 80, - low_watermark: 50, - }; - assert_eq!(config.high_watermark_bytes(), 800); - assert_eq!(config.low_watermark_bytes(), 500); - } - - #[test] - fn test_backpressure_policy_projects_to_concurrency_and_core_config() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 2000, - high_watermark: 75, - low_watermark: 40, - }; - let concurrency = config.to_concurrency_policy(); - let core = config.to_core_config(); - - assert_eq!(concurrency.buffer_size, config.buffer_size); - assert_eq!(concurrency.high_watermark, config.high_watermark); - assert_eq!(concurrency.low_watermark, config.low_watermark); - assert_eq!(core.high_water_mark, 0.75); - assert_eq!(core.low_water_mark, 0.40); - assert!(core.enabled); - } - - #[test] - fn test_backpressure_pipe_consumes_concurrency_policy_thresholds() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 2000, - high_watermark: 75, - low_watermark: 40, - }; - let concurrency = config.to_concurrency_policy(); - let pipe = BackpressurePipe::with_config(config); - - assert_eq!(pipe.capacity(), concurrency.buffer_size); - assert_eq!(pipe.high_watermark_bytes, concurrency.high_watermark_bytes()); - assert_eq!(pipe.low_watermark_bytes, concurrency.low_watermark_bytes()); - } - - #[test] - fn test_backpressure_monitor_consumes_concurrency_policy_thresholds() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 2000, - high_watermark: 75, - low_watermark: 40, - }; - let concurrency = config.to_concurrency_policy(); - let monitor = BackpressureMonitor::with_config(config); - - assert_eq!(monitor.meta().buffer_capacity, concurrency.buffer_size); - assert_eq!(monitor.high_watermark_bytes, concurrency.high_watermark_bytes()); - assert_eq!(monitor.low_watermark_bytes, concurrency.low_watermark_bytes()); - } - - #[test] - fn test_backpressure_state_display() { - assert_eq!(format!("{}", BackpressureState::Normal), "normal"); - assert_eq!(format!("{}", BackpressureState::HighWatermark), "high_watermark"); - assert_eq!(format!("{}", BackpressureState::BackpressureApplied), "backpressure_applied"); - } - - #[test] - fn test_backpressure_monitor() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 1000, - high_watermark: 80, - low_watermark: 50, - }; - let monitor = BackpressureMonitor::with_config(config); - - // Initially normal - assert_eq!(monitor.state(), BackpressureState::Normal); - assert_eq!(monitor.meta().buffer_capacity, 1000); - assert_eq!(monitor.meta().usage_percent, 0.0); - - // Write to reach high watermark - let state = monitor.on_write(850); - assert_eq!(state, BackpressureState::HighWatermark); - assert_eq!(monitor.meta().usage_percent, 85.0); - - // Read to go below low watermark - let state = monitor.on_read(400); - assert_eq!(state, BackpressureState::Normal); - assert_eq!(monitor.meta().usage_percent, 45.0); - } - - #[tokio::test] - async fn test_backpressure_pipe_creation() { - let pipe = BackpressurePipe::new(); - assert_eq!(pipe.capacity(), 4 * 1024 * 1024); - assert_eq!(pipe.state(), BackpressureState::Normal); - assert_eq!(pipe.meta().buffer_capacity, 4 * 1024 * 1024); - assert!(pipe.meta().age <= pipe.age()); - } - - #[test] - fn test_backpressure_pipe_state_transitions() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 1000, - high_watermark: 80, - low_watermark: 50, - }; - let pipe = BackpressurePipe::with_config(config); - - assert_eq!(pipe.state(), BackpressureState::Normal); - assert_eq!(pipe.meta().state, BackpressureState::Normal); - - pipe.record_write(850); - assert_eq!(pipe.state(), BackpressureState::BackpressureApplied); - assert_eq!(pipe.meta().state, BackpressureState::BackpressureApplied); - - pipe.record_read(400); - assert_eq!(pipe.state(), BackpressureState::Normal); - assert_eq!(pipe.meta().state, BackpressureState::Normal); - } -} diff --git a/rustfs/src/storage/concurrent_fix_test.rs b/rustfs/src/storage/concurrent_fix_test.rs index 2fedeced1..a80865976 100644 --- a/rustfs/src/storage/concurrent_fix_test.rs +++ b/rustfs/src/storage/concurrent_fix_test.rs @@ -14,17 +14,15 @@ //! Integration tests for concurrent request fix. //! -//! These tests verify that the timeout, backpressure, and deadlock detection -//! mechanisms work correctly under high concurrency scenarios. +//! These tests verify that the timeout and deadlock detection mechanisms work +//! correctly under high concurrency scenarios. #[cfg(test)] mod tests { - use crate::storage::backpressure::{BackpressureMonitor, BackpressureState, ObjectPipeBackpressurePolicy}; use crate::storage::concurrency::{IoLoadLevel, IoPriority}; use crate::storage::deadlock_detector::{ DeadlockDetector, LockInfo, LockType, RequestHangDetectionPolicy, RequestResourceTracker, }; - use crate::storage::lock_optimizer::{LockOptimizeConfig, LockOptimizer, LockStats}; use crate::storage::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper, TimedGetObjectResult}; use std::time::Duration; @@ -114,82 +112,6 @@ mod tests { } } - // ============================================ - // Backpressure Tests - // ============================================ - - #[test] - fn test_backpressure_config_defaults() { - let config = ObjectPipeBackpressurePolicy::default(); - assert_eq!(config.buffer_size, 4 * 1024 * 1024); // 4MB - assert_eq!(config.high_watermark, 80); - assert_eq!(config.low_watermark, 50); - } - - #[test] - fn test_backpressure_monitor_state_transitions() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 1000, - high_watermark: 80, - low_watermark: 50, - }; - let monitor = BackpressureMonitor::with_config(config); - - // Initially normal - assert_eq!(monitor.state(), BackpressureState::Normal); - - // Write to reach high watermark - let state = monitor.on_write(850); - assert_eq!(state, BackpressureState::HighWatermark); - - // Read to go below low watermark - let state = monitor.on_read(400); - assert_eq!(state, BackpressureState::Normal); - } - - #[test] - fn test_backpressure_usage_percent() { - let config = ObjectPipeBackpressurePolicy { - buffer_size: 1000, - high_watermark: 80, - low_watermark: 50, - }; - let monitor = BackpressureMonitor::with_config(config); - - monitor.on_write(500); - assert!((monitor.usage_percent() - 50.0).abs() < 1.0); - } - - // ============================================ - // Lock Optimizer Tests - // ============================================ - - #[test] - fn test_lock_optimize_config_defaults() { - let config = LockOptimizeConfig::default(); - assert!(config.enabled); - assert_eq!(config.acquire_timeout, Duration::from_secs(5)); - } - - #[test] - fn test_lock_stats_tracking() { - let stats = LockStats::new(); - - stats.record_acquire(); - stats.record_early_release(Duration::from_millis(100)); - stats.record_early_release(Duration::from_millis(200)); - - assert_eq!(stats.locks_acquired.load(std::sync::atomic::Ordering::Relaxed), 1); - assert_eq!(stats.locks_released_early.load(std::sync::atomic::Ordering::Relaxed), 2); - assert_eq!(stats.max_hold_time(), Duration::from_millis(200)); - } - - #[test] - fn test_lock_optimizer_creation() { - let optimizer = LockOptimizer::new(); - assert!(optimizer.is_enabled()); - } - // ============================================ // I/O Priority Tests // ============================================ diff --git a/rustfs/src/storage/lock_optimizer.rs b/rustfs/src/storage/lock_optimizer.rs deleted file mode 100644 index 3636c50c3..000000000 --- a/rustfs/src/storage/lock_optimizer.rs +++ /dev/null @@ -1,458 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Lock Optimization for GetObject Operations. -//! -//! This module provides optimized lock management for read operations, -//! reducing lock contention by releasing locks early (after metadata read) -//! rather than holding them for the entire data transfer duration. -//! -//! # Migration Note -//! -//! For new code, consider using `rustfs_io_core::LockOptimizer` which provides -//! the same core functionality with better separation of concerns. This module -//! remains for backward compatibility and storage-specific configuration. -//! -//! ```ignore -//! // Recommended: Use io-core directly -//! use rustfs_io_core::LockOptimizer; -//! let optimizer = LockOptimizer::with_defaults(); -//! ``` - -// Allow dead_code for public API that may be used by external modules or future features -//! # Key Features -//! -//! - Early lock release after metadata read -//! - Lock hold time monitoring -//! - Configurable optimization (can be disabled for debugging) -//! - Lock contention metrics emitted through the shared metrics pipeline -//! -//! # Architecture -//! -//! ```text -//! Traditional: [Acquire Lock] --> [Read Metadata] --> [Transfer Data] --> [Release Lock] -//! |<------------------ Lock Held ------------------>| -//! -//! Optimized: [Acquire Lock] --> [Read Metadata] --> [Release Lock] --> [Transfer Data] -//! |<- Lock Held ->| -//! ``` - -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant}; -use tracing::debug; - -use metrics::histogram; - -/// Lock optimization configuration. -#[derive(Debug, Clone)] -pub struct LockOptimizeConfig { - /// Whether to enable lock optimization. - /// When enabled, read locks are released after metadata read. - /// When disabled, locks are held for the entire operation (traditional behavior). - pub enabled: bool, - /// Lock acquisition timeout. - pub acquire_timeout: Duration, -} - -impl Default for LockOptimizeConfig { - fn default() -> Self { - Self { - enabled: rustfs_config::DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE, - acquire_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT), - } - } -} - -impl LockOptimizeConfig { - /// Load configuration from environment variables. - pub fn from_env() -> Self { - let enabled = rustfs_utils::get_env_bool( - rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, - rustfs_config::DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE, - ); - let acquire_timeout = Duration::from_secs(rustfs_utils::get_env_u64( - rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, - rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT, - )); - - Self { - enabled, - acquire_timeout, - } - } -} - -/// Statistics for lock optimization monitoring. -#[derive(Debug, Default)] -pub struct LockStats { - /// Total locks acquired. - pub locks_acquired: AtomicU64, - /// Total locks released early. - pub locks_released_early: AtomicU64, - /// Total lock hold time in microseconds. - pub total_hold_time_us: AtomicU64, - /// Maximum lock hold time in microseconds. - pub max_hold_time_us: AtomicU64, -} - -impl LockStats { - /// Create new lock statistics. - pub fn new() -> Self { - Self::default() - } - - /// Record a lock acquisition. - pub fn record_acquire(&self) { - self.locks_acquired.fetch_add(1, Ordering::Relaxed); - } - - /// Record an early lock release. - pub fn record_early_release(&self, hold_time: Duration) { - self.locks_released_early.fetch_add(1, Ordering::Relaxed); - self.record_hold_time(hold_time); - } - - /// Record lock hold time. - fn record_hold_time(&self, hold_time: Duration) { - let hold_time_us = hold_time.as_micros() as u64; - self.total_hold_time_us.fetch_add(hold_time_us, Ordering::Relaxed); - - // Update max hold time - let mut current_max = self.max_hold_time_us.load(Ordering::Relaxed); - while hold_time_us > current_max { - match self - .max_hold_time_us - .compare_exchange_weak(current_max, hold_time_us, Ordering::Relaxed, Ordering::Relaxed) - { - Ok(_) => break, - Err(actual) => current_max = actual, - } - } - } - - /// Get average hold time. - pub fn avg_hold_time(&self) -> Duration { - let total = self.total_hold_time_us.load(Ordering::Relaxed); - let count = self.locks_released_early.load(Ordering::Relaxed); - total.checked_div(count).map(Duration::from_micros).unwrap_or(Duration::ZERO) - } - - /// Get maximum hold time. - pub fn max_hold_time(&self) -> Duration { - Duration::from_micros(self.max_hold_time_us.load(Ordering::Relaxed)) - } -} - -/// Global lock statistics. -static LOCK_STATS: std::sync::OnceLock> = std::sync::OnceLock::new(); - -/// Get global lock statistics. -pub fn get_lock_stats() -> Arc { - LOCK_STATS.get_or_init(|| Arc::new(LockStats::new())).clone() -} - -/// An optimized lock guard that supports early release. -/// -/// This wraps the actual lock guard and provides: -/// - Early release capability (before drop) -/// - Hold time tracking -/// - Metrics reporting -pub struct OptimizedLockGuard { - /// The underlying lock guard. - guard: Option, - /// When the lock was acquired. - acquire_time: Instant, - /// Whether the lock has been released. - released: bool, - /// Lock resource name (for logging). - resource: String, - /// Statistics reference. - stats: Arc, -} - -impl OptimizedLockGuard { - /// Create a new optimized lock guard. - pub fn new(guard: G, resource: impl Into) -> Self { - let stats = get_lock_stats(); - stats.record_acquire(); - - Self { - guard: Some(guard), - acquire_time: Instant::now(), - released: false, - resource: resource.into(), - stats, - } - } - - /// Get the lock hold time so far. - pub fn hold_time(&self) -> Duration { - self.acquire_time.elapsed() - } - - /// Check if the lock has been released. - pub fn is_released(&self) -> bool { - self.released - } - - /// Release the lock early (before drop). - /// - /// This is the key optimization: releasing the lock after - /// metadata read rather than waiting for the entire operation. - pub fn early_release(&mut self) { - if self.released { - return; - } - - let hold_time = self.hold_time(); - self.guard.take(); - self.released = true; - - self.stats.record_early_release(hold_time); - - histogram!("rustfs_lock_hold_duration_seconds").record(hold_time.as_secs_f64()); - - debug!( - resource = %self.resource, - hold_time_ms = hold_time.as_millis(), - "Lock released early (optimization active)" - ); - } - - /// Get a reference to the underlying guard. - pub fn as_ref(&self) -> Option<&G> { - if self.released { None } else { self.guard.as_ref() } - } -} - -impl Drop for OptimizedLockGuard { - fn drop(&mut self) { - if !self.released { - let hold_time = self.hold_time(); - self.guard.take(); - self.released = true; - - self.stats.record_early_release(hold_time); - - histogram!("rustfs_lock_hold_duration_seconds").record(hold_time.as_secs_f64()); - - debug!( - resource = %self.resource, - hold_time_ms = hold_time.as_millis(), - "Lock released on drop (normal release)" - ); - } - } -} - -/// A scope guard that releases a lock when it goes out of scope. -/// -/// This is a simpler version of OptimizedLockGuard for cases -/// where we just need RAII semantics without tracking. -pub struct LockScopeGuard { - guard: Option, -} - -impl LockScopeGuard { - /// Create a new scope guard. - pub fn new(guard: G) -> Self { - Self { guard: Some(guard) } - } - - /// Release the lock early. - pub fn release(&mut self) { - self.guard.take(); - } -} - -impl Drop for LockScopeGuard { - fn drop(&mut self) { - self.guard.take(); - } -} - -/// Helper for managing lock optimization in GetObject operations. -/// -/// This provides a clean interface for the common pattern: -/// 1. Acquire lock -/// 2. Read metadata -/// 3. Release lock (if optimization enabled) -/// 4. Transfer data (without lock) -pub struct LockOptimizer { - /// Configuration. - config: LockOptimizeConfig, -} - -impl LockOptimizer { - /// Create a new lock optimizer with default configuration. - pub fn new() -> Self { - Self { - config: LockOptimizeConfig::from_env(), - } - } - - /// Create a new lock optimizer with custom configuration. - pub fn with_config(config: LockOptimizeConfig) -> Self { - Self { config } - } - - /// Check if lock optimization is enabled. - pub fn is_enabled(&self) -> bool { - self.config.enabled - } - - /// Get the lock acquisition timeout. - pub fn acquire_timeout(&self) -> Duration { - self.config.acquire_timeout - } - - /// Wrap a lock guard for optimization. - pub fn wrap_guard(&self, guard: G, resource: impl Into) -> OptimizedLockGuard { - OptimizedLockGuard::new(guard, resource) - } - - /// Execute a metadata read operation with lock optimization. - /// - /// This is the main entry point for optimized lock usage: - /// - If optimization is enabled: lock is released after metadata_fn completes - /// - If optimization is disabled: lock is held until the returned guard is dropped - /// - /// # Arguments - /// - /// * `guard` - The lock guard to optimize - /// * `resource` - Resource name for logging - /// * `metadata_fn` - Function to read metadata while holding lock - /// - /// # Returns - /// - /// A tuple of (metadata result, optional guard to hold for later release) - pub async fn with_optimized_lock( - &self, - guard: G, - resource: impl Into, - metadata_fn: F, - ) -> (T, Option>) - where - F: FnOnce() -> Fut, - Fut: std::future::Future, - { - let resource = resource.into(); - let mut optimized = OptimizedLockGuard::new(guard, &resource); - - // Execute metadata read while holding lock - let result = metadata_fn().await; - - if self.config.enabled { - // Release lock early - optimized.early_release(); - (result, None) - } else { - // Keep lock for caller to release - (result, Some(optimized)) - } - } -} - -impl Default for LockOptimizer { - fn default() -> Self { - Self::new() - } -} - -/// Check if lock optimization is enabled globally. -pub fn is_lock_optimization_enabled() -> bool { - rustfs_utils::get_env_bool( - rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, - rustfs_config::DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE, - ) -} - -#[cfg(test)] -#[allow(unused_imports)] -mod tests { - use super::{LockOptimizeConfig, LockOptimizer, LockStats, OptimizedLockGuard}; - use std::sync::Mutex; - use std::sync::atomic::Ordering; - use std::time::Duration; - - #[test] - fn test_lock_optimize_config_default() { - let config = LockOptimizeConfig::default(); - assert!(config.enabled); - assert_eq!(config.acquire_timeout, Duration::from_secs(5)); - } - - #[test] - fn test_lock_stats() { - let stats = LockStats::new(); - - stats.record_acquire(); - stats.record_early_release(Duration::from_millis(100)); - stats.record_early_release(Duration::from_millis(200)); - - assert_eq!(stats.locks_acquired.load(Ordering::Relaxed), 1); - assert_eq!(stats.locks_released_early.load(Ordering::Relaxed), 2); - assert_eq!(stats.max_hold_time(), Duration::from_millis(200)); - } - - #[test] - fn test_optimized_lock_guard() { - let guard = Mutex::new(42); - let locked = guard.lock().unwrap(); - let mut optimized = OptimizedLockGuard::new(locked, "test-resource"); - - assert!(!optimized.is_released()); - assert!(optimized.hold_time() < Duration::from_secs(1)); - - optimized.early_release(); - assert!(optimized.is_released()); - } - - #[test] - fn test_lock_optimizer() { - let optimizer = LockOptimizer::new(); - assert!(optimizer.is_enabled()); - } - - #[tokio::test] - async fn test_with_optimized_lock_enabled() { - let optimizer = LockOptimizer::new(); - let guard = Mutex::new(42); - let locked = guard.lock().unwrap(); - - let (result, returned_guard) = optimizer.with_optimized_lock(locked, "test-resource", || async { 100 }).await; - - assert_eq!(result, 100); - // With optimization enabled, guard should be None (released early) - assert!(returned_guard.is_none()); - } - - #[tokio::test] - async fn test_with_optimized_lock_disabled() { - let config = LockOptimizeConfig { - enabled: false, - acquire_timeout: Duration::from_secs(5), - }; - let optimizer = LockOptimizer::with_config(config); - let guard = Mutex::new(42); - let locked = guard.lock().unwrap(); - - let (result, returned_guard) = optimizer.with_optimized_lock(locked, "test-resource", || async { 100 }).await; - - assert_eq!(result, 100); - // With optimization disabled, guard should be Some (held for later) - assert!(returned_guard.is_some()); - } -} diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 58b6a0ce9..e0817da90 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -13,12 +13,10 @@ // limitations under the License. pub mod access; -pub mod backpressure; pub mod concurrency; pub mod deadlock_detector; pub mod ecfs; pub(crate) mod helper; -pub mod lock_optimizer; pub mod options; pub mod request_context; pub mod rpc; From 51497cb5335b5f02bdf16355bb413bad182da757 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 12:57:44 +0800 Subject: [PATCH 19/22] fix(ecstore): give peer REST failures op and bucket context (#6200) --- .../src/cluster/rpc/peer_rest_client.rs | 196 +++++++++++++++--- .../ecstore/src/cluster/rpc/peer_s3_client.rs | 2 + 2 files changed, 166 insertions(+), 32 deletions(-) diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 1426fc91f..3bac43266 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -86,6 +86,25 @@ const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30); const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024; const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024; +/// Error for a peer that reported `success = false` without an `error_info` payload. +/// +/// Same shape as `peer_s3_client::peer_failure_without_details`, over `StorageError` +/// instead of `DiskError`. The message names the operation (and the bucket, where the +/// operation has one) and nothing else, for two reasons: +/// +/// - `finalize_result` classifies failures by message substring, so any text matching +/// `message_has_network_needle` would take an answering peer offline and evict its +/// connection over a plain application-level rejection. +/// - Quorum aggregation (`reduce_errs`) buckets `Io` errors by kind plus rendered +/// message, so a per-peer detail such as the peer address would split one shared +/// failure into single-count buckets and downgrade the dominant error. +fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error { + match bucket { + Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")), + None => Error::other(format!("{op}: peer returned failure without error details")), + } +} + fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result { if !response.success { return Err(Error::other( @@ -696,7 +715,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("local_storage_info", None)); } let data = response.storage_info; @@ -719,7 +738,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("server_info", None)); } let data = response.server_properties; @@ -742,7 +761,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_cpus", None)); } let data = response.cpus; @@ -765,7 +784,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_net_info", None)); } let data = response.net_info; @@ -788,7 +807,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_partitions", None)); } let data = response.partitions; @@ -811,7 +830,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_os_info", None)); } let data = response.os_info; @@ -832,7 +851,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_se_linux_info", None)); } let data = response.sys_services; @@ -857,7 +876,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_sys_config", None)); } let data = response.sys_config; @@ -882,7 +901,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_sys_errors", None)); } let data = response.sys_errors; @@ -907,7 +926,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_mem_info", None)); } let data = response.mem_info; @@ -939,7 +958,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_metrics", None)); } let data = response.realtime_metrics; @@ -964,7 +983,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_live_events", None)); } Ok(PeerLiveEventsBatch { @@ -989,7 +1008,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("get_proc_info", None)); } let data = response.proc_info; @@ -1016,7 +1035,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("start_profiling", None)); } Ok(()) } @@ -1323,7 +1342,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket))); } Ok(()) } @@ -1346,7 +1365,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("delete_bucket_metadata", Some(bucket))); } Ok(()) } @@ -1369,7 +1388,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("delete_policy", None)); } Ok(()) } @@ -1392,7 +1411,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_policy", None)); } Ok(()) } @@ -1417,7 +1436,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_policy_mapping", None)); } Ok(()) } @@ -1440,7 +1459,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("delete_user", None)); } Ok(()) } @@ -1463,7 +1482,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("delete_service_account", None)); } Ok(()) } @@ -1487,7 +1506,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_user", None)); } Ok(()) } @@ -1510,7 +1529,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_service_account", None)); } Ok(()) } @@ -1533,7 +1552,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_group", None)); } Ok(()) } @@ -1554,7 +1573,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("reload_site_replication_config", None)); } Ok(()) } @@ -1597,7 +1616,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("signal_service", None)); } validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?; Ok(response) @@ -1667,7 +1686,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("reload_pool_meta", None)); } Ok(()) @@ -1691,7 +1710,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("stop_rebalance", None)); } Ok(()) @@ -1725,7 +1744,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("load_rebalance_meta", None)); } Ok(()) @@ -1753,7 +1772,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("start_decommission", None)); } Ok(()) @@ -1777,7 +1796,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("decommission_cancel", None)); } Ok(()) @@ -1801,7 +1820,7 @@ impl PeerRestClient { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } - return Err(Error::other("")); + return Err(peer_failure_without_details("clear_decommission", None)); } Ok(()) @@ -1947,6 +1966,8 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO mod tests { use super::*; use crate::config::com::STORAGE_CLASS_SUB_SYS; + use crate::disk::error::DiskError; + use crate::disk::error_reduce::reduce_errs; use crate::layout::{disks_layout::DisksLayout, endpoints::SetupType}; use rustfs_config::{ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_STARTUP_TOPOLOGY_WAIT_MODE}; use serde_json::Value; @@ -3098,4 +3119,115 @@ mod tests { && span.get("request_id").and_then(Value::as_str) == Some("req-peer-rest") })); } + + /// Every operation name passed to `peer_failure_without_details` in this file. + const PEER_FAILURE_OPS: &[&str] = &[ + "local_storage_info", + "server_info", + "get_cpus", + "get_net_info", + "get_partitions", + "get_os_info", + "get_se_linux_info", + "get_sys_config", + "get_sys_errors", + "get_mem_info", + "get_metrics", + "get_live_events", + "get_proc_info", + "start_profiling", + "load_bucket_metadata", + "delete_bucket_metadata", + "delete_policy", + "load_policy", + "load_policy_mapping", + "delete_user", + "delete_service_account", + "load_user", + "load_service_account", + "load_group", + "reload_site_replication_config", + "signal_service", + "reload_pool_meta", + "stop_rebalance", + "load_rebalance_meta", + "start_decommission", + "decommission_cancel", + "clear_decommission", + ]; + + #[test] + fn peer_failure_without_details_names_operation_and_bucket() { + for op in PEER_FAILURE_OPS { + let message = peer_failure_without_details(op, None).to_string(); + assert!(message.contains(op), "{op} message must name the operation: {message}"); + } + + for op in ["load_bucket_metadata", "delete_bucket_metadata"] { + let message = peer_failure_without_details(op, Some("ops-bucket")).to_string(); + assert!(message.contains(op), "{op} message must name the operation: {message}"); + assert!(message.contains("ops-bucket"), "{op} message must name the bucket: {message}"); + } + } + + #[test] + fn peer_failure_without_details_keeps_one_reduce_errs_bucket_per_operation() { + // reduce_errs groups Io errors by kind plus rendered message: peers failing the + // same operation must stay a single dominant error instead of one bucket per peer. + let per_peer_errs = (0..4) + .map(|_| Some(DiskError::from(peer_failure_without_details("load_bucket_metadata", Some("shared"))))) + .collect::>(); + let (count, dominant) = reduce_errs(&per_peer_errs, &[]); + assert_eq!(count, 4, "one shared failure must not split into per-peer buckets"); + assert_eq!( + dominant, + Some(DiskError::from(peer_failure_without_details("load_bucket_metadata", Some("shared")))) + ); + + assert_ne!( + peer_failure_without_details("load_bucket_metadata", Some("shared")).to_string(), + peer_failure_without_details("delete_bucket_metadata", Some("shared")).to_string() + ); + assert_ne!( + peer_failure_without_details("load_bucket_metadata", Some("bucket-a")).to_string(), + peer_failure_without_details("load_bucket_metadata", Some("bucket-b")).to_string() + ); + } + + #[test] + fn peer_failure_without_details_never_reads_as_a_network_failure() { + // `finalize_result` marks the peer offline and evicts its connection whenever the + // message matches a network needle. A peer that answered `success = false` is alive, + // so no operation or bucket name may push this text over that classifier. + for op in PEER_FAILURE_OPS { + let err = peer_failure_without_details(op, None); + assert!( + !PeerRestClient::is_network_like_error(&err), + "{op} must not read as a transport failure: {err}" + ); + + let scoped = peer_failure_without_details(op, Some("bucket-name")); + assert!( + !PeerRestClient::is_network_like_error(&scoped), + "{op} must not read as a transport failure: {scoped}" + ); + } + + // The bucket name is caller-supplied. Every needle carries a space, which S3 bucket + // names cannot, and the name is closed by `)` before the literal text resumes, so no + // needle can straddle the boundary either. + for bucket in [ + "timed-out", + "connection-reset", + "transport-error", + "broken-pipe", + "unavailable-logs", + ] { + let err = peer_failure_without_details("load_bucket_metadata", Some(bucket)); + assert!( + !PeerRestClient::is_network_like_error(&err), + "bucket {bucket} must not push the message over the network classifier: {err}" + ); + } + } } diff --git a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs index 02c9f75f1..d44651dce 100644 --- a/crates/ecstore/src/cluster/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_s3_client.rs @@ -220,6 +220,8 @@ fn pool_write_quorum(participant_count: usize) -> usize { /// buckets `Error::Io` by kind plus rendered message, so any per-peer detail (address, /// timing) would split one shared failure into single-count buckets and downgrade a real /// dominant error into `ErasureWriteQuorum`. +/// +/// `peer_rest_client` carries the same helper over `StorageError` for the same response shape. fn peer_failure_without_details(op: &str, bucket: Option<&str>) -> Error { match bucket { Some(bucket) => Error::other(format!("{op}({bucket}): peer returned failure without error details")), From 9ef059c9083f637e45666ec95ad6263fdaa5723b Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:18:13 +0800 Subject: [PATCH 20/22] ci(package): auto-trigger DEB/RPM packaging on releases and upload to GitHub release assets (#6202) --- .github/workflows/package.yml | 95 +++++++++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 11 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index d44205fa0..b9536d263 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -15,28 +15,35 @@ # Package Workflow - Build DEB/RPM packages # # This workflow builds DEB and RPM packages from pre-built Linux binaries -# and uploads them to Cloudflare R2. +# and uploads them to Cloudflare R2 and the GitHub release. # # Trigger: -# - release published: automatically package when a GitHub release is published -# - workflow_dispatch: manual trigger with optional tag/run_id +# - workflow_run: automatically package after "Build and Release" completes +# for a release tag (the mac/windows/linux binaries are already uploaded +# to the GitHub release before packaging starts) +# - workflow_dispatch: manual fallback (backfill / re-run) with optional tag/run_id # # Flow: -# 1. Find the Build workflow run for the release tag +# 1. Resolve the triggering Build workflow run for the release tag # 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts # 3. Build DEB packages for amd64 and arm64 # 4. Build RPM packages for x86_64 and aarch64 -# 5. Upload all packages to Cloudflare R2 +# 5. Upload all packages to Cloudflare R2 and the GitHub release name: Package DEB/RPM permissions: - contents: read + # contents: write is required to upload packages to the GitHub release + contents: write actions: read on: - release: - types: [ published ] + # Follows the same pattern as docker.yml: run after the release build + # workflow completes, so packaging is triggered only by release tags + # (e.g. 1.0.0-rc.2, 1.0.0-rc.3), never by development builds. + workflow_run: + workflows: [ "Build and Release" ] + types: [ completed ] workflow_dispatch: inputs: tag: @@ -49,13 +56,26 @@ on: type: string concurrency: - group: ${{ github.workflow }}-${{ github.event.release.tag_name || github.event.inputs.tag || github.run_id }} + group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.event.inputs.tag || github.run_id }} cancel-in-progress: true +env: + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }} + jobs: # Resolve which build run to use and extract version info resolve: name: Resolve Build + # Auto-trigger only from successful tag builds of "Build and Release". + # Tag pushes arrive as event == push with head_branch != main (a + # non-main push head_branch is the release tag name). Manual dispatch + # stays available as a fallback for backfills and re-runs. + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch != 'main') runs-on: ubuntu-latest timeout-minutes: 10 outputs: @@ -75,8 +95,8 @@ jobs: set -euo pipefail # Determine tag - if [[ "${{ github.event_name }}" == "release" ]]; then - TAG="${{ github.event.release.tag_name }}" + if [[ "${{ github.event_name }}" == "workflow_run" ]]; then + TAG="${HEAD_BRANCH}" elif [[ -n "$INPUT_TAG" ]]; then TAG="$INPUT_TAG" else @@ -93,6 +113,11 @@ jobs: BUILD_RUN_ID="$INPUT_RUN_ID" echo "Using explicit build run ID: $BUILD_RUN_ID" + elif [[ "${{ github.event_name }}" == "workflow_run" ]]; then + # Use the Build and Release run that triggered this workflow + BUILD_RUN_ID="${WORKFLOW_RUN_ID}" + echo "Using triggering workflow run: $BUILD_RUN_ID" + elif [[ -n "$TAG" ]]; then # Find the build run that produced this tag echo "Looking for build run for tag: $TAG" @@ -456,6 +481,54 @@ jobs: echo "✅ Latest packages updated" fi + - name: Upload packages to GitHub Release + if: needs.resolve.outputs.tag != '' + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + + TAG="${{ needs.resolve.outputs.tag }}" + DEB_FILE="${{ steps.deb.outputs.deb_file }}" + RPM_FILE="${{ steps.rpm.outputs.rpm_file }}" + + # Upload the packages, then refresh the release checksums so the new + # assets are covered, matching the binary release flow. + for f in "$DEB_FILE" "$RPM_FILE"; do + if [[ -n "$f" && -f "$f" ]]; then + echo "📤 Uploading $(basename "$f") to GitHub release ${TAG}..." + gh release upload "$TAG" "$f" --clobber + fi + done + + CHECKSUM_DIR="$(mktemp -d)" + gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \ + -D "$CHECKSUM_DIR" --clobber 2>/dev/null || true + + for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do + asset="${spec%%:*}" + checksum_cmd="${spec##*:}" + checksum_file="${CHECKSUM_DIR}/${asset}" + + touch "$checksum_file" + + for f in "$DEB_FILE" "$RPM_FILE"; do + if [[ -n "$f" && -f "$f" ]]; then + base="$(basename "$f")" + # Remove any stale entry, then append the fresh digest + grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true + mv "${checksum_file}.tmp" "$checksum_file" + (cd "$(dirname "$f")" && "$checksum_cmd" -- "$base") >> "$checksum_file" + fi + done + + echo "📤 Updating ${asset} for release ${TAG}..." + gh release upload "$TAG" "$checksum_file" --clobber + done + + echo "✅ GitHub release assets updated" + # Summary summary: name: Summary From 60eb139db9306f6ada4ed5090437a86bd41d5e95 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 13:27:38 +0800 Subject: [PATCH 21/22] refactor: import x-amz-checksum header names from the shared constants (#6193) Co-authored-by: houseme --- crates/ecstore/src/client/api_get_options.rs | 3 +- crates/ecstore/src/client/transition_api.rs | 16 ++++++---- rustfs/src/admin/router.rs | 31 ++++++++++---------- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/crates/ecstore/src/client/api_get_options.rs b/crates/ecstore/src/client/api_get_options.rs index 503b44f3a..3438453b3 100644 --- a/crates/ecstore/src/client/api_get_options.rs +++ b/crates/ecstore/src/client/api_get_options.rs @@ -20,6 +20,7 @@ #![allow(clippy::all)] use http::{HeaderMap, HeaderName, HeaderValue}; +use rustfs_utils::http::headers::AMZ_CHECKSUM_MODE; use std::collections::HashMap; use time::OffsetDateTime; use tracing::warn; @@ -76,7 +77,7 @@ impl GetObjectOptions { } } if self.checksum { - headers.insert(HeaderName::from_static("x-amz-checksum-mode"), HeaderValue::from_static("ENABLED")); + headers.insert(HeaderName::from_static(AMZ_CHECKSUM_MODE), HeaderValue::from_static("ENABLED")); } headers } diff --git a/crates/ecstore/src/client/transition_api.rs b/crates/ecstore/src/client/transition_api.rs index 6ad802418..be8e0a510 100644 --- a/crates/ecstore/src/client/transition_api.rs +++ b/crates/ecstore/src/client/transition_api.rs @@ -54,6 +54,10 @@ use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE; use rustfs_rio::HashReader; use rustfs_utils::HashAlgorithm; use rustfs_utils::{ + http::headers::{ + AMZ_CHECKSUM_CRC32, AMZ_CHECKSUM_CRC32C, AMZ_CHECKSUM_CRC64NVME, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_SHA1, + AMZ_CHECKSUM_SHA256, + }, net::get_endpoint_url, retry::{DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer}, }; @@ -1383,12 +1387,12 @@ pub(crate) fn to_object_info_for_provider( }; // Extract checksums - let checksum_crc32 = get_header("x-amz-checksum-crc32"); - let checksum_crc32c = get_header("x-amz-checksum-crc32c"); - let checksum_sha1 = get_header("x-amz-checksum-sha1"); - let checksum_sha256 = get_header("x-amz-checksum-sha256"); - let checksum_crc64nvme = get_header("x-amz-checksum-crc64nvme"); - let checksum_mode = get_header("x-amz-checksum-mode"); + let checksum_crc32 = get_header(AMZ_CHECKSUM_CRC32); + let checksum_crc32c = get_header(AMZ_CHECKSUM_CRC32C); + let checksum_sha1 = get_header(AMZ_CHECKSUM_SHA1); + let checksum_sha256 = get_header(AMZ_CHECKSUM_SHA256); + let checksum_crc64nvme = get_header(AMZ_CHECKSUM_CRC64NVME); + let checksum_mode = get_header(AMZ_CHECKSUM_MODE); // Build and return the ObjectInfo struct Ok(ObjectInfo { diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 9eb893792..717de0bee 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -67,6 +67,9 @@ use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_s3_types::EventName; use rustfs_signer::pre_sign_v4; use rustfs_utils::egress::{OutboundDnsResolver, OutboundPolicy}; +use rustfs_utils::http::headers::{ + AMZ_CHECKSUM_CRC32, AMZ_CHECKSUM_CRC32C, AMZ_CHECKSUM_CRC64NVME, AMZ_CHECKSUM_SHA1, AMZ_CHECKSUM_SHA256, AMZ_CHECKSUM_TYPE, +}; use rustfs_utils::http::{ SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, get_source_scheme, insert_header, @@ -1031,28 +1034,24 @@ fn build_get_object_response_headers(output: &GetObjectOutput, base_headers: &He )?; } if let Some(checksum_crc32) = &output.checksum_crc32 { - insert_string_header(&mut headers, HeaderName::from_static("x-amz-checksum-crc32"), checksum_crc32.clone())?; + insert_string_header(&mut headers, HeaderName::from_static(AMZ_CHECKSUM_CRC32), checksum_crc32.clone())?; } if let Some(checksum_crc32c) = &output.checksum_crc32c { - insert_string_header(&mut headers, HeaderName::from_static("x-amz-checksum-crc32c"), checksum_crc32c.clone())?; + insert_string_header(&mut headers, HeaderName::from_static(AMZ_CHECKSUM_CRC32C), checksum_crc32c.clone())?; } if let Some(checksum_crc64nvme) = &output.checksum_crc64nvme { - insert_string_header( - &mut headers, - HeaderName::from_static("x-amz-checksum-crc64nvme"), - checksum_crc64nvme.clone(), - )?; + insert_string_header(&mut headers, HeaderName::from_static(AMZ_CHECKSUM_CRC64NVME), checksum_crc64nvme.clone())?; } if let Some(checksum_sha1) = &output.checksum_sha1 { - insert_string_header(&mut headers, HeaderName::from_static("x-amz-checksum-sha1"), checksum_sha1.clone())?; + insert_string_header(&mut headers, HeaderName::from_static(AMZ_CHECKSUM_SHA1), checksum_sha1.clone())?; } if let Some(checksum_sha256) = &output.checksum_sha256 { - insert_string_header(&mut headers, HeaderName::from_static("x-amz-checksum-sha256"), checksum_sha256.clone())?; + insert_string_header(&mut headers, HeaderName::from_static(AMZ_CHECKSUM_SHA256), checksum_sha256.clone())?; } if let Some(checksum_type) = &output.checksum_type { insert_string_header( &mut headers, - HeaderName::from_static("x-amz-checksum-type"), + HeaderName::from_static(AMZ_CHECKSUM_TYPE), checksum_type.as_str().to_string(), )?; } @@ -1114,12 +1113,12 @@ fn clear_object_lambda_variant_headers(headers: &mut HeaderMap) { http::header::ETAG, http::header::LAST_MODIFIED, http::header::EXPIRES, - HeaderName::from_static("x-amz-checksum-crc32"), - HeaderName::from_static("x-amz-checksum-crc32c"), - HeaderName::from_static("x-amz-checksum-crc64nvme"), - HeaderName::from_static("x-amz-checksum-sha1"), - HeaderName::from_static("x-amz-checksum-sha256"), - HeaderName::from_static("x-amz-checksum-type"), + HeaderName::from_static(AMZ_CHECKSUM_CRC32), + HeaderName::from_static(AMZ_CHECKSUM_CRC32C), + HeaderName::from_static(AMZ_CHECKSUM_CRC64NVME), + HeaderName::from_static(AMZ_CHECKSUM_SHA1), + HeaderName::from_static(AMZ_CHECKSUM_SHA256), + HeaderName::from_static(AMZ_CHECKSUM_TYPE), HeaderName::from_static("x-amz-tagging-count"), HeaderName::from_static("x-amz-request-route"), HeaderName::from_static("x-amz-request-token"), From 355c8d2e2292596bbbb034fca1fbcec998dc935a Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Tue, 18 Aug 2026 13:40:10 +0800 Subject: [PATCH 22/22] fix(admin): classify missing kms config by error variant (#6196) --- .../ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs | 2 +- crates/ecstore/src/error/mod.rs | 8 ++++++++ rustfs/src/admin/handlers/kms_dynamic.rs | 8 ++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index ec5aaf74e..326b798ee 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -1783,7 +1783,7 @@ impl TransitionState { .await; } global_metrics().record_scanner_transition_failed(1); - if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) && !err.to_string().contains("use of closed network connection") { + if !is_err_version_not_found(&err) && !is_err_object_not_found(&err) && !is_network_or_host_down(&err.to_string(), false) { error!( event = EVENT_LIFECYCLE_TIER_OPERATION_FAILED, component = LOG_COMPONENT_ECSTORE, diff --git a/crates/ecstore/src/error/mod.rs b/crates/ecstore/src/error/mod.rs index 76dcc7a0c..f8215d908 100644 --- a/crates/ecstore/src/error/mod.rs +++ b/crates/ecstore/src/error/mod.rs @@ -1116,6 +1116,14 @@ mod tests { assert!(encoder_source.is::()); } + // The lifecycle transition worker relies on this arm alone to suppress the + // closed-connection noise (`bucket_lifecycle_ops.rs`); dropping it here would + // silently turn shutdown races back into `error!` log spam. + #[test] + fn is_network_or_host_down_covers_closed_network_connection() { + assert!(is_network_or_host_down("transition failed: use of closed network connection", false)); + } + // Regression for #952 (ECA-11): an all-`DiskNotFound` slice (every drive in // every set unreachable) must NOT be classified as "all not found", // otherwise ListObjects silently returns an empty listing and masks a full diff --git a/rustfs/src/admin/handlers/kms_dynamic.rs b/rustfs/src/admin/handlers/kms_dynamic.rs index 887ec7fad..d7ee7a0f7 100644 --- a/rustfs/src/admin/handlers/kms_dynamic.rs +++ b/rustfs/src/admin/handlers/kms_dynamic.rs @@ -22,6 +22,7 @@ use crate::admin::runtime_sources::{ current_object_store_handle_for_context, current_or_init_kms_runtime_service_manager, }; use crate::admin::storage_api::config::{read_admin_config, save_admin_config}; +use crate::admin::storage_api::error::StorageError; use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ADMIN_PREFIX, RemoteAddr}; use hyper::{Method, StatusCode}; @@ -278,8 +279,11 @@ pub async fn load_kms_config() -> Option { } }, Err(e) => { - // Config not found is normal on first run - if e.to_string().contains("ConfigNotFound") || e.to_string().contains("not found") { + // Config not found is normal on first run: `read_config` maps a missing or + // empty config object to `ConfigNotFound`, so that variant is the only + // "absent" signal reaching here. Every other not-found variant (disk, + // volume, bucket) means degraded storage and must stay a warning. + if matches!(e, StorageError::ConfigNotFound) { info!( component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_KMS,