Compare commits

..

15 Commits

Author SHA1 Message Date
houseme 5ff5a7a06e Merge branch 'main' into cxymds/fix-1927-durable-checkpoint
Signed-off-by: houseme <housemecn@gmail.com>
2026-08-23 12:27:11 +08:00
Zhengchao An b6ba89d9e4 docs(testing): document CI gate matrix (#6412) 2026-08-23 12:09:06 +08:00
houseme 648d5166e2 feat(allocator): replace mimalloc/libmimalloc-sys with rustfs-mimalloc/rustfs-mimalloc-sys (#6404)
Replace the upstream xonatius/mimalloc_rust.git fork (mimalloc + libmimalloc-sys)
with the published rustfs-mimalloc (v0.5.0) and rustfs-mimalloc-sys (v0.5.0) crates
from crates.io.

The new crates are based on mimalloc V3 (v3.5.0) and provide:
- MiMalloc global allocator with safe API (collect, stats_json, process_info)
- Heap management and arena operations (heap module)
- Full FFI bindings to mimalloc V3

Changes:
- Workspace deps: mimalloc + libmimalloc-sys (git) → rustfs-mimalloc + rustfs-mimalloc-sys (crates.io)
- allocator_reclaim.rs: libmimalloc_sys::mi_collect → rustfs_mimalloc::MiMalloc::collect
- memory_observability.rs: raw FFI mi_stats_get_json → MiMalloc::stats_json()
- main.rs: heap ownership tests use Heap::contains() (V3 API)
- deny.toml: remove xonatius/mimalloc_rust.git from allow-git

Co-authored-by: heihutu <heihutu@gmail.com>
2026-08-23 12:07:25 +08:00
houseme 84eb5aebef fix(ecstore): remove inline write debug noise (#6408)
* fix(ecstore): remove inline write debug noise

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(ecstore): satisfy warning-as-error lints

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
2026-08-23 12:07:20 +08:00
overtrue c58accabe7 fix(ecstore): support Windows checkpoint CAS 2026-08-23 07:58:22 +08:00
马登山 4d92835d7b merge main and fix swift lint 2026-08-23 06:57:56 +08:00
overtrue 71db3327ad fix(heal): reset unverified checkpoint progress 2026-08-23 06:17:31 +08:00
overtrue 68e95d497e fix(heal): require current checkpoint digest 2026-08-23 04:19:04 +08:00
overtrue 8350d73be9 fix(storage): bound conditional file lock artifacts 2026-08-23 02:24:27 +08:00
overtrue 9a2a33a7eb fix(heal): canonicalize checkpoint integrity digest 2026-08-23 00:09:14 +08:00
cxymds 5df26a13ba Merge branch 'main' into cxymds/fix-1927-durable-checkpoint 2026-08-22 22:06:41 +08:00
马登山 e3a362989f fix(heal): atomically authenticate checkpoints 2026-08-22 22:05:23 +08:00
马登山 0cd2ae20e2 fix(heal): fail closed on tampered resume checkpoints 2026-08-22 15:32:12 +08:00
cxymds 55a7fa9f03 Merge branch 'main' into cxymds/fix-1927-durable-checkpoint 2026-08-22 11:25:52 +08:00
马登山 9f51f37a0d fix(heal): make resume checkpoints crash consistent 2026-08-21 22:04:23 +08:00
32 changed files with 1188 additions and 1034 deletions
+2 -1
View File
@@ -30,7 +30,8 @@ make build-docker BUILD_OS=ubuntu22.04
- Crate membership: `Cargo.toml` `[workspace].members`
- Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
- Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)
- CI gates: `.github/workflows/ci.yml` (source of truth; never copy its steps into docs)
- CI workflow steps: `.github/workflows/`; event, timeout, and required-status
matrix: [docs/testing/ci-gates.md](docs/testing/ci-gates.md)
- Test-layer taxonomy, per-layer entry commands, serial/nextest rules, flake
policy: [docs/testing/README.md](docs/testing/README.md)
- Tier/ILM transition debugging (xl.meta inspection, versionId tracing):
+2
View File
@@ -70,6 +70,8 @@ make pre-pr
> For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
> For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
### 🔒 Automated Pre-commit Hooks
#### What `make pre-commit` and `make pre-pr` actually run
Generated
+23 -27
View File
@@ -1858,9 +1858,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.4.3"
version = "1.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -2522,12 +2522,6 @@ dependencies = [
"subtle",
]
[[package]]
name = "cty"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
[[package]]
name = "curve25519-dalek"
version = "4.1.3"
@@ -5988,15 +5982,6 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libmimalloc-sys"
version = "0.1.49"
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
dependencies = [
"cc",
"cty",
]
[[package]]
name = "libredox"
version = "0.1.20"
@@ -6397,14 +6382,6 @@ dependencies = [
"synstructure 0.13.2",
]
[[package]]
name = "mimalloc"
version = "0.1.52"
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
dependencies = [
"libmimalloc-sys",
]
[[package]]
name = "mime"
version = "0.3.17"
@@ -9162,13 +9139,11 @@ dependencies = [
"insta",
"jiff",
"libc",
"libmimalloc-sys",
"libsystemd",
"matchit 0.9.2",
"md-5 0.11.0",
"metrics",
"metrics-util",
"mimalloc",
"mime_guess",
"opentelemetry",
"opentelemetry_sdk",
@@ -9204,6 +9179,8 @@ dependencies = [
"rustfs-lock",
"rustfs-log-analyzer",
"rustfs-madmin",
"rustfs-mimalloc",
"rustfs-mimalloc-sys",
"rustfs-notify",
"rustfs-object-capacity",
"rustfs-object-data-cache",
@@ -9587,6 +9564,7 @@ dependencies = [
"serde",
"serde_json",
"serial_test",
"sha2 0.11.0",
"temp-env",
"tempfile",
"thiserror 2.0.20",
@@ -9875,6 +9853,24 @@ dependencies = [
"tokio",
]
[[package]]
name = "rustfs-mimalloc"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a406f4aa07084301d485beec873af6dccc8e3f8762da244743df92038b1db1a6"
dependencies = [
"rustfs-mimalloc-sys",
]
[[package]]
name = "rustfs-mimalloc-sys"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3051b819175f58445d4c369a72f0ab88149f3885ba8bea2aff3be01f53fe7cd"
dependencies = [
"cc",
]
[[package]]
name = "rustfs-notify"
version = "1.0.0-rc.3"
+2 -2
View File
@@ -350,8 +350,8 @@ russh-sftp = "2.4.0"
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
rustfs-mimalloc = { version = "0.5.0" }
rustfs-mimalloc-sys = { version = "0.5.0" }
hotpath = { version = "0.23.3", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.48" }
+11 -334
View File
@@ -23,32 +23,21 @@
//! unconsumed intents is the consumer's job (see `rustfs-heal`
//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
use std::collections::HashMap;
use std::collections::hash_map::RandomState;
use std::hash::{BuildHasher, Hash};
use std::sync::atomic::AtomicU64;
use std::sync::atomic::AtomicUsize;
use std::sync::{
Arc, Mutex, OnceLock,
Arc, OnceLock,
atomic::{AtomicBool, Ordering},
};
use std::time::{Duration, Instant};
use tokio::sync::mpsc;
use uuid::Uuid;
/// Bounded capacity of the global MRF channel. Backpressure is resolved by
/// dropping (and counting) intents, never by blocking the producer.
const MRF_CHANNEL_CAPACITY: usize = 8192;
const MRF_COALESCER_SHARDS: usize = 16;
const MRF_COALESCER_MAX_KEYS: usize = 8192;
const MRF_COALESCER_MAX_BYTES: usize = 16 * 1024 * 1024;
const MRF_COALESCER_TTL: Duration = Duration::from_secs(60);
const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
/// Why an intent was produced. Drives the heal priority mapping on the
/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
/// PartialWrite -> Normal).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MrfKind {
/// Erasure decode failed while serving a read (read path).
DecodeFailure,
@@ -78,52 +67,12 @@ pub struct MrfIntent {
/// Version the intent targets, as raw UUID bytes.
pub version_id: Option<[u8; 16]>,
pub kind: MrfKind,
/// Stable erasure-set scope when the producer has it. Kept optional so
/// metadata corruption and legacy producers do not invent a scope.
pub scope: Option<MrfScope>,
/// Generation of the node-local ingress lease. It is not persisted in
/// the journal; replayed records acquire a fresh lease when re-enqueued.
pub lease: Option<MrfIngressLease>,
pub enqueued_at_ms: u64,
/// Times this intent has already been offered to the heal manager.
/// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
pub attempts: u8,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct MrfScope {
pub pool_index: u32,
pub set_index: u32,
}
/// Opaque generation used to release exactly the admission that created an
/// ingress entry. A generation prevents a late terminal callback from
/// deleting a newer retry for the same identity (ABA).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct MrfIngressLease(u64);
impl MrfIngressLease {
const fn new(value: u64) -> Self {
Self(value)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MrfDropReason {
Disabled,
Uninitialized,
Full,
OversizedIdentity,
CoalescerFull,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MrfIngressResult {
Enqueued,
Coalesced,
Dropped(MrfDropReason),
}
/// Consumer-side retry ceiling before an intent is given up on.
pub const MRF_MAX_ATTEMPTS: u8 = 3;
@@ -138,159 +87,6 @@ impl MrfIntent {
static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
struct MrfIdentityKey {
kind: MrfKind,
bucket: Arc<str>,
object: Arc<str>,
version_id: Option<[u8; 16]>,
scope: Option<MrfScope>,
}
#[derive(Debug)]
struct IngressEntry {
lease: MrfIngressLease,
expires_at: Instant,
bytes: usize,
}
type MrfCoalescerShard = Mutex<HashMap<MrfIdentityKey, IngressEntry>>;
type MrfCoalescer = Box<[MrfCoalescerShard]>;
static MRF_COALESCER: OnceLock<MrfCoalescer> = OnceLock::new();
static NEXT_MRF_LEASE: AtomicU64 = AtomicU64::new(1);
static MRF_COALESCER_COUNT: AtomicUsize = AtomicUsize::new(0);
static MRF_COALESCER_BYTES: AtomicUsize = AtomicUsize::new(0);
static MRF_HASH_STATE: OnceLock<RandomState> = OnceLock::new();
fn coalescer() -> &'static [MrfCoalescerShard] {
MRF_COALESCER.get_or_init(|| {
(0..MRF_COALESCER_SHARDS)
.map(|_| Mutex::new(HashMap::new()))
.collect::<Vec<_>>()
.into_boxed_slice()
})
}
fn key_shard(key: &MrfIdentityKey) -> usize {
let hash = MRF_HASH_STATE.get_or_init(RandomState::new).hash_one(key);
usize::try_from(hash).unwrap_or(0) % MRF_COALESCER_SHARDS
}
fn canonical_version(version_id: Option<Uuid>) -> Option<[u8; 16]> {
version_id
.filter(|version| !version.is_nil())
.map(|version| *version.as_bytes())
}
fn canonical_identity(
kind: MrfKind,
version_id: Option<[u8; 16]>,
scope: Option<MrfScope>,
) -> (Option<[u8; 16]>, Option<MrfScope>) {
let version_id = version_id.filter(|bytes| *bytes != [0; 16]);
match kind {
MrfKind::MetadataCorruption => (None, None),
MrfKind::DecodeFailure | MrfKind::PartialWrite => (version_id, scope),
}
}
fn identity_estimated_bytes(key: &MrfIdentityKey) -> usize {
64usize
.saturating_add(key.bucket.len())
.saturating_add(key.object.len())
.saturating_add(key.version_id.map_or(0, |_| 16))
.saturating_add(key.scope.map_or(0, |_| 8))
}
fn reserve(counter: &AtomicUsize, limit: usize, amount: usize) -> bool {
let mut current = counter.load(Ordering::Relaxed);
loop {
let Some(next) = current.checked_add(amount) else {
return false;
};
if next > limit {
return false;
}
match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => return true,
Err(observed) => current = observed,
}
}
}
fn coalescer_admit(key: MrfIdentityKey) -> Result<MrfIngressLease, MrfIngressResult> {
let shard = key_shard(&key);
let mut entries = coalescer()[shard]
.lock()
.map_err(|_| MrfIngressResult::Dropped(MrfDropReason::CoalescerFull))?;
let now = Instant::now();
let before = entries.len();
let mut expired_bytes = 0usize;
entries.retain(|_, entry| {
if entry.expires_at > now {
true
} else {
expired_bytes = expired_bytes.saturating_add(entry.bytes);
false
}
});
let evicted = before.saturating_sub(entries.len());
if evicted > 0 {
MRF_COALESCER_COUNT.fetch_sub(evicted, Ordering::Relaxed);
MRF_COALESCER_BYTES.fetch_sub(expired_bytes, Ordering::Relaxed);
let evicted = u64::try_from(evicted).unwrap_or(u64::MAX);
metrics::counter!("rustfs_heal_mrf_coalescer_expired_total").increment(evicted);
metrics::counter!("rustfs_heal_mrf_coalescer_evictions_total").increment(evicted);
}
if entries.contains_key(&key) {
metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
return Err(MrfIngressResult::Coalesced);
}
let bytes = identity_estimated_bytes(&key);
let count_reserved = reserve(&MRF_COALESCER_COUNT, MRF_COALESCER_MAX_KEYS, 1);
let bytes_reserved = count_reserved && reserve(&MRF_COALESCER_BYTES, MRF_COALESCER_MAX_BYTES, bytes);
if !count_reserved || !bytes_reserved {
if count_reserved {
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
}
metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "coalescer_full").increment(1);
return Err(MrfIngressResult::Dropped(MrfDropReason::CoalescerFull));
}
let lease = MrfIngressLease::new(NEXT_MRF_LEASE.fetch_add(1, Ordering::Relaxed));
if entries
.insert(
key,
IngressEntry {
lease,
expires_at: now + MRF_COALESCER_TTL,
bytes,
},
)
.is_some()
{
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
return Err(MrfIngressResult::Coalesced);
}
Ok(lease)
}
fn coalescer_release(key: &MrfIdentityKey, lease: Option<MrfIngressLease>) {
let Some(lease) = lease else {
return;
};
if let Ok(mut entries) = coalescer()[key_shard(key)].lock() {
let should_remove = entries.get(key).is_some_and(|entry| entry.lease == lease);
if should_remove {
let bytes = entries.remove(key).map(|entry| entry.bytes).unwrap_or(0);
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
}
}
}
/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
/// this before touching the channel so the disabled path stays allocation- and
/// sync-free.
@@ -326,90 +122,21 @@ pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
/// This runs on IO error paths, so it stays synchronous and cheap: one
/// bounded allocation for the two `Arc<str>` handles plus the channel slot.
pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
matches!(
try_send_mrf_intent_typed(kind, bucket, object, version_id, None),
MrfIngressResult::Enqueued
)
}
/// Typed ingress result. `Coalesced` means an equivalent in-flight channel
/// intent already exists; it is not a second executable or durable admission.
pub fn try_send_mrf_intent_typed(
kind: MrfKind,
bucket: &str,
object: &str,
version_id: Option<Uuid>,
scope: Option<MrfScope>,
) -> MrfIngressResult {
if !mrf_delivery_enabled() {
return MrfIngressResult::Dropped(MrfDropReason::Disabled);
return false;
}
let Some(sender) = GLOBAL_MRF_SENDER.get() else {
return MrfIngressResult::Dropped(MrfDropReason::Uninitialized);
};
if bucket.len() > MRF_MAX_IDENTITY_COMPONENT || object.len() > MRF_MAX_IDENTITY_COMPONENT {
return MrfIngressResult::Dropped(MrfDropReason::OversizedIdentity);
}
let (version_id, scope) = canonical_identity(kind, canonical_version(version_id), scope);
let key = MrfIdentityKey {
kind,
bucket: Arc::from(bucket),
object: Arc::from(object),
version_id,
scope,
};
let lease = match coalescer_admit(key.clone()) {
Ok(lease) => lease,
Err(result) => return result,
return false;
};
let intent = MrfIntent {
bucket: key.bucket.clone(),
object: key.object.clone(),
version_id: key.version_id,
bucket: Arc::from(bucket),
object: Arc::from(object),
version_id: version_id.map(|vid| *vid.as_bytes()),
kind,
scope,
lease: Some(lease),
enqueued_at_ms: unix_now_ms(),
attempts: 0,
};
match sender.try_send(intent) {
Ok(()) => MrfIngressResult::Enqueued,
Err(mpsc::error::TrySendError::Full(_)) => {
coalescer_release(&key, Some(lease));
metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "channel_full").increment(1);
MrfIngressResult::Dropped(MrfDropReason::Full)
}
Err(mpsc::error::TrySendError::Closed(_)) => {
coalescer_release(&key, Some(lease));
MrfIngressResult::Dropped(MrfDropReason::Uninitialized)
}
}
}
/// Release the ingress key once the consumer owns the intent.
pub fn release_mrf_intent(intent: &MrfIntent) {
release_mrf_identity(intent.kind, &intent.bucket, &intent.object, intent.version_id, intent.scope, intent.lease);
}
pub fn release_mrf_identity(
kind: MrfKind,
bucket: &str,
object: &str,
version_id: Option<[u8; 16]>,
scope: Option<MrfScope>,
lease: Option<MrfIngressLease>,
) {
let (version_id, scope) = canonical_identity(kind, version_id, scope);
coalescer_release(
&MrfIdentityKey {
kind,
bucket: Arc::from(bucket),
object: Arc::from(object),
version_id,
scope,
},
lease,
);
sender.try_send(intent).is_ok()
}
fn unix_now_ms() -> u64 {
@@ -417,8 +144,7 @@ fn unix_now_ms() -> u64 {
// failure would be a bug rather than something to handle here.
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.ok()
.and_then(|d| u64::try_from(d.as_millis()).ok())
.map(|d| d.as_millis() as u64)
.unwrap_or(0)
}
@@ -489,60 +215,12 @@ mod tests {
object: Arc::from("object"),
version_id: Some([0u8; 16]),
kind: MrfKind::DecodeFailure,
scope: None,
lease: None,
enqueued_at_ms: 0,
attempts: 0,
};
assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
}
#[test]
fn ingress_duplicate_identity_coalesces_and_releases_for_retry() {
let key = MrfIdentityKey {
kind: MrfKind::DecodeFailure,
bucket: Arc::from("ingress-test-bucket"),
object: Arc::from("ingress-test-object"),
version_id: Some([9; 16]),
scope: Some(MrfScope {
pool_index: 3,
set_index: 4,
}),
};
let lease = coalescer_admit(key.clone()).expect("first identity should be admitted");
for _ in 0..999 {
assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced));
}
coalescer_release(&key, Some(lease));
let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry");
coalescer_release(&key, Some(retry_lease));
}
#[test]
fn ingress_identity_preserves_kind_scope_and_version_boundaries() {
let (nil_version, nil_scope) = canonical_identity(
MrfKind::DecodeFailure,
Some([0; 16]),
Some(MrfScope {
pool_index: 1,
set_index: 2,
}),
);
assert_eq!(nil_version, None, "nil UUID is the unversioned identity");
assert!(nil_scope.is_some());
let (metadata_version, metadata_scope) = canonical_identity(
MrfKind::MetadataCorruption,
Some([7; 16]),
Some(MrfScope {
pool_index: 1,
set_index: 2,
}),
);
assert_eq!(metadata_version, None);
assert_eq!(metadata_scope, None);
}
#[tokio::test]
async fn try_send_delivers_and_respects_capacity() {
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
@@ -552,7 +230,6 @@ mod tests {
let intent = receiver.recv().await.expect("intent should arrive");
assert_eq!(intent.kind, MrfKind::DecodeFailure);
assert_eq!(intent.bucket.as_ref(), "b");
release_mrf_intent(&intent);
// Disable delivery: producers become no-ops.
set_mrf_delivery_enabled(false);
@@ -562,8 +239,8 @@ mod tests {
// Fill the bounded channel past capacity: excess intents are dropped,
// never blocking.
let mut accepted = 0;
for index in 0..(MRF_CHANNEL_CAPACITY + 64) {
if try_send_mrf_intent(MrfKind::PartialWrite, "b", &format!("o-{index}"), None) {
for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
accepted += 1;
}
}
+76 -6
View File
@@ -8000,10 +8000,15 @@ impl DiskAPI for LocalDisk {
use std::io::Write as _;
let file_path = self.io_get_object_path(volume, path)?;
let lock_path = file_path.with_extension("rustfs-cas.lock");
let path = path.to_string();
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
return Ok(tokio::task::spawn_blocking(move || {
// A persistent directory lock bounds metadata growth. Removing
// per-target lock files can split flock ownership across inodes.
let lock_path = file_path
.parent()
.ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "conditional file has no parent"))?
.join(".rustfs-cas.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
@@ -8068,7 +8073,25 @@ impl DiskAPI for LocalDisk {
.map_err(DiskError::from)??);
}
#[cfg(not(unix))]
#[cfg(windows)]
{
let file_path = self.io_get_object_path(volume, path)?;
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
let publication_root = self.publication_root.clone();
return Ok(tokio::task::spawn_blocking(move || {
os::compare_and_update_control_file(
&file_path,
expected.as_deref(),
replacement.as_deref(),
sync_metadata,
&publication_root,
)
})
.await
.map_err(DiskError::from)??);
}
#[cfg(not(any(unix, windows)))]
{
let _ = (volume, path, expected, replacement);
Err(DiskError::MethodNotAllowed)
@@ -21823,9 +21846,9 @@ mod test {
assert!(matches!(results[1].as_ref().unwrap_err(), DiskError::Io(_)));
}
#[cfg(unix)]
#[cfg(any(unix, windows))]
#[tokio::test]
async fn conditional_file_update_never_deletes_a_new_owner() {
async fn windows_and_unix_conditional_file_update_never_deletes_a_new_owner() {
use tempfile::tempdir;
let dir = tempdir().expect("temp dir should be created");
@@ -21856,8 +21879,18 @@ mod test {
disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
.await
.expect("new owner marker should remain"),
owner_b
owner_b.clone()
);
assert_eq!(
disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Some(owner_b), None)
.await
.expect("current owner should remove marker"),
ConditionalFileUpdate::Updated
);
assert!(matches!(
disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH).await,
Err(DiskError::FileNotFound)
));
}
#[cfg(unix)]
@@ -21872,7 +21905,10 @@ mod test {
let marker_path = disk
.get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
.expect("marker path should resolve");
let lock_path = marker_path.with_extension("rustfs-cas.lock");
let lock_path = marker_path
.parent()
.expect("marker path should have a parent")
.join(".rustfs-cas.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
@@ -21893,6 +21929,40 @@ mod test {
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
}
#[cfg(windows)]
#[tokio::test]
async fn windows_conditional_file_update_returns_would_block_when_marker_lock_is_contended() {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
ensure_test_volume(&disk, RUSTFS_META_BUCKET).await;
let marker_path = disk
.get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
.expect("marker path should resolve");
let lock_path = marker_path
.parent()
.expect("marker path should have a parent")
.join(".rustfs-cas.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.read(true)
.write(true)
.open(lock_path)
.expect("marker lock should open");
lock.try_lock().expect("marker lock should be held");
let err = tokio::time::timeout(
Duration::from_secs(1),
disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, None, Some(Bytes::from_static(b"owner"))),
)
.await
.expect("contended conditional update must not block")
.expect_err("contended conditional update must retry");
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
}
#[cfg(target_os = "linux")]
#[tokio::test]
async fn replacement_io_paths_stay_under_the_mount_lease() {
+85
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(windows)]
use crate::disk::ConditionalFileUpdate;
use crate::disk::error::DiskError;
use crate::disk::error::Result;
use crate::disk::error_conv::to_file_error;
@@ -3458,6 +3460,89 @@ fn read_windows_relative_file(file_path: &Path, parent_guard: &ExistingBaseDirec
Ok(Some(data))
}
#[cfg(windows)]
pub(crate) fn compare_and_update_control_file(
file_path: &Path,
expected: Option<&[u8]>,
replacement: Option<&[u8]>,
sync_metadata: bool,
publication_root: &PublicationRoot,
) -> io::Result<ConditionalFileUpdate> {
use windows_sys::{
Wdk::Storage::FileSystem::{
FILE_NON_DIRECTORY_FILE, FILE_OPEN, FILE_OPEN_IF, FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT,
},
Win32::Storage::FileSystem::{
DELETE, FILE_ATTRIBUTE_NORMAL, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_WRITE_DATA, SYNCHRONIZE,
},
};
let parent = file_path
.parent()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file has no parent"))?;
let parent_guard = lock_windows_directory_tree(parent, Some(parent), publication_root)?;
let lock = open_windows_relative(
parent_guard.last_handle()?,
std::ffi::OsStr::new(".rustfs-cas.lock"),
SYNCHRONIZE | FILE_READ_ATTRIBUTES | FILE_WRITE_DATA,
FILE_SHARE_READ | FILE_SHARE_WRITE,
FILE_OPEN_IF,
FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
FILE_ATTRIBUTE_NORMAL,
true,
)?;
validate_windows_owned_file(&lock)?;
match lock.as_file().try_lock() {
Ok(()) => {}
Err(std::fs::TryLockError::WouldBlock) => return Err(io::Error::from(io::ErrorKind::WouldBlock)),
Err(std::fs::TryLockError::Error(err)) => return Err(err),
}
let current = read_windows_relative_file(file_path, &parent_guard)?;
let matches = match (&current, expected) {
(None, None) => true,
(Some(current), Some(expected)) => current.as_slice() == expected,
_ => false,
};
if !matches {
return Ok(match current {
None => ConditionalFileUpdate::Missing,
Some(_) => ConditionalFileUpdate::Mismatch,
});
}
match replacement {
Some(replacement) => RenameDestinationPathGuard {
directory: parent.to_path_buf(),
_directory_guard: parent_guard,
}
.write_file_for_path_access(file_path, replacement, sync_metadata, sync_metadata)?,
None => {
let file_name = file_path
.file_name()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file must have a name"))?;
let file = open_windows_relative(
parent_guard.last_handle()?,
file_name,
DELETE | SYNCHRONIZE | FILE_READ_ATTRIBUTES,
FILE_SHARE_READ,
FILE_OPEN,
FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
0,
true,
)?;
validate_windows_owned_file(&file)?;
set_windows_file_delete_on_close(&file, true)?;
drop(file);
if sync_metadata {
fsync_dir_std(parent)?;
}
}
}
Ok(ConditionalFileUpdate::Updated)
}
#[cfg(windows)]
fn open_windows_directory_component(
parent: &WindowsDirectoryHandle,
@@ -1412,11 +1412,8 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
// Reservation won: this sighting owns the repair records for the object,
// including the durable journal intent when the caller asked for one.
if let Some((kind, version_uuid)) = mrf_intent
&& let (Ok(pool_index), Ok(set_index)) = (u32::try_from(pool_index), u32::try_from(set_index))
{
let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
let _ = rustfs_common::mrf_channel::try_send_mrf_intent_typed(kind, bucket, object, version_uuid, Some(scope));
if let Some((kind, version_uuid)) = mrf_intent {
rustfs_common::mrf_channel::try_send_mrf_intent(kind, bucket, object, version_uuid);
}
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
+7 -31
View File
@@ -2124,26 +2124,13 @@ impl SetDisks {
let put_object_size = known_put_object_storage_size(data.size());
let shard_file_size_raw = erasure.shard_file_size(put_object_size);
let is_inline_buffer =
storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
let shard_file_size = shard_file_size_raw;
let shard_size = erasure.shard_size();
let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
let direct_inline_commit = matches!(write_path, SmallWritePath::Inline);
{
use std::io::Write;
let msg = format!(
"INLINE_DEBUG: bucket={} obj={} size={} shard_fs={} ds={} bs={} inline={} direct={} path={} iblock={} ver={}\n",
bucket, object, put_object_size, shard_file_size_raw, erasure.data_shards, fi.erasure.block_size,
is_inline_buffer, direct_inline_commit, write_path.metric_label(), storage_class_config.inline_block(), opts.versioned
);
if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open("/tmp/rustfs_inline_debug.log") {
let _ = f.write_all(msg.as_bytes());
}
let _ = std::io::stderr().write_all(msg.as_bytes());
}
rustfs_io_metrics::record_put_object_path(write_path.metric_label());
let writer_setup_stage_start = collect_stage_timing.then(Instant::now);
let (mut writers, errors) = if direct_inline_commit {
@@ -6650,23 +6637,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
// MRF journal intent: partial-write recovery must survive a restart
// (HS-01); the heal request below remains the in-memory fast path.
let version_uuid = if version_id.is_empty() {
Some(None)
} else {
uuid::Uuid::try_parse(version_id).ok().map(Some)
};
if let Some(version_uuid) = version_uuid
&& let (Ok(pool_index), Ok(set_index)) = (u32::try_from(self.pool_index), u32::try_from(self.set_index))
{
let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
let _ = rustfs_common::mrf_channel::try_send_mrf_intent_typed(
rustfs_common::mrf_channel::MrfKind::PartialWrite,
bucket,
object,
version_uuid,
Some(scope),
);
}
rustfs_common::mrf_channel::try_send_mrf_intent(
rustfs_common::mrf_channel::MrfKind::PartialWrite,
bucket,
object,
uuid::Uuid::try_parse(version_id).ok(),
);
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
bucket.to_string(),
Some(object.to_string()),
+1 -1
View File
@@ -3194,7 +3194,7 @@ impl ECStore {
// Default return value
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut accounting = vec![None; objects.len()];
let accounting = vec![None; objects.len()];
let mut del_errs = Vec::with_capacity(objects.len());
for _ in 0..objects.len() {
@@ -271,7 +271,7 @@ pub(super) fn resolve_latest_object_info_candidates(
.filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
.collect::<Vec<_>>();
latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx));
latest_candidates.sort_by_key(|candidate| std::cmp::Reverse(candidate.idx));
let Some(winner) = latest_candidates.first() else {
return Err(Error::ErasureReadQuorum);
+1
View File
@@ -91,6 +91,7 @@ metrics = { workspace = true }
base64 = { workspace = true }
bytes = { workspace = true }
crc-fast = { workspace = true }
sha2 = { workspace = true }
[dev-dependencies]
serde_json = { workspace = true, features = ["raw_value"] }
+5
View File
@@ -373,6 +373,11 @@ impl ErasureSetHealer {
set_disk_id: &str,
buckets: &[String],
) -> Result<(ResumeManager, CheckpointManager)> {
if self.replacement_task_id.is_none() && CheckpointManager::is_blocked(&self.disk, task_id).await {
return Err(Error::TaskExecutionFailed {
message: format!("Resume task {task_id} has a blocked checkpoint"),
});
}
// check if resume state exists
let has_resume_state = if self.replacement_task_id.is_some() {
ResumeManager::has_replacement_intent(&self.disk, task_id).await
+3 -54
View File
@@ -119,9 +119,6 @@ struct MrfRepairNoticeTarget {
bucket: Arc<str>,
object: Arc<str>,
version_id: Option<[u8; 16]>,
kind: rustfs_common::mrf_channel::MrfKind,
scope: Option<rustfs_common::mrf_channel::MrfScope>,
lease: Option<rustfs_common::mrf_channel::MrfIngressLease>,
}
#[derive(Debug, Clone)]
@@ -892,19 +889,7 @@ impl HealManager {
}
fn remove_mrf_repair_notice_targets_for_task(&self, task_id: &str) {
let targets = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(task_id);
if let Some(targets) = targets {
for target in targets {
rustfs_common::mrf_channel::release_mrf_identity(
target.kind,
&target.bucket,
&target.object,
target.version_id,
target.scope,
target.lease,
);
}
}
lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(task_id);
}
fn insert_mrf_repair_notice_target(
@@ -1294,20 +1279,7 @@ impl HealManager {
}
self.task_aliases.lock().await.clear();
self.retrying_heals.lock().await.clear();
let mrf_targets = {
let mut registry = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets);
registry.drain().flat_map(|(_, targets)| targets).collect::<Vec<_>>()
};
for target in mrf_targets {
rustfs_common::mrf_channel::release_mrf_identity(
target.kind,
&target.bucket,
&target.object,
target.version_id,
target.scope,
target.lease,
);
}
lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).clear();
crate::set_heal_queue_length(0);
// update state
@@ -1339,32 +1311,12 @@ impl HealManager {
.await
}
#[cfg(test)]
pub(crate) async fn submit_mrf_heal_request_with_receipt(
&self,
request: HealRequest,
bucket: Arc<str>,
object: Arc<str>,
version_id: Option<[u8; 16]>,
) -> Result<HealAdmissionReceipt> {
let kind = match &request.heal_type {
HealType::Metadata { .. } => rustfs_common::mrf_channel::MrfKind::MetadataCorruption,
HealType::ECDecode { .. } => rustfs_common::mrf_channel::MrfKind::DecodeFailure,
_ => rustfs_common::mrf_channel::MrfKind::PartialWrite,
};
self.submit_mrf_heal_request_with_receipt_and_identity(request, bucket, object, version_id, kind, None, None)
.await
}
pub(crate) async fn submit_mrf_heal_request_with_receipt_and_identity(
&self,
request: HealRequest,
bucket: Arc<str>,
object: Arc<str>,
version_id: Option<[u8; 16]>,
kind: rustfs_common::mrf_channel::MrfKind,
scope: Option<rustfs_common::mrf_channel::MrfScope>,
lease: Option<rustfs_common::mrf_channel::MrfIngressLease>,
) -> Result<HealAdmissionReceipt> {
self.submit_heal_request_with_receipt_alias_and_mrf_notice(
request,
@@ -1373,9 +1325,6 @@ impl HealManager {
bucket,
object,
version_id,
kind,
scope,
lease,
}),
)
.await
@@ -1590,7 +1539,7 @@ impl HealManager {
Self::insert_mrf_repair_notice_target(&mut targets, &task_id, target);
}
if let Some(displaced_task_id) = &displaced_task_id {
self.remove_mrf_repair_notice_targets_for_task(displaced_task_id);
lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(displaced_task_id);
}
drop(retrying_heals);
drop(queue);
+1 -12
View File
@@ -506,18 +506,7 @@ impl HealManager {
&displaced_terminal,
)
.await;
if let Some(targets) = lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id) {
for target in targets {
rustfs_common::mrf_channel::release_mrf_identity(
target.kind,
&target.bucket,
&target.object,
target.version_id,
target.scope,
target.lease,
);
}
}
lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id);
}
if matches!(admission, HealAdmissionResult::Accepted) {
if should_notify {
+1 -5
View File
@@ -299,11 +299,7 @@ impl PriorityHealQueue {
/// Create a deduplication key from a heal request
pub(super) fn make_dedup_key(request: &HealRequest) -> String {
let base = Self::make_dedup_key_for_type(&request.heal_type);
match (&request.heal_type, request.options.set_key()) {
(HealType::Object { .. } | HealType::ECDecode { .. }, Some(scope)) => format!("{base}:scope:{scope}"),
_ => base,
}
Self::make_dedup_key_for_type(&request.heal_type)
}
pub(super) fn make_dedup_key_for_type(heal_type: &HealType) -> String {
+1 -36
View File
@@ -349,8 +349,6 @@ impl HealManager {
let notice_targets = take_mrf_repair_notice_targets(&mrf_repair_notice_targets_clone, &task_id);
if successful_completion {
emit_mrf_repaired_events(notice_targets);
} else {
release_mrf_repair_notice_targets(notice_targets);
}
task_aliases_clone
.lock()
@@ -640,19 +638,7 @@ pub(super) fn running_heal_set_counts(active_heals: &HashMap<String, Arc<HealTas
}
fn remove_mrf_repair_notice_targets(registry: &Arc<StdMutex<HashMap<String, Vec<MrfRepairNoticeTarget>>>>, task_id: &str) {
let targets = lock_mrf_repair_notice_targets(registry).remove(task_id);
if let Some(targets) = targets {
for target in targets {
rustfs_common::mrf_channel::release_mrf_identity(
target.kind,
&target.bucket,
&target.object,
target.version_id,
target.scope,
target.lease,
);
}
}
lock_mrf_repair_notice_targets(registry).remove(task_id);
}
fn take_mrf_repair_notice_targets(
@@ -685,27 +671,6 @@ fn move_mrf_repair_notice_targets(
fn emit_mrf_repaired_events(targets: Vec<MrfRepairNoticeTarget>) {
for target in targets {
rustfs_common::mrf_channel::note_mrf_repaired(&target.bucket, &target.object, target.version_id);
rustfs_common::mrf_channel::release_mrf_identity(
target.kind,
&target.bucket,
&target.object,
target.version_id,
target.scope,
target.lease,
);
}
}
fn release_mrf_repair_notice_targets(targets: Vec<MrfRepairNoticeTarget>) {
for target in targets {
rustfs_common::mrf_channel::release_mrf_identity(
target.kind,
&target.bucket,
&target.object,
target.version_id,
target.scope,
target.lease,
);
}
}
+63 -346
View File
@@ -37,7 +37,7 @@ use crate::heal::manager::HealManager;
use metrics::{counter, gauge};
use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult};
use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfIntent};
use std::collections::{HashSet, VecDeque};
use std::collections::VecDeque;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
@@ -48,27 +48,15 @@ use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
/// Journal location inside the metadata bucket, following the resume-state
/// layout.
pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin";
/// The scoped path is the authoritative snapshot for new readers and carries
/// both v1 and v2 records. The legacy path is only a v1 compatibility mirror;
/// older readers ignore the authoritative path, while new readers never merge
/// the two files. This prevents a partial two-file flush from fabricating a
/// mixed epoch.
pub(crate) const MRF_SCOPED_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal-scoped.bin";
/// Record format tag.
const MRF_JOURNAL_FORMAT: u8 = 1;
/// Record layout version.
const MRF_JOURNAL_VERSION: u8 = 1;
const MRF_JOURNAL_VERSION_SCOPED: u8 = 2;
/// Fixed header size: format, version, kind, attempts, enqueued_at_ms,
/// has_version flag.
const MRF_RECORD_FIXED_HEAD: usize = 1 + 1 + 1 + 1 + 8 + 1;
const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
fn metric_f64(value: usize) -> f64 {
f64::from(u32::try_from(value).unwrap_or(u32::MAX))
}
#[derive(Debug, Clone)]
pub(crate) struct MrfConsumerConfig {
@@ -113,90 +101,40 @@ impl Default for MrfConsumerConfig {
/// incoming intent (never a resident one) and counts the loss.
pub(crate) struct MrfQueue {
pending: VecDeque<MrfIntent>,
pending_keys: HashSet<MrfQueueKey>,
bytes: usize,
capacity: usize,
byte_budget: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
struct MrfQueueKey {
kind: rustfs_common::mrf_channel::MrfKind,
bucket: Arc<str>,
object: Arc<str>,
version_id: Option<[u8; 16]>,
scope: Option<rustfs_common::mrf_channel::MrfScope>,
}
fn queue_key(intent: &MrfIntent) -> MrfQueueKey {
let version_id = intent.version_id.filter(|bytes| *bytes != [0; 16]);
let scope = (!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption))
.then_some(intent.scope)
.flatten();
MrfQueueKey {
kind: intent.kind,
bucket: intent.bucket.clone(),
object: intent.object.clone(),
version_id,
scope,
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum MrfQueuePushResult {
Enqueued,
Coalesced,
Rejected,
}
impl MrfQueue {
pub(crate) fn new(capacity: usize, byte_budget: usize) -> Self {
Self {
pending: VecDeque::new(),
pending_keys: HashSet::new(),
bytes: 0,
capacity,
byte_budget,
}
}
pub(crate) fn try_push_typed(&mut self, intent: MrfIntent) -> MrfQueuePushResult {
if intent.bucket.len() > MRF_MAX_IDENTITY_COMPONENT || intent.object.len() > MRF_MAX_IDENTITY_COMPONENT {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "identity_oversized").increment(1);
return MrfQueuePushResult::Rejected;
}
let key = queue_key(&intent);
if self.pending_keys.contains(&key) {
counter!("rustfs_heal_mrf_coalesced_total", "layer" => "queue").increment(1);
return MrfQueuePushResult::Coalesced;
}
/// Returns `false` (after counting) when either ceiling would be crossed.
pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool {
let cost = intent.estimated_bytes();
if self.pending.len() >= self.capacity || self.bytes + cost > self.byte_budget {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1);
return MrfQueuePushResult::Rejected;
return false;
}
self.bytes += cost;
self.pending_keys.insert(key);
self.pending.push_back(intent);
MrfQueuePushResult::Enqueued
}
/// Bool compatibility adapter: only a newly executable queue item is
/// reported as accepted; a coalesced duplicate is not durable admission.
#[cfg(test)]
pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool {
matches!(self.try_push_typed(intent), MrfQueuePushResult::Enqueued)
true
}
pub(crate) fn pop_front(&mut self) -> Option<MrfIntent> {
let intent = self.pending.pop_front()?;
self.pending_keys.remove(&queue_key(&intent));
self.bytes = self.bytes.saturating_sub(intent.estimated_bytes());
Some(intent)
}
pub(crate) fn push_back(&mut self, intent: MrfIntent) {
self.pending_keys.insert(queue_key(&intent));
self.bytes += intent.estimated_bytes();
self.pending.push_back(intent);
}
@@ -219,24 +157,10 @@ impl MrfQueue {
// ---------------------------------------------------------------------------
/// Append one encoded record to `out`.
pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) -> bool {
let Ok(bucket_len) = u32::try_from(intent.bucket.len()) else {
return false;
};
let Ok(object_len) = u32::try_from(intent.object.len()) else {
return false;
};
let scope = (!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption))
.then_some(intent.scope)
.flatten();
let version_id = intent.version_id.filter(|bytes| *bytes != [0; 16]);
pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) {
let start = out.len();
out.push(MRF_JOURNAL_FORMAT);
out.push(if scope.is_some() {
MRF_JOURNAL_VERSION_SCOPED
} else {
MRF_JOURNAL_VERSION
});
out.push(MRF_JOURNAL_VERSION);
out.push(match intent.kind {
rustfs_common::mrf_channel::MrfKind::DecodeFailure => 1,
rustfs_common::mrf_channel::MrfKind::MetadataCorruption => 2,
@@ -244,36 +168,27 @@ pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) -> bool {
});
out.push(intent.attempts);
out.extend_from_slice(&intent.enqueued_at_ms.to_le_bytes());
match version_id {
match intent.version_id {
Some(bytes) => {
out.push(1);
out.extend_from_slice(&bytes);
}
None => out.push(0),
}
if let Some(scope) = scope {
out.extend_from_slice(&scope.pool_index.to_le_bytes());
out.extend_from_slice(&scope.set_index.to_le_bytes());
}
out.extend_from_slice(&bucket_len.to_le_bytes());
out.extend_from_slice(&object_len.to_le_bytes());
out.extend_from_slice(&(intent.bucket.len() as u32).to_le_bytes());
out.extend_from_slice(&(intent.object.len() as u32).to_le_bytes());
out.extend_from_slice(intent.bucket.as_bytes());
out.extend_from_slice(intent.object.as_bytes());
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
hasher.update(&out[start..]);
let Ok(checksum) = u32::try_from(hasher.finalize()) else {
out.truncate(start);
return false;
};
out.extend_from_slice(&checksum.to_le_bytes());
true
out.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes());
}
fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
if data.len() < MRF_RECORD_FIXED_HEAD + 8 {
return None;
}
if data[0] != MRF_JOURNAL_FORMAT || !matches!(data[1], MRF_JOURNAL_VERSION | MRF_JOURNAL_VERSION_SCOPED) {
if data[0] != MRF_JOURNAL_FORMAT || data[1] != MRF_JOURNAL_VERSION {
return None;
}
let kind = match data[2] {
@@ -283,38 +198,24 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
_ => return None,
};
let attempts = data[3];
let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().ok()?);
let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().expect("slice length checked"));
let has_version = data[12] != 0;
let mut cursor = MRF_RECORD_FIXED_HEAD;
let version_id = if has_version {
if data.len() < cursor + 16 {
return None;
}
let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().ok()?;
let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().expect("slice length checked");
cursor += 16;
Some(bytes)
} else {
None
};
let scope = if data[1] == MRF_JOURNAL_VERSION_SCOPED {
if data.len() < cursor + 8 {
return None;
}
let pool_index = u32::from_le_bytes(data[cursor..cursor + 4].try_into().ok()?);
let set_index = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().ok()?);
cursor += 8;
Some(rustfs_common::mrf_channel::MrfScope { pool_index, set_index })
} else {
None
};
if data.len() < cursor + 8 {
return None;
}
let bucket_len = usize::try_from(u32::from_le_bytes(data[cursor..cursor + 4].try_into().ok()?)).ok()?;
let object_len = usize::try_from(u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().ok()?)).ok()?;
if bucket_len > MRF_MAX_IDENTITY_COMPONENT || object_len > MRF_MAX_IDENTITY_COMPONENT {
return None;
}
let bucket_len = u32::from_le_bytes(data[cursor..cursor + 4].try_into().expect("slice length checked")) as usize;
let object_len = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().expect("slice length checked")) as usize;
cursor += 8;
let body_end = cursor.checked_add(bucket_len)?.checked_add(object_len)?;
let record_end = body_end.checked_add(4)?;
@@ -323,7 +224,7 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
}
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
hasher.update(&data[..body_end]);
if u32::try_from(hasher.finalize()).ok()? != u32::from_le_bytes(data[body_end..record_end].try_into().ok()?) {
if (hasher.finalize() as u32) != u32::from_le_bytes(data[body_end..record_end].try_into().expect("slice length checked")) {
return None;
}
let bucket = std::sync::Arc::from(std::str::from_utf8(&data[cursor..cursor + bucket_len]).ok()?);
@@ -334,12 +235,6 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
object,
version_id,
kind,
scope: if matches!(kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) {
None
} else {
scope
},
lease: None,
enqueued_at_ms,
attempts,
},
@@ -374,9 +269,9 @@ async fn journal_disks() -> Vec<DiskStore> {
map.values().flatten().cloned().collect()
}
async fn read_journal(path: &str) -> Option<Vec<u8>> {
async fn read_journal() -> Option<Vec<u8>> {
for disk in journal_disks().await {
match disk.read_all(super::RUSTFS_META_BUCKET, path).await {
match disk.read_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH).await {
Ok(bytes) => return Some(bytes.to_vec()),
Err(_) => continue,
}
@@ -387,51 +282,35 @@ async fn read_journal(path: &str) -> Option<Vec<u8>> {
/// Write the snapshot to every local disk; returns true when at least one
/// disk accepted it, so a total write failure keeps the runtime dirty and
/// the next tick retries the persist.
async fn write_journal(path: &str, data: &[u8]) -> bool {
async fn write_journal(data: &[u8]) -> bool {
let payload = bytes::Bytes::copy_from_slice(data);
let mut any_persisted = false;
for disk in journal_disks().await {
match disk.write_all(super::RUSTFS_META_BUCKET, path, payload.clone()).await {
match disk
.write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone())
.await
{
Ok(()) => any_persisted = true,
Err(err) => warn_mrf_journal_write(&err),
}
}
if !data.is_empty() {
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
}
gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64);
any_persisted
}
async fn delete_journal(path: &str) -> bool {
let disks = journal_disks().await;
if disks.is_empty() {
counter!("rustfs_heal_mrf_journal_delete_failures_total").increment(1);
return false;
}
let mut all_deleted = true;
for disk in disks {
let result = disk
async fn delete_journal() {
for disk in journal_disks().await {
let _ = disk
.delete(
super::RUSTFS_META_BUCKET,
path,
MRF_JOURNAL_PATH,
crate::heal::storage_api::owner::EcstoreDeleteOptions::default(),
)
.await;
if let Err(err) = result {
// Delete is idempotent: a compatibility mirror that was never
// written (or was already removed) is clean, not a retry state.
if !matches!(err, super::DiskError::FileNotFound | super::DiskError::VolumeNotFound) {
all_deleted = false;
}
}
}
if !all_deleted {
counter!("rustfs_heal_mrf_journal_delete_failures_total").increment(1);
}
all_deleted
}
async fn delete_journals() -> bool {
let authoritative_deleted = delete_journal(MRF_SCOPED_JOURNAL_PATH).await;
let legacy_deleted = delete_journal(MRF_JOURNAL_PATH).await;
authoritative_deleted && legacy_deleted
}
fn warn_mrf_journal_write(err: &super::DiskError) {
@@ -452,10 +331,7 @@ fn warn_mrf_journal_write(err: &super::DiskError) {
pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
let bucket = intent.bucket.to_string();
let object = intent.object.to_string();
let version_id = intent
.version_id
.filter(|bytes| *bytes != [0; 16])
.map(|bytes| Uuid::from_bytes(bytes).to_string());
let version_id = intent.version_id.map(|bytes| Uuid::from_bytes(bytes).to_string());
let (heal_type, priority) = match intent.kind {
rustfs_common::mrf_channel::MrfKind::DecodeFailure => (
HealType::ECDecode {
@@ -475,28 +351,18 @@ pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
HealPriority::Normal,
),
};
let mut options = HealOptions::default();
if !matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption)
&& let Some(scope) = intent.scope
{
options.pool_index = usize::try_from(scope.pool_index).ok();
options.set_index = usize::try_from(scope.set_index).ok();
}
let mut request = HealRequest::new(heal_type, options, priority);
let mut request = HealRequest::new(heal_type, HealOptions::default(), priority);
request.source = rustfs_common::heal_channel::HealRequestSource::Mrf;
request
}
async fn submit_mrf_heal_request(manager: &HealManager, intent: &MrfIntent) -> crate::Result<HealAdmissionResult> {
let receipt = manager
.submit_mrf_heal_request_with_receipt_and_identity(
.submit_mrf_heal_request_with_receipt(
build_heal_request(intent),
intent.bucket.clone(),
intent.object.clone(),
intent.version_id,
intent.kind,
intent.scope,
intent.lease,
)
.await?;
Ok(receipt.result)
@@ -521,38 +387,16 @@ struct MrfRuntime {
}
impl MrfRuntime {
fn snapshot(&self) -> (Vec<u8>, Vec<u8>) {
let mut authoritative = Vec::new();
let mut legacy = Vec::new();
fn snapshot(&self) -> Vec<u8> {
let mut buf = Vec::new();
for intent in self.queue.intents() {
let scoped_identity =
!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) && intent.scope.is_some();
if !encode_intent(intent, &mut authoritative) {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "journal_identity_oversized").increment(1);
}
if !scoped_identity && !encode_intent(intent, &mut legacy) {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "journal_identity_oversized").increment(1);
}
encode_intent(intent, &mut buf);
}
(authoritative, legacy)
buf
}
async fn flush(&mut self) {
let (authoritative, legacy) = self.snapshot();
let authoritative_persisted = write_journal(MRF_SCOPED_JOURNAL_PATH, &authoritative).await;
if !authoritative.is_empty() {
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
}
gauge!("rustfs_heal_mrf_journal_bytes").set(metric_f64(authoritative.len()));
// Publish the compatibility mirror only after the authoritative
// snapshot has reached at least one disk. This ordering prevents an
// old reader from observing a newer epoch that a new reader cannot
// see when the canonical write is unavailable.
let legacy_persisted = authoritative_persisted && write_journal(MRF_JOURNAL_PATH, &legacy).await;
// Keep dirty until both the authoritative snapshot and its
// compatibility mirror have been accepted; otherwise a one-sided
// failure would never retry the missing file.
let persisted = authoritative_persisted && legacy_persisted;
let persisted = write_journal(&self.snapshot()).await;
self.new_since_flush = 0;
// Keep the dirty flag when every disk write failed: a clean backlog
// would otherwise never rewrite, losing the periodic persist retry a
@@ -560,7 +404,7 @@ impl MrfRuntime {
if persisted {
self.dirty = false;
}
self.journal_on_disk |= authoritative_persisted || legacy_persisted;
self.journal_on_disk = true;
}
/// Drain pending intents into the heal manager until it is full, the
@@ -586,7 +430,6 @@ impl MrfRuntime {
intent.attempts = intent.attempts.saturating_add(1);
if intent.attempts >= MRF_MAX_ATTEMPTS {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1);
rustfs_common::mrf_channel::release_mrf_intent(&intent);
continue;
}
self.queue.push_back(intent);
@@ -595,13 +438,11 @@ impl MrfRuntime {
}
Ok(HealAdmissionResult::Dropped(_)) => {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "admission_policy").increment(1);
rustfs_common::mrf_channel::release_mrf_intent(&intent);
}
Err(_) => {
intent.attempts = intent.attempts.saturating_add(1);
if intent.attempts >= MRF_MAX_ATTEMPTS {
counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1);
rustfs_common::mrf_channel::release_mrf_intent(&intent);
continue;
}
self.queue.push_back(intent);
@@ -610,8 +451,8 @@ impl MrfRuntime {
}
}
}
gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(self.queue.depth()));
gauge!("rustfs_heal_mrf_queue_bytes").set(metric_f64(self.queue.bytes()));
gauge!("rustfs_heal_mrf_queue_depth").set(self.queue.depth() as f64);
gauge!("rustfs_heal_mrf_queue_bytes").set(self.queue.bytes() as f64);
}
}
@@ -655,12 +496,7 @@ pub async fn replay_journal_once(manager: &Arc<HealManager>) -> usize {
let config = MrfConsumerConfig::default();
let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes);
let mut backoff_until: Option<tokio::time::Instant> = None;
replay_into(manager, &mut queue, &mut backoff_until).await.replayed
}
struct ReplayOutcome {
replayed: usize,
journal_on_disk: bool,
replay_into(manager, &mut queue, &mut backoff_until).await
}
/// Shared replay core: read + decode + re-arm + delete, then drain what fits.
@@ -668,25 +504,11 @@ async fn replay_into(
manager: &Arc<HealManager>,
queue: &mut MrfQueue,
backoff_until: &mut Option<tokio::time::Instant>,
) -> ReplayOutcome {
// The scoped file is a complete authoritative snapshot. Fall back to the
// legacy mirror only when the authoritative path is unavailable; merging
// both files could combine records from different flush epochs.
let data = match read_journal(MRF_SCOPED_JOURNAL_PATH).await {
Some(data) => data,
None => match read_journal(MRF_JOURNAL_PATH).await {
Some(data) => data,
None => {
return ReplayOutcome {
replayed: 0,
journal_on_disk: false,
};
}
},
) -> usize {
let Some(data) = read_journal().await else {
return 0;
};
let mut intents = Vec::new();
let (decoded, truncated) = decode_journal(&data);
intents.extend(decoded);
let (intents, truncated) = decode_journal(&data);
if truncated > 0 {
tracing::warn!(
target: "rustfs::heal::mrf",
@@ -694,15 +516,12 @@ async fn replay_into(
"MRF journal had a torn tail; truncated records were discarded"
);
}
counter!("rustfs_heal_mrf_replayed_total").increment(u64::try_from(intents.len()).unwrap_or(u64::MAX));
counter!("rustfs_heal_mrf_replayed_total").increment(intents.len() as u64);
let replayed = intents.len();
for intent in intents {
let result = queue.try_push_typed(intent.clone());
if !matches!(result, MrfQueuePushResult::Enqueued) {
rustfs_common::mrf_channel::release_mrf_intent(&intent);
}
queue.try_push(intent);
}
let journal_on_disk = !delete_journals().await;
delete_journal().await;
// Drain the replayed intents immediately; whatever the manager refuses
// stays armed in `queue` for the consumer's retry loop.
@@ -722,10 +541,7 @@ async fn replay_into(
}
}
}
ReplayOutcome {
replayed,
journal_on_disk,
}
replayed
}
/// Replay the journal, then keep draining the channel into the heal manager
@@ -743,8 +559,7 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
// Replay: read the journal, re-arm intents (duplicates are merged by the
// manager's dedup key), then drop the file so the next flush starts clean.
let replay = replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
runtime.journal_on_disk = replay.journal_on_disk;
replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
// The replay deleted the journal file; anything still pending (e.g. the
// manager was full and backoff armed) must be re-persisted by the next
// flush or a crash before it would lose those intents.
@@ -772,14 +587,9 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
return;
}
for intent in batch.drain(..) {
match runtime.queue.try_push_typed(intent.clone()) {
MrfQueuePushResult::Enqueued => {
runtime.new_since_flush += 1;
runtime.dirty = true;
}
MrfQueuePushResult::Coalesced | MrfQueuePushResult::Rejected => {
rustfs_common::mrf_channel::release_mrf_intent(&intent);
}
if runtime.queue.try_push(intent) {
runtime.new_since_flush += 1;
runtime.dirty = true;
}
}
runtime.dispatch(manager.as_ref()).await;
@@ -803,14 +613,13 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
TickAction::DeleteJournal => {
// All intents consumed: remove the journal so a restart
// replays nothing (mirrors MinIO's post-replay unlink).
if delete_journals().await {
runtime.journal_on_disk = false;
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
}
delete_journal().await;
runtime.journal_on_disk = false;
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
}
TickAction::Idle => {}
}
gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(runtime.queue.depth()));
gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64);
}
}
}
@@ -855,8 +664,6 @@ mod tests {
object: StdArc::from(object),
version_id: Some([7u8; 16]),
kind: MrfKind::DecodeFailure,
scope: None,
lease: None,
enqueued_at_ms: 1_700_000_000_000,
attempts,
}
@@ -887,101 +694,17 @@ mod tests {
fn queue_enforces_count_and_byte_ceilings() {
let mut queue = MrfQueue::new(2, usize::MAX);
assert!(queue.try_push(intent("b", "o", 0)));
assert!(queue.try_push(intent("b", "o2", 0)));
assert!(!queue.try_push(intent("b", "o3", 0)), "count ceiling must drop");
assert!(queue.try_push(intent("b", "o", 0)));
assert!(!queue.try_push(intent("b", "o", 0)), "count ceiling must drop");
let mut tiny = MrfQueue::new(usize::MAX, intent("bucket", "object", 0).estimated_bytes());
assert!(tiny.try_push(intent("bucket", "object", 0)));
assert!(
!tiny.try_push(intent("bucket", "object2", 0)),
!tiny.try_push(intent("bucket", "object", 0)),
"byte budget must drop before the second intent fits"
);
}
#[test]
fn duplicate_mrf_intents_coalesce_to_one_execution() {
let mut queue = MrfQueue::new(1000, usize::MAX);
let mut enqueued = 0;
let mut coalesced = 0;
assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
enqueued += 1;
for _ in 0..999 {
match queue.try_push_typed(intent("bucket", "object", 0)) {
MrfQueuePushResult::Coalesced => coalesced += 1,
other => panic!("duplicate intent was not coalesced: {other:?}"),
}
}
assert_eq!(enqueued, 1);
assert_eq!(coalesced, 999);
assert_eq!(queue.depth(), 1);
}
#[test]
fn mrf_dedupe_does_not_merge_adjacent_version_pool_or_kind() {
let mut queue = MrfQueue::new(8, usize::MAX);
let mut first = intent("bucket", "object", 0);
first.kind = MrfKind::PartialWrite;
first.scope = Some(rustfs_common::mrf_channel::MrfScope {
pool_index: 1,
set_index: 1,
});
assert!(queue.try_push(first.clone()));
first.version_id = Some([8u8; 16]);
assert!(queue.try_push(first));
let mut other_scope = intent("bucket", "object", 0);
other_scope.kind = MrfKind::PartialWrite;
other_scope.scope = Some(rustfs_common::mrf_channel::MrfScope {
pool_index: 2,
set_index: 1,
});
assert!(queue.try_push(other_scope));
let mut other_kind = intent("bucket", "object", 0);
other_kind.kind = MrfKind::DecodeFailure;
other_kind.scope = None;
assert!(queue.try_push(other_kind));
assert_eq!(queue.depth(), 4);
}
#[test]
fn mrf_dedupe_full_returns_rejected_with_durable_pending() {
let mut queue = MrfQueue::new(1, usize::MAX);
assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
assert_eq!(queue.try_push_typed(intent("bucket", "other", 0)), MrfQueuePushResult::Rejected);
assert_eq!(queue.depth(), 1);
let mut snapshot = Vec::new();
assert!(encode_intent(queue.intents().next().expect("resident intent"), &mut snapshot));
assert!(!snapshot.is_empty(), "the resident intent remains journalable after rejection");
}
#[test]
fn mrf_dedupe_failure_releases_key_for_retry() {
let mut queue = MrfQueue::new(1, usize::MAX);
assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
let _failed = queue.pop_front().expect("queued intent");
assert_eq!(queue.try_push_typed(intent("bucket", "object", 1)), MrfQueuePushResult::Enqueued);
assert_eq!(queue.depth(), 1);
}
#[test]
fn mrf_dedupe_key_and_map_are_bounded() {
let mut queue = MrfQueue::new(2, usize::MAX);
assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
assert_eq!(queue.try_push_typed(intent("bucket", "other", 0)), MrfQueuePushResult::Enqueued);
assert_eq!(queue.pending_keys.len(), 2);
assert_eq!(queue.try_push_typed(intent("bucket", "third", 0)), MrfQueuePushResult::Rejected);
assert_eq!(queue.depth(), 2);
}
#[test]
fn cross_node_duplicate_execution_remains_idempotent() {
// Node-local ingress maps intentionally do not merge across nodes;
// the manager's existing identity key absorbs the duplicate later.
let mut node_a = MrfQueue::new(8, usize::MAX);
let mut node_b = MrfQueue::new(8, usize::MAX);
assert_eq!(node_a.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
assert_eq!(node_b.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
}
#[test]
fn journal_roundtrip_preserves_intents() {
let intents = vec![
@@ -992,8 +715,6 @@ mod tests {
object: StdArc::from("object/c"),
version_id: None,
kind: MrfKind::MetadataCorruption,
scope: None,
lease: None,
enqueued_at_ms: 5,
attempts: 1,
},
@@ -1045,8 +766,6 @@ mod tests {
object: StdArc::from("o"),
version_id: None,
kind: MrfKind::MetadataCorruption,
scope: None,
lease: None,
enqueued_at_ms: 0,
attempts: 0,
});
@@ -1058,8 +777,6 @@ mod tests {
object: StdArc::from("o"),
version_id: None,
kind: MrfKind::PartialWrite,
scope: None,
lease: None,
enqueued_at_ms: 0,
attempts: 0,
});
+1
View File
@@ -51,6 +51,7 @@ const RESUME_STATE_FILE: &str = "ahm_resume_state.json";
const REPLACEMENT_INTENT_FILE: &str = "ahm_replacement_intent.json";
const RESUME_PROGRESS_FILE: &str = "ahm_progress.json";
pub(super) const RESUME_CHECKPOINT_FILE: &str = "ahm_checkpoint.json";
pub(super) const RESUME_CHECKPOINT_BLOCKED_FILE: &str = "ahm_checkpoint.blocked";
const REPLACEMENT_COMPLETION_PROOF_FILE: &str = "ahm_replacement_completion_proof.json";
const REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
const REPLACEMENT_INTENT_SEAL_FILE: &str = "ahm_replacement_intent_seal";
+317 -21
View File
@@ -13,26 +13,31 @@
// limitations under the License.
use crate::{Error, Result};
use base64::Engine as _;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::RwLock;
use tokio::sync::{Mutex as AsyncMutex, RwLock};
use tracing::{debug, warn};
use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
use super::super::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes};
use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt, RUSTFS_META_BUCKET};
use super::{
LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_FILE, delete_resume_file, path_to_str,
validate_resume_task_id,
LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_BLOCKED_FILE, RESUME_CHECKPOINT_FILE,
delete_resume_file, path_to_str, validate_resume_task_id,
};
const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state";
const RESUME_CHECKPOINT_DIGEST_FILE: &str = "ahm_checkpoint.sha256";
const CHECKPOINT_PER_VERSION_SCHEMA: u32 = 5;
/// Current on-disk schema version for `ResumeCheckpoint`. Same rationale as
/// `CURRENT_RESUME_SCHEMA`: pre-per-version dedup identities are not comparable
/// to the new `compose_key` identities, so a stale checkpoint is discarded.
pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 5;
pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 6;
/// resume checkpoint
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -57,6 +62,11 @@ pub struct ResumeCheckpoint {
pub failed_objects: HashSet<String>,
/// skipped objects
pub skipped_objects: HashSet<String>,
/// Integrity digest over the checkpoint with this field set to `None`.
/// Keeping it in the checkpoint makes the payload and its authentication
/// record one CAS generation instead of two independently-written files.
#[serde(default)]
pub integrity_digest: Option<String>,
}
impl ResumeCheckpoint {
@@ -70,6 +80,7 @@ impl ResumeCheckpoint {
processed_objects: HashSet::new(),
failed_objects: HashSet::new(),
skipped_objects: HashSet::new(),
integrity_digest: None,
}
}
@@ -116,17 +127,111 @@ pub struct CheckpointManager {
disk: DiskStore,
checkpoint: Arc<RwLock<ResumeCheckpoint>>,
throttle: Mutex<PersistThrottle>,
save_lock: AsyncMutex<()>,
last_saved: Mutex<Option<EcstoreDiskBytes>>,
}
impl CheckpointManager {
fn blocked_path(task_id: &str) -> std::path::PathBuf {
Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}"))
}
/// Return whether a checkpoint was permanently isolated after a malformed
/// or unsupported snapshot was observed.
pub(crate) async fn is_blocked(disk: &DiskStore, task_id: &str) -> bool {
if validate_resume_task_id(task_id).is_err() {
return false;
}
let blocked_path = Self::blocked_path(task_id);
let Ok(path) = path_to_str(&blocked_path) else {
return false;
};
match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
Ok(_) => true,
Err(crate::heal::DiskError::FileNotFound) => false,
Err(_) => true,
}
}
/// Validate the checkpoint while enumerating resumable state. This reads
/// the checkpoint once and also isolates malformed or unsupported data.
pub(crate) async fn is_resumable(disk: &DiskStore, task_id: &str) -> Result<bool> {
validate_resume_task_id(task_id)?;
if Self::is_blocked(disk, task_id).await {
return Err(Error::InvalidCheckpoint(format!("Resume task {task_id} has a blocked checkpoint")));
}
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
let Ok(path) = path_to_str(&file_path) else {
return Err(Error::InvalidCheckpoint("Resume checkpoint path is not valid UTF-8".to_string()));
};
match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
Ok(bytes) if bytes.is_empty() => Ok(true),
Ok(bytes) => Self::load_from_data(disk.clone(), task_id, bytes.to_vec())
.await
.map(|_| true),
Err(crate::heal::DiskError::FileNotFound) => Ok(true),
Err(error) => Err(error.into()),
}
}
async fn block_invalid_snapshot(disk: &DiskStore, task_id: &str) {
// This marker is intentionally version-agnostic: an unsupported reader
// must stop selector retries until an operator cleans up the snapshot.
let blocked_path = Self::blocked_path(task_id);
let Ok(path) = path_to_str(&blocked_path) else {
return;
};
let result = EcstoreDiskAPI::compare_and_update_file(
disk.as_ref(),
RUSTFS_META_BUCKET,
path,
None,
Some(EcstoreDiskBytes::from_static(b"blocked")),
)
.await;
match result {
Ok(EcstoreConditionalFileUpdate::Updated | EcstoreConditionalFileUpdate::Mismatch) => {}
Ok(EcstoreConditionalFileUpdate::Missing) => warn!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_CHECKPOINT_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_RESUME,
task_id,
state = "blocked_marker_write_failed",
error = "marker target disappeared",
"Heal checkpoint could not persist its blocked marker"
),
Err(error) => warn!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_CHECKPOINT_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_RESUME,
task_id,
state = "blocked_marker_write_failed",
error = %error,
"Heal checkpoint could not persist its blocked marker"
),
}
}
/// create new checkpoint manager
pub async fn new(disk: DiskStore, task_id: String) -> Result<Self> {
validate_resume_task_id(&task_id)?;
let checkpoint_volume = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}");
if let Err(error) = EcstoreDiskAPI::make_volume(disk.as_ref(), &checkpoint_volume).await
&& error != crate::heal::DiskError::VolumeExists
{
return Err(Error::TaskExecutionFailed {
message: format!("Failed to create checkpoint volume: {error}"),
});
}
let checkpoint = ResumeCheckpoint::new(task_id);
let manager = Self {
disk,
checkpoint: Arc::new(RwLock::new(checkpoint)),
throttle: Mutex::new(PersistThrottle::new()),
save_lock: AsyncMutex::new(()),
last_saved: Mutex::new(None),
};
// save initial checkpoint
@@ -140,6 +245,7 @@ impl CheckpointManager {
error = %e,
"Heal checkpoint persistence failed"
);
return Err(e);
}
Ok(manager)
}
@@ -148,11 +254,22 @@ impl CheckpointManager {
pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result<Self> {
validate_resume_task_id(task_id)?;
let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?;
let mut checkpoint: ResumeCheckpoint =
serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to deserialize checkpoint: {e}"),
})?;
Self::load_from_data(disk, task_id, checkpoint_data).await
}
async fn load_from_data(disk: DiskStore, task_id: &str, checkpoint_data: Vec<u8>) -> Result<Self> {
validate_resume_task_id(task_id)?;
let mut checkpoint: ResumeCheckpoint = match serde_json::from_slice(&checkpoint_data) {
Ok(checkpoint) => checkpoint,
Err(error) => {
Self::block_invalid_snapshot(&disk, task_id).await;
return Err(Error::TaskExecutionFailed {
message: format!("Failed to deserialize checkpoint: {error}"),
});
}
};
if checkpoint.task_id != task_id {
Self::block_invalid_snapshot(&disk, task_id).await;
return Err(Error::TaskExecutionFailed {
message: "Resume checkpoint task id does not match filename".to_string(),
});
@@ -163,6 +280,7 @@ impl CheckpointManager {
// identities. Discard the stale sets and position, then stamp the
// current schema so the scan restarts cleanly.
if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA {
Self::block_invalid_snapshot(&disk, task_id).await;
return Err(Error::TaskExecutionFailed {
message: format!(
"Checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}",
@@ -170,7 +288,45 @@ impl CheckpointManager {
),
});
}
if checkpoint.schema_version < CURRENT_CHECKPOINT_SCHEMA {
let integrity_verified = if let Some(expected) = checkpoint.integrity_digest.as_deref() {
let actual = Self::checkpoint_digest(&Self::serialize_without_digest(&checkpoint)?);
if expected != actual {
Self::block_invalid_snapshot(&disk, task_id).await;
return Err(Error::InvalidCheckpoint(format!(
"Resume checkpoint digest does not match task {task_id}"
)));
}
true
} else if checkpoint.schema_version >= CURRENT_CHECKPOINT_SCHEMA {
Self::block_invalid_snapshot(&disk, task_id).await;
return Err(Error::InvalidCheckpoint(format!(
"Resume checkpoint digest is missing for task {task_id}"
)));
} else {
let digest_path = Self::digest_path(task_id);
let digest_path = path_to_str(&digest_path)?;
match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, digest_path).await {
Ok(expected) => {
let actual = Self::checkpoint_digest(&checkpoint_data);
if expected.as_ref() != actual.as_bytes() {
Self::block_invalid_snapshot(&disk, task_id).await;
return Err(Error::InvalidCheckpoint(format!(
"Resume checkpoint digest does not match task {task_id}"
)));
}
true
}
Err(crate::heal::DiskError::FileNotFound) => false,
Err(error) => {
return Err(Error::TaskExecutionFailed {
message: format!("Failed to read checkpoint digest: {error}"),
});
}
}
};
if checkpoint.schema_version < CHECKPOINT_PER_VERSION_SCHEMA || !integrity_verified {
warn!(
target: "rustfs::heal::resume",
event = EVENT_HEAL_CHECKPOINT_STATE,
@@ -187,13 +343,15 @@ impl CheckpointManager {
checkpoint.skipped_objects.clear();
checkpoint.current_bucket_index = 0;
checkpoint.current_object_index = 0;
checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA;
}
checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA;
Ok(Self {
disk,
checkpoint: Arc::new(RwLock::new(checkpoint)),
throttle: Mutex::new(PersistThrottle::new()),
save_lock: AsyncMutex::new(()),
last_saved: Mutex::new(Some(EcstoreDiskBytes::from(checkpoint_data))),
})
}
@@ -204,7 +362,7 @@ impl CheckpointManager {
}
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
match path_to_str(&file_path) {
Ok(path_str) => match disk.read_all(RUSTFS_META_BUCKET, path_str).await {
Ok(path_str) => match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path_str).await {
Ok(data) => !data.is_empty(),
Err(_) => false,
},
@@ -292,6 +450,8 @@ impl CheckpointManager {
let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
delete_resume_file(&self.disk, &checkpoint_file).await?;
delete_resume_file(&self.disk, &Self::digest_path(&task_id)).await?;
delete_resume_file(&self.disk, &Self::blocked_path(&task_id)).await?;
debug!(
target: "rustfs::heal::resume",
@@ -307,21 +467,130 @@ impl CheckpointManager {
/// save checkpoint to disk
async fn save_checkpoint(&self) -> Result<()> {
let checkpoint = self.checkpoint.read().await;
// Serialize saves and take the snapshot only after acquiring the lock:
// a slower writer must not publish a snapshot taken before a newer one.
let _save_guard = self.save_lock.lock().await;
let checkpoint = self.checkpoint.read().await.clone();
validate_resume_task_id(&checkpoint.task_id)?;
let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to serialize checkpoint: {e}"),
})?;
let unsigned_checkpoint_data = Self::serialize_without_digest(&checkpoint)?;
let digest = Self::checkpoint_digest(&unsigned_checkpoint_data);
let mut persisted_checkpoint = checkpoint.clone();
persisted_checkpoint.integrity_digest = Some(digest);
let checkpoint_data =
EcstoreDiskBytes::from(serde_json::to_vec(&persisted_checkpoint).map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to serialize checkpoint: {e}"),
})?);
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", checkpoint.task_id, RESUME_CHECKPOINT_FILE));
let path_str = path_to_str(&file_path)?;
self.disk
.write_all(RUSTFS_META_BUCKET, path_str, checkpoint_data.into())
let last_saved = self
.last_saved
.lock()
.map_err(|_| Error::TaskExecutionFailed {
message: "Checkpoint save state lock is poisoned; refusing to save".to_string(),
})?
.clone();
let update = EcstoreDiskAPI::compare_and_update_file(
self.disk.as_ref(),
RUSTFS_META_BUCKET,
path_str,
last_saved.clone(),
Some(checkpoint_data.clone()),
)
.await
.map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to save checkpoint: {e}"),
})?;
let expected = match update {
EcstoreConditionalFileUpdate::Updated => None,
EcstoreConditionalFileUpdate::Missing => {
return Err(Error::TaskExecutionFailed {
message: "Checkpoint was removed after this manager saved it; refusing to recreate it".to_string(),
});
}
EcstoreConditionalFileUpdate::Mismatch => {
// A healthy manager normally completes the CAS above without
// another read or JSON parse. Inspect only after a mismatch so
// corruption and future schemas cannot be overwritten blindly.
let existing = match HealDiskExt::read_all(self.disk.as_ref(), RUSTFS_META_BUCKET, path_str).await {
Ok(existing) => existing,
Err(crate::heal::DiskError::FileNotFound) => {
return Err(Error::TaskExecutionFailed {
message: "Checkpoint was removed after this manager saved it; refusing to recreate it".to_string(),
});
}
Err(error) => {
return Err(Error::TaskExecutionFailed {
message: format!("Failed to inspect checkpoint after CAS mismatch: {error}"),
});
}
};
if existing.is_empty() && last_saved.is_none() {
Some(existing)
} else {
let current: ResumeCheckpoint = match serde_json::from_slice(&existing) {
Ok(current) => current,
Err(error) => {
Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
return Err(Error::TaskExecutionFailed {
message: format!("Existing checkpoint is corrupt: {error}"),
});
}
};
if current.task_id != checkpoint.task_id {
Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
return Err(Error::TaskExecutionFailed {
message: "Existing checkpoint task id does not match filename".to_string(),
});
}
if current.schema_version > CURRENT_CHECKPOINT_SCHEMA {
Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
return Err(Error::TaskExecutionFailed {
message: format!(
"Existing checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}",
current.schema_version
),
});
}
if last_saved.as_ref().is_none_or(|saved| saved.as_ref() != existing.as_ref()) {
return Err(Error::TaskExecutionFailed {
message: "Checkpoint changed since this manager loaded it; refusing to overwrite newer progress"
.to_string(),
});
}
Some(existing)
}
}
};
if let Some(expected) = expected {
match EcstoreDiskAPI::compare_and_update_file(
self.disk.as_ref(),
RUSTFS_META_BUCKET,
path_str,
Some(expected),
Some(checkpoint_data.clone()),
)
.await
.map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to save checkpoint: {e}"),
})?;
message: format!("Failed to save checkpoint after CAS mismatch: {e}"),
})? {
EcstoreConditionalFileUpdate::Updated => {}
EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch => {
return Err(Error::TaskExecutionFailed {
message: "Checkpoint changed while saving; refusing to overwrite newer progress".to_string(),
});
}
}
}
let mut last_saved = self.last_saved.lock().map_err(|_| Error::TaskExecutionFailed {
message: "Checkpoint save state lock is poisoned after save".to_string(),
})?;
*last_saved = Some(checkpoint_data);
debug!(
target: "rustfs::heal::resume",
@@ -341,11 +610,38 @@ impl CheckpointManager {
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
let path_str = path_to_str(&file_path)?;
disk.read_all(RUSTFS_META_BUCKET, path_str)
HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path_str)
.await
.map(|bytes| bytes.to_vec())
.map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to read checkpoint file: {e}"),
})
}
fn serialize_without_digest(checkpoint: &ResumeCheckpoint) -> Result<Vec<u8>> {
let mut unsigned = checkpoint.clone();
unsigned.integrity_digest = None;
let mut value = serde_json::to_value(&unsigned).map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to serialize checkpoint: {e}"),
})?;
for field in ["processed_objects", "failed_objects", "skipped_objects"] {
let Some(values) = value.get_mut(field).and_then(serde_json::Value::as_array_mut) else {
return Err(Error::TaskExecutionFailed {
message: format!("Failed to canonicalize checkpoint field: {field}"),
});
};
values.sort_by(|left, right| left.as_str().cmp(&right.as_str()));
}
serde_json::to_vec(&value).map_err(|e| Error::TaskExecutionFailed {
message: format!("Failed to serialize checkpoint: {e}"),
})
}
fn checkpoint_digest(checkpoint_data: &[u8]) -> String {
base64::engine::general_purpose::STANDARD.encode(Sha256::digest(checkpoint_data))
}
fn digest_path(task_id: &str) -> std::path::PathBuf {
Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_DIGEST_FILE}"))
}
}
+389
View File
@@ -1600,6 +1600,32 @@ async fn test_checkpoint_schema_v4_discarded_on_load() {
temp_dir.close().expect("remove schema test directory");
}
#[tokio::test]
async fn downgraded_unsigned_checkpoint_resets_untrusted_progress() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
manager.add_processed_object("victim-a".to_string()).await.unwrap();
manager.update_position(2, 500).await.unwrap();
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
let bytes = disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path).await.unwrap();
let mut downgraded: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
downgraded["schema_version"] = serde_json::json!(CURRENT_CHECKPOINT_SCHEMA - 1);
downgraded.as_object_mut().unwrap().remove("integrity_digest");
downgraded["processed_objects"] = serde_json::json!(["victim-b"]);
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, serde_json::to_vec(&downgraded).unwrap().into())
.await
.expect("write downgraded checkpoint");
let manager = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap();
let checkpoint = manager.get_checkpoint().await;
assert_eq!(checkpoint.schema_version, CURRENT_CHECKPOINT_SCHEMA);
assert_eq!(checkpoint.current_bucket_index, 0);
assert_eq!(checkpoint.current_object_index, 0);
assert!(checkpoint.processed_objects.is_empty());
temp_dir.close().unwrap();
}
#[tokio::test]
async fn current_normal_resume_schema_preserves_progress() {
let (temp_dir, disk) = schema_test_disk().await;
@@ -1675,6 +1701,369 @@ async fn future_resume_and_checkpoint_schemas_are_rejected() {
temp_dir.close().expect("remove schema test directory");
}
#[tokio::test]
async fn checkpoint_save_does_not_replace_a_non_empty_truncated_snapshot() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create checkpoint manager");
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
let truncated = b"{\"schema_version\":5,\"task_id\":";
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, truncated.as_slice().into())
.await
.expect("write truncated checkpoint fixture");
let error = manager
.update_position(2, 7)
.await
.expect_err("a truncated checkpoint must fail closed during save");
assert!(error.to_string().contains("Existing checkpoint is corrupt"));
assert_eq!(
disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
.await
.expect("read truncated checkpoint fixture"),
truncated.as_slice()
);
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
temp_dir.close().expect("remove checkpoint save test directory");
}
#[tokio::test]
async fn checkpoint_save_does_not_replace_a_future_schema_snapshot() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create checkpoint manager");
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
let mut future = ResumeCheckpoint::new(task_id.clone());
future.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1;
let future_bytes = serde_json::to_vec(&future).expect("serialize future checkpoint fixture");
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, future_bytes.clone().into())
.await
.expect("write future checkpoint fixture");
let error = manager
.update_position(2, 7)
.await
.expect_err("a future schema must fail closed during save");
assert!(error.to_string().contains("Existing checkpoint schema"));
assert_eq!(
disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
.await
.expect("read future checkpoint fixture"),
future_bytes
);
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
temp_dir.close().expect("remove future schema test directory");
}
#[tokio::test]
async fn checkpoint_digest_rejects_same_length_progress_tampering() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create checkpoint manager");
manager
.add_processed_object("victim-a".to_string())
.await
.expect("persist checkpoint progress");
manager.update_position(1, 1).await.expect("flush checkpoint progress");
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
let original = disk
.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
.await
.expect("read checkpoint fixture");
let tampered = original
.windows(b"victim-a".len())
.position(|window| window == b"victim-a")
.map(|index| {
let mut bytes = original.to_vec();
bytes[index..index + b"victim-a".len()].copy_from_slice(b"victim-b");
bytes
})
.expect("checkpoint should contain the processed object");
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, tampered.into())
.await
.expect("write tampered checkpoint fixture");
assert!(CheckpointManager::load_from_disk(disk.clone(), &task_id).await.is_err());
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
temp_dir.close().expect("remove digest test directory");
}
#[tokio::test]
async fn checkpoint_integrity_survives_missing_legacy_sidecar() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
manager.update_position(2, 9).await.unwrap();
let digest_path = format!("{BUCKET_META_PREFIX}/{task_id}_ahm_checkpoint.sha256");
delete_resume_file(&disk, Path::new(&digest_path)).await.unwrap();
let restored = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap();
let checkpoint = restored.get_checkpoint().await;
assert_eq!(checkpoint.current_bucket_index, 2);
assert_eq!(checkpoint.current_object_index, 9);
assert!(checkpoint.integrity_digest.is_some());
temp_dir.close().unwrap();
}
#[tokio::test]
async fn checkpoint_integrity_survives_multi_object_reload() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
for index in 0..32 {
manager.add_processed_object(format!("processed-{index}")).await.unwrap();
manager.add_failed_object(format!("failed-{index}")).await.unwrap();
manager.add_skipped_object(format!("skipped-{index}")).await.unwrap();
}
manager.update_position(2, 9).await.unwrap();
CheckpointManager::load_from_disk(disk, &task_id)
.await
.expect("a healthy multi-object checkpoint must survive reload");
temp_dir.close().unwrap();
}
#[tokio::test]
async fn checkpoint_integrity_rejects_a_removed_embedded_digest() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
manager.update_position(2, 9).await.unwrap();
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
let bytes = disk
.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
.await
.expect("read checkpoint fixture");
let mut value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
value["current_object_index"] = serde_json::json!(10);
value.as_object_mut().unwrap().remove("integrity_digest");
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, serde_json::to_vec(&value).unwrap().into())
.await
.expect("write tampered checkpoint fixture");
assert!(
CheckpointManager::load_from_disk(disk.clone(), &task_id).await.is_err(),
"a current checkpoint without its embedded digest must fail closed"
);
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
temp_dir.close().unwrap();
}
#[tokio::test]
async fn new_checkpoint_manager_rebuilds_an_empty_snapshot() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, EcstoreDiskBytes::new())
.await
.expect("write empty checkpoint fixture");
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("a new manager must rebuild an empty checkpoint");
manager
.update_position(3, 11)
.await
.expect("rebuilt checkpoint must remain writable");
assert!(CheckpointManager::has_checkpoint(&disk, &task_id).await);
temp_dir.close().expect("remove empty checkpoint test directory");
}
#[tokio::test]
async fn deleted_checkpoint_is_not_recreated_by_an_old_manager() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create checkpoint manager");
manager.cleanup().await.expect("delete checkpoint fixture");
let error = manager
.update_position(1, 2)
.await
.expect_err("an old manager must not resurrect a deleted checkpoint");
assert!(error.to_string().contains("removed after this manager saved it"));
assert!(!CheckpointManager::has_checkpoint(&disk, &task_id).await);
temp_dir.close().expect("remove deleted checkpoint test directory");
}
#[cfg(unix)]
#[tokio::test]
async fn checkpoint_cleanup_leaves_no_task_specific_lock_artifact() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create checkpoint manager");
let lock_path = Path::new(BUCKET_META_PREFIX)
.join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"))
.with_extension("rustfs-cas.lock");
let lock_path = temp_dir.path().join(RUSTFS_META_BUCKET).join(lock_path);
manager.cleanup().await.expect("delete checkpoint fixture");
assert!(
!lock_path.exists(),
"successful checkpoint cleanup must not leave a task-specific lock artifact"
);
}
#[tokio::test]
async fn an_empty_blocked_marker_still_blocks_resume_selection() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create checkpoint manager");
let blocked_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
disk.write_all(RUSTFS_META_BUCKET, &blocked_path, EcstoreDiskBytes::new())
.await
.expect("write empty blocked marker fixture");
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
assert!(CheckpointManager::is_resumable(&disk, &task_id).await.is_err());
// Recovery requires replacing/cleaning the snapshot, then removing the
// marker; ordinary selector retries are intentionally not an unlock path.
manager.cleanup().await.expect("clean blocked checkpoint");
assert!(!CheckpointManager::is_blocked(&disk, &task_id).await);
temp_dir.close().expect("remove empty blocked marker test directory");
}
#[tokio::test]
async fn resumable_selector_skips_healthy_tasks_with_blocked_markers() {
let (temp_dir, disk) = schema_test_disk().await;
let tasks = [
(ResumeUtils::generate_task_id(), EcstoreDiskBytes::new()),
(ResumeUtils::generate_task_id(), EcstoreDiskBytes::from_static(b"blocked")),
];
for (task_id, marker) in &tasks {
ResumeManager::new(
disk.clone(),
task_id.clone(),
"erasure_set".to_string(),
"pool_0_set_0".to_string(),
vec!["bucket".to_string()],
)
.await
.expect("create healthy resume state");
CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create healthy checkpoint");
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
let checkpoint_bytes = disk
.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
.await
.expect("read healthy checkpoint before blocking");
let marker_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
disk.write_all(RUSTFS_META_BUCKET, &marker_path, marker.clone())
.await
.expect("write blocked marker");
assert!(ResumeUtils::get_resumable_tasks(&disk).await.is_err());
assert_eq!(
disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
.await
.expect("read healthy checkpoint after blocking"),
checkpoint_bytes
);
}
temp_dir.close().expect("remove blocked selector test directory");
}
#[tokio::test]
async fn stale_checkpoint_manager_cannot_overwrite_newer_progress() {
let (temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let first = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("create first checkpoint manager");
let second = CheckpointManager::load_from_disk(disk.clone(), &task_id)
.await
.expect("load second checkpoint manager");
second
.update_position(4, 20)
.await
.expect("persist newer checkpoint progress");
let error = first
.update_position(1, 3)
.await
.expect_err("stale checkpoint manager must not overwrite newer progress");
assert!(error.to_string().contains("newer progress"));
let persisted = CheckpointManager::load_from_disk(disk.clone(), &task_id)
.await
.expect("load newer checkpoint progress")
.get_checkpoint()
.await;
assert_eq!(persisted.current_bucket_index, 4);
assert_eq!(persisted.current_object_index, 20);
temp_dir.close().expect("remove stale manager test directory");
}
#[tokio::test]
async fn resumable_selector_isolates_future_and_corrupt_checkpoints() {
let (temp_dir, disk) = schema_test_disk().await;
let future_task = ResumeUtils::generate_task_id();
let corrupt_task = ResumeUtils::generate_task_id();
for task_id in [&future_task, &corrupt_task] {
ResumeManager::new(
disk.clone(),
task_id.to_string(),
"erasure_set".to_string(),
"pool_0_set_0".to_string(),
vec!["bucket".to_string()],
)
.await
.expect("create resumable state fixture");
}
let future_path = format!("{BUCKET_META_PREFIX}/{future_task}_{RESUME_CHECKPOINT_FILE}");
let mut future = ResumeCheckpoint::new(future_task.clone());
future.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1;
let future_bytes = serde_json::to_vec(&future).expect("serialize future checkpoint fixture");
disk.write_all(RUSTFS_META_BUCKET, &future_path, future_bytes.clone().into())
.await
.expect("write future checkpoint fixture");
let corrupt_path = format!("{BUCKET_META_PREFIX}/{corrupt_task}_{RESUME_CHECKPOINT_FILE}");
let corrupt_bytes = b"{truncated";
disk.write_all(RUSTFS_META_BUCKET, &corrupt_path, corrupt_bytes.as_slice().into())
.await
.expect("write corrupt checkpoint fixture");
assert!(CheckpointManager::is_resumable(&disk, &future_task).await.is_err());
assert!(CheckpointManager::is_resumable(&disk, &corrupt_task).await.is_err());
assert!(ResumeUtils::get_resumable_tasks(&disk).await.is_err());
for (task_id, path, bytes) in [
(&future_task, future_path, future_bytes),
(&corrupt_task, corrupt_path, corrupt_bytes.to_vec()),
] {
assert_eq!(
disk.read_all(RUSTFS_META_BUCKET, &path)
.await
.expect("read isolated checkpoint bytes"),
bytes
);
let blocked_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
assert!(
!disk
.read_all(RUSTFS_META_BUCKET, &blocked_path)
.await
.expect("read checkpoint blocked marker")
.is_empty()
);
}
temp_dir.close().expect("remove selector isolation test directory");
}
#[test]
fn test_persist_throttle_batches_until_threshold() {
let mut throttle = PersistThrottle::new();
+2 -1
View File
@@ -21,7 +21,7 @@ use uuid::Uuid;
use super::super::{BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
use super::replacement::{ReplacementPhase, ReplacementRecoveryRecord};
use super::{
EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE,
CheckpointManager, EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE,
REPLACEMENT_INTENT_FILE, RESUME_STATE_FILE, ResumeManager, ResumeStateFile, is_replacement_intent, path_to_str,
replacement_recovery_corruption_for_state_load, replacement_recovery_dir, validate_resume_task_id,
};
@@ -67,6 +67,7 @@ impl ResumeUtils {
// Extract task ID from filename: {task_id}_ahm_resume_state.json
if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}"))
&& validate_resume_task_id(task_id).is_ok()
&& CheckpointManager::is_resumable(disk, task_id).await?
{
task_ids.push(task_id.to_string());
}
+2 -72
View File
@@ -35,7 +35,6 @@ use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, Pool
const META_BUCKET: &str = ".rustfs.sys";
const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin";
const SCOPED_JOURNAL_REL: &str = "buckets/.heal/mrf/journal-scoped.bin";
async fn heal_env() -> (Vec<std::path::PathBuf>, Arc<dyn HealStorageAPI>) {
let env = rustfs_test_utils::TestECStoreEnv::builder()
@@ -80,18 +79,14 @@ fn journal_record(kind: u8, bucket: &str, object: &str, version: Option<[u8; 16]
body
}
fn write_journal_path_to_disks(disk_paths: &[std::path::PathBuf], relative_path: &str, data: &[u8]) {
fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
for path in disk_paths {
let journal = path.join(META_BUCKET).join(relative_path);
let journal = path.join(META_BUCKET).join(JOURNAL_REL);
std::fs::create_dir_all(journal.parent().expect("journal parent")).expect("create journal dir");
std::fs::write(&journal, data).expect("write journal fixture");
}
}
fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
write_journal_path_to_disks(disk_paths, JOURNAL_REL, data);
}
async fn wait_until<F, Fut>(deadline: Duration, mut probe: F) -> bool
where
F: FnMut() -> Fut,
@@ -192,73 +187,8 @@ async fn journal_replay_arms_intents_and_deletes_the_file() {
.all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()),
"the journal file must be removed after a successful replay"
);
assert!(
disk_paths
.iter()
.all(|path| !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()),
"the authoritative journal file must also be removed after replay"
);
let snapshot = manager.operations_snapshot().await;
assert_eq!(snapshot.queued_by_priority.urgent, 1, "the decode-failure record must replay as Urgent");
assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal");
}
/// A canonical snapshot and its compatibility mirror may differ after a
/// partial flush. Replay must choose the complete canonical epoch instead of
/// combining records that never coexisted in memory.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
let (disk_paths, storage) = heal_env().await;
let mut endpoints: Vec<Endpoint> = disk_paths
.iter()
.map(|p| Endpoint::try_from(p.to_string_lossy().as_ref()).expect("endpoint from disk path"))
.collect();
for (i, endpoint) in endpoints.iter_mut().enumerate() {
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(i);
}
let pool = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: endpoints.len(),
endpoints: Endpoints::from(endpoints),
cmd_line: "mrf-authoritative-test".to_string(),
platform: String::new(),
};
init_local_disks(EndpointServerPools::from(vec![pool]))
.await
.expect("local disks should register");
let authoritative = journal_record(1, "authoritative-bucket", "authoritative-object", None, 0);
let legacy = journal_record(1, "legacy-bucket", "legacy-object", None, 0);
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &authoritative);
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &legacy);
let manager = make_manager(storage);
let replayed = mrf_queue::replay_journal_once(&manager).await;
assert_eq!(replayed, 1, "only the authoritative snapshot epoch may replay");
let snapshot = manager.operations_snapshot().await;
assert_eq!(snapshot.queued_by_source.mrf, 1);
assert!(
disk_paths.iter().all(|path| {
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
}),
"replay cleanup must remove both journal paths"
);
// A scoped-only snapshot is valid during a rollout where no legacy
// compatibility mirror was written. Missing legacy files must not leave
// the runtime in a permanent cleanup-retry state.
let scoped_only = journal_record(1, "scoped-only-bucket", "scoped-only-object", None, 0);
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &scoped_only);
assert_eq!(mrf_queue::replay_journal_once(&manager).await, 1);
assert!(disk_paths.iter().all(|path| {
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
}));
}
+2 -3
View File
@@ -1375,14 +1375,13 @@ impl FolderScanner {
// Single-flight (backlog#1894 axis A) — the
// recording mode and its guarantees are pinned by
// corrupt_metadata_recording below.
let mrf_result = rustfs_common::mrf_channel::try_send_mrf_intent_typed(
let mrf_accepted = rustfs_common::mrf_channel::try_send_mrf_intent(
rustfs_common::mrf_channel::MrfKind::MetadataCorruption,
&item.bucket,
&object,
None,
None,
);
match corrupt_metadata_recording(mrf_result) {
match corrupt_metadata_recording(mrf_accepted) {
CorruptMetadataRecording::LedgerOnly => {
// Recorded as Full (retry-later): admission
// for this target happens in the MRF
@@ -50,12 +50,11 @@ pub(super) enum CorruptMetadataRecording {
ImmediateAndLedger,
}
pub(super) fn corrupt_metadata_recording(result: rustfs_common::mrf_channel::MrfIngressResult) -> CorruptMetadataRecording {
match result {
rustfs_common::mrf_channel::MrfIngressResult::Enqueued | rustfs_common::mrf_channel::MrfIngressResult::Coalesced => {
CorruptMetadataRecording::LedgerOnly
}
rustfs_common::mrf_channel::MrfIngressResult::Dropped(_) => CorruptMetadataRecording::ImmediateAndLedger,
pub(super) fn corrupt_metadata_recording(mrf_accepted: bool) -> CorruptMetadataRecording {
if mrf_accepted {
CorruptMetadataRecording::LedgerOnly
} else {
CorruptMetadataRecording::ImmediateAndLedger
}
}
+2 -14
View File
@@ -48,20 +48,8 @@ fn scanner_alert_wire_names_match_canonical_event_names() {
/// the backstop survives regardless of delivery.
#[test]
fn corrupt_metadata_recording_maps_delivery_to_backstop() {
assert_eq!(
corrupt_metadata_recording(rustfs_common::mrf_channel::MrfIngressResult::Enqueued),
CorruptMetadataRecording::LedgerOnly
);
assert_eq!(
corrupt_metadata_recording(rustfs_common::mrf_channel::MrfIngressResult::Coalesced),
CorruptMetadataRecording::LedgerOnly
);
assert_eq!(
corrupt_metadata_recording(rustfs_common::mrf_channel::MrfIngressResult::Dropped(
rustfs_common::mrf_channel::MrfDropReason::Full
)),
CorruptMetadataRecording::ImmediateAndLedger
);
assert_eq!(corrupt_metadata_recording(true), CorruptMetadataRecording::LedgerOnly);
assert_eq!(corrupt_metadata_recording(false), CorruptMetadataRecording::ImmediateAndLedger);
}
fn cooldown_map_len() -> usize {
-3
View File
@@ -43,9 +43,6 @@ allow-git = [
# RustFS fork carrying presigned expiry and constant-time authentication fixes.
# owner: rustfs-maintainers review: 2026-10
"https://github.com/rustfs/s3s.git",
# MiMalloc fork pinned for hotpath allocation counting support.
# owner: houseme review: 2026-10
"https://github.com/xonatius/mimalloc_rust.git",
]
[bans]
+149
View File
@@ -0,0 +1,149 @@
# CI gate matrix
This file is the source of truth for which validation runs on each event, its
configured wall-clock budget, and whether it can block a merge. Test taxonomy,
naming, and nextest serialization rules remain in [README.md](README.md); e2e
membership and counts remain in
[e2e-suite-inventory.md](e2e-suite-inventory.md).
The distinction between **required** and **report-only** is load-bearing:
a failing job blocks a merge only when its exact check name is present in the
live `main` ruleset. A workflow name, a `merge_group` trigger, or a red PR check
does not make a job required by itself.
## Required merge checks
The live `main` ruleset (`6436880`) currently requires exactly these contexts:
| Required context | Producer | Validation |
|---|---|---|
| `CLA Check` | `.github/workflows/cla.yml` | Contributor agreement |
| `Quick Checks` | `.github/workflows/ci.yml` | Formatting and repository guard scripts |
| `Test and Lint` | `.github/workflows/ci.yml` | Clippy, workspace nextest excluding `e2e_test`, doctests, and migration proofs |
For pull requests limited to the paths excluded by the main CI workflow,
`.github/workflows/ci-docs-only.yml` reports `Quick Checks` and
`Test and Lint` under the same names. It runs the real quick checks and the
planning-document guard; it does not claim that Rust compilation or runtime
tests ran. Despite the workflow name, these paths also include selected deploy,
workflow, and lock files.
Verify the live rule rather than trusting this snapshot before changing merge
policy:
```bash
gh api repos/rustfs/rustfs/rulesets/6436880 \
--jq '.rules[] | select(.type == "required_status_checks") | .parameters'
```
The ruleset currently has `strict_required_status_checks_policy=false`.
`Continuous Integration` accepts `merge_group` events and runs `e2e-full` for
them, but `End-to-End Tests (full merge gate)` is not currently a required
context. Therefore the repository is prepared to test a merge-queue SHA, but
the workflow alone does not prove that every merge passed that lane.
## Pull request and merge matrix
Budgets below are job `timeout-minutes`, not typical runtimes. “Report-only”
means the result is visible and actionable but is not in the live required
context list.
| Event | Validation | Budget | Merge status | Reproduction |
|---|---|---:|---|---|
| PR, non-doc change | `Quick Checks` | 10 min | Required | `make pre-commit` (broader local umbrella) |
| PR, non-doc change | `Test and Lint` | 90 min | Required | `cargo nextest run --profile ci --all --exclude e2e_test` |
| PR, non-doc change | `Typos` | 10 min | Report-only | `typos` |
| PR, non-doc change | `ILM Integration (serial)` | 90 min | Report-only | Use the exact command in `.github/workflows/ci.yml` |
| PR, non-doc change | rio-v2 / swift / sftp test-and-lint variants | 90 min each | Report-only | `cargo nextest run` with the workflow's feature set |
| PR, non-doc change | `Build RustFS Debug Binary` | 30 min | Report-only; prerequisite for black-box lanes | `cargo build -p rustfs --bins` |
| PR, non-doc change | `io_uring Integration (real)` | 30 min | Report-only | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
| PR, non-doc change | `End-to-End Tests` (`e2e-smoke` plus `s3s-e2e`) | 30 min | Report-only | `cargo nextest run --profile e2e-smoke -p e2e_test`; then `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` |
| PR, non-doc change | `S3 Implemented Tests` | 60 min | Report-only | Build `rustfs`, then run `scripts/s3-tests/run.sh` with `DEPLOY_MODE=binary`, `TEST_MODE=single`, and `MAXFAIL=0` |
| PR, non-doc change | `S3 Lifecycle Behavior Tests` | 30 min | Report-only | Use the accelerated scanner environment in `.github/workflows/ci.yml` with `scripts/s3-tests/run.sh` |
| PR touching dependency or workflow inputs | Cargo Deny / Workflow Pin Report / Dependency Review | 20 / 5 / 30 min | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
| PR touching architecture rules or architecture docs | `Architecture Migration Rules` | 10 min | Report-only | `scripts/check_architecture_migration_rules.sh` |
| PR touching Nix or workspace manifests | `Nix Build & Check` | 60 min | Report-only | `nix flake check` |
| PR limited to main-CI-excluded paths | companion `Quick Checks` and `Test and Lint` | 10 min each | Required | `git diff --check`; `make doc-paths-check` when documentation paths changed |
| `merge_group` | Standard CI plus `e2e-full` | 55 min for `e2e-full` | Standard required contexts only; `e2e-full` report-only | `cargo nextest run --profile e2e-full -p e2e_test` |
| Push to `main` | Standard CI plus `e2e-full` | 55 min for `e2e-full` | Post-merge detection | Same as `merge_group` |
| PR touching fuzz inputs or harness paths | Build plus five 60-second fuzz smoke targets | 60 min build; 30 min per target | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` |
| PR touching selected ecstore disk/format paths | `Rename Safety` on Windows | 60 min | Report-only | Run the four `cargo test -p rustfs-ecstore --lib <filter>` commands in `windows-filesystem.yml` on Windows |
The authoritative e2e filters live in `.config/nextest.toml`; extend a profile
instead of adding a second ad-hoc selector. Before a profile runs,
`scripts/check_test_wiring.py` compares its exact membership to the committed
digest so a silent test drop fails closed.
## Scheduled and manual validation
Scheduled lanes are independent fault domains. They do not block a pull
request, but their workflow-local gate can fail the run and scheduled failures
are routed to the shared failure-issue action. The scheduled-validation
watchdog and freshness workflow separately detect incomplete runs and missing
schedules.
| Cadence (UTC unless noted) | Workflow / validation | Budget | Verdict and artifacts | Reproduction |
|---|---|---:|---|---|
| Daily 02:17 | Fuzz: five nightly corpus targets | 60 min build; 60 min per target | Gate; corpus/crash artifacts, scheduled failure alert | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
| Daily 03:17 | MinIO interop (EC + SSE read parity) | 40 min | Gate; scheduled failure alert | Dispatch `minio-interop.yml` or follow its pinned Docker fixture steps |
| Daily 04:29 | Replication / cluster-fault / protocol e2e | 45 / 90 / 90 min | Three independent gates; JUnit, membership, and server logs | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` |
| Daily 06:31 | Warp performance A/B | 180 min | Regression budget gate; A/B summaries and server logs | `bash scripts/run_hotpath_warp_abba.sh --help` |
| Daily 00:07 Asia/Shanghai (16:07 UTC previous day) | Nightly GNU build and Vault lanes | 150 / 90 / 60 min | Build, live Vault, and HA failover gates | Use the commands and pinned Vault images in `nightly-gnu.yml` |
| Daily 03:23 | Security Audit | 20 / 5 min, plus 30 min on PR dependency review | Cargo Deny and workflow-pin gates; scheduled failure alert | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
| Daily 23:47 | Scheduled Validation Freshness | 10 min | Fails when a critical schedule was never created or is stale | Dispatch `scheduled-validation-freshness.yml` |
| Sunday 00:11 | Full `Continuous Integration` matrix | Per-job budgets above | Weekly variant coverage, including dormant rio-v2 binary/e2e lanes | Dispatch `ci.yml` |
| Sunday 01:13 | Seven-platform build matrix | 150 min per platform | Build/package integrity; scheduled failure alert | Dispatch `build.yml` with an exact platform set |
| Sunday 02:19 | Ceph s3-tests full sweep: single and real four-node, four shards each | 180 min per shard | Compatibility gate; report, JUnit, exact node IDs, and server logs | `scripts/s3-tests/run.sh` against an existing single or distributed target |
| Sunday 06:41 | Mint | 120 min | **Report-only by design**; per-suite PASS/FAIL/NA and raw `log.json` | Reproduce the pinned Docker sequence in `mint.yml` or dispatch it |
| Sunday 07:43 | Workspace line coverage | 120 min | Report-only trend; lcov and JSON retained 90 days | `make coverage` |
| Monthly, day 1 06:37 | Runner Hygiene | 15 min | Validates runner ephemerality; scheduled failure alert | Dispatch `runner-hygiene.yml` |
Manual `workflow_dispatch` exists for the scheduled workflows above. Manual
runs are debugging evidence and intentionally do not open scheduled-failure
issues. A manual performance run may explicitly allow a known regression; that
override must not be treated as an ordinary passing baseline.
## Release validation
Release validation is post-merge and tag-driven; it does not substitute for a
pull-request gate.
| Event | Validation | Budget | Result |
|---|---|---:|---|
| Push to `main` or weekly schedule | `Build and Release` platform matrix | 150 min per platform | Build artifacts for all selected targets; no release publication on a main push |
| Valid release or preview tag | `Build and Release` plus asset checks | 150 min per platform | Draft release, checksummed assets, and publish step |
| Successful non-preview release-tag build | Docker image build and image scan | 60 min build; 30 min scan | Multi-architecture images plus vulnerability report |
| Successful release-tag build | DEB/RPM packaging | 30 min per architecture | Packages and checksum files uploaded to the release |
| Successful non-preview release-tag build | Helm template test and package | 30 min build; 30 min publish | Versioned chart and repository index |
Use an exact preview tag for end-to-end release rehearsal. Manual dispatches
are backfill/debug paths and do not prove the automatic `workflow_run` chain.
## Evidence requirements
A green check is useful only when it proves the intended behavior ran:
- Record the exact commit SHA and run URL.
- Separate product failure from runner prerequisites, service readiness, and
cancellation. Repair the precondition, then rerun the exact workload.
- Preserve membership manifests, JUnit, raw compatibility logs, seeds, and
server logs where the workflow provides them.
- For a bug fix or a new fault checker, provide sensitivity evidence: the old
behavior or an intentional mutation must fail the new oracle, and the fixed
behavior must pass it.
- Never promote a report-only lane to required from one green run. Require at
least 14 days and 30 representative pull requests with at least 99% complete
execution, then update the ruleset and this table together.
## Change checklist
Update this file in the same pull request when any of these change:
- workflow triggers, job names, timeouts, or nextest profile ownership;
- required status contexts or strict/merge-queue policy;
- scheduled cadence, alert routing, artifact contract, or local reproduction;
- report-only versus gating semantics.
Do not copy per-module test counts here. Update
[e2e-suite-inventory.md](e2e-suite-inventory.md) and its enforced membership
digest instead.
+2 -2
View File
@@ -336,13 +336,13 @@ opentelemetry = { workspace = true }
tracing-opentelemetry = { workspace = true }
# Data structures
hashbrown = { workspace = true, features = ["serde", "rayon"] }
mimalloc = { workspace = true }
rustfs-mimalloc = { workspace = true }
[target.'cfg(target_os = "linux")'.dependencies]
libsystemd.workspace = true
[target.'cfg(not(target_os = "windows"))'.dependencies]
libmimalloc-sys.workspace = true
rustfs-mimalloc-sys.workspace = true
[dev-dependencies]
uuid = { workspace = true, features = ["v4", "v5", "fast-rng", "macro-diagnostics"] }
+1 -7
View File
@@ -369,14 +369,8 @@ pub fn allocator_reclaim_controller_snapshot(ctx: &CancellationToken) -> Allocat
}
#[cfg(not(target_os = "windows"))]
#[allow(unsafe_code)]
fn collect_allocator_memory(force: bool) -> Result<(), String> {
// SAFETY: `mi_collect` is provided by the active global allocator backend
// on this target family. It is explicitly intended to reclaim retained
// pages/segments and does not require additional invariants from the caller.
unsafe {
libmimalloc_sys::mi_collect(force);
}
rustfs_mimalloc::MiMalloc::collect(force);
Ok(())
}
+10 -8
View File
@@ -26,22 +26,22 @@ struct MiMallocAllocator;
unsafe impl GlobalAlloc for MiMallocAllocator {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
// SAFETY: the caller upholds GlobalAlloc's contract for layout.
unsafe { mimalloc::MiMalloc.alloc(layout) }
unsafe { rustfs_mimalloc::MiMalloc.alloc(layout) }
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
// SAFETY: the caller upholds GlobalAlloc's contract for layout.
unsafe { mimalloc::MiMalloc.alloc_zeroed(layout) }
unsafe { rustfs_mimalloc::MiMalloc.alloc_zeroed(layout) }
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
// SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
unsafe { mimalloc::MiMalloc.dealloc(ptr, layout) }
unsafe { rustfs_mimalloc::MiMalloc.dealloc(ptr, layout) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
// SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
unsafe { mimalloc::MiMalloc.realloc(ptr, layout, new_size) }
unsafe { rustfs_mimalloc::MiMalloc.realloc(ptr, layout, new_size) }
}
}
@@ -51,7 +51,7 @@ static GLOBAL: hotpath::CountingAllocator<MiMallocAllocator> = hotpath::Counting
#[cfg(not(all(feature = "hotpath", feature = "hotpath-alloc")))]
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
static GLOBAL: rustfs_mimalloc::MiMalloc = rustfs_mimalloc::MiMalloc;
fn main() {
let _hotpath_guard = hotpath::HotpathGuardBuilder::new("main").build();
@@ -71,8 +71,9 @@ mod tests {
allocation.extend_from_slice(&[7_u8; 64]);
assert_eq!(allocation.len(), 64);
let heap = rustfs_mimalloc::heap::Heap::main();
// SAFETY: the live Vec pointer is valid to inspect for heap ownership.
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(allocation.as_ptr().cast()) });
assert!(unsafe { heap.contains(allocation.as_ptr()) });
}
#[test]
@@ -85,12 +86,13 @@ mod tests {
let layout = Layout::from_size_align(32, 8).expect("valid test allocation layout");
let grown_layout = Layout::from_size_align(64, 8).expect("valid grown test allocation layout");
let allocator = super::MiMallocAllocator;
let heap = rustfs_mimalloc::heap::Heap::main();
// SAFETY: The pointer is checked for null before use and later released
// through the same allocator with the corresponding layout.
let ptr = unsafe { allocator.alloc_zeroed(layout) };
assert!(!ptr.is_null());
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(ptr.cast()) });
assert!(unsafe { heap.contains(ptr) });
assert!(unsafe { std::slice::from_raw_parts(ptr, 32).iter().all(|byte| *byte == 0) });
// SAFETY: `ptr` was allocated by `allocator` with `layout`; on failure
@@ -102,7 +104,7 @@ mod tests {
panic!("mimalloc realloc failed in allocator smoke test");
}
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(grown_ptr.cast()) });
assert!(unsafe { heap.contains(grown_ptr) });
// SAFETY: `grown_ptr` was reallocated by `allocator` and is released
// with the matching grown layout.
unsafe { allocator.dealloc(grown_ptr, grown_layout) };
+19 -36
View File
@@ -17,10 +17,7 @@ use rustfs_io_metrics::{
record_cpu_usage, record_memory_usage, record_process_memory_split,
};
use serde::Serialize;
#[cfg(any(test, not(target_os = "windows")))]
use serde_json::Value;
#[cfg(not(target_os = "windows"))]
use std::ffi::CStr;
use std::path::Path;
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;
@@ -231,7 +228,18 @@ fn read_cgroup_memory_snapshot() -> Option<CgroupMemorySnapshot> {
read_cgroup_v2().or_else(read_cgroup_v1)
}
#[cfg(any(test, not(target_os = "windows")))]
fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
let json = rustfs_mimalloc::MiMalloc::stats_json();
if json.is_empty() {
return None;
}
let observation = parse_mimalloc_stats_json(&json)?;
Some(AllocatorMemorySnapshot {
backend: crate::allocator_reclaim::allocator_backend(),
observation,
})
}
fn numeric_json_value(value: &Value) -> Option<u64> {
match value {
Value::Number(number) => number
@@ -242,7 +250,6 @@ fn numeric_json_value(value: &Value) -> Option<u64> {
}
}
#[cfg(any(test, not(target_os = "windows")))]
fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
match value {
Value::Object(fields) => fields
@@ -254,7 +261,6 @@ fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
}
}
#[cfg(any(test, not(target_os = "windows")))]
fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option<u64> {
match value {
Value::Object(fields) => {
@@ -271,12 +277,10 @@ fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option<u64>
}
}
#[cfg(any(test, not(target_os = "windows")))]
fn mimalloc_stat_current(value: &Value, metric: &str) -> Option<u64> {
mimalloc_stat_field(value, metric, "current")
}
#[cfg(any(test, not(target_os = "windows")))]
fn mimalloc_stat_sum(value: &Value, metrics: &[&str], field: &str) -> Option<u64> {
metrics
.iter()
@@ -285,7 +289,6 @@ fn mimalloc_stat_sum(value: &Value, metrics: &[&str], field: &str) -> Option<u64
.filter(|value| *value > 0)
}
#[cfg(any(test, not(target_os = "windows")))]
fn parse_mimalloc_stats_json(stats_json: &str) -> Option<AllocatorMemoryObservation> {
let value = serde_json::from_str::<Value>(stats_json).ok()?;
let malloc_metrics = ["malloc_normal", "malloc_huge"];
@@ -312,33 +315,6 @@ fn parse_mimalloc_stats_json(stats_json: &str) -> Option<AllocatorMemoryObservat
}
}
#[cfg(not(target_os = "windows"))]
#[allow(unsafe_code)]
fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
// SAFETY: `mi_stats_get_json` returns a null-terminated JSON buffer owned by
// mimalloc when called with a null input buffer. The mimalloc API requires
// freeing that buffer with `mi_free`; parsing finishes before the buffer is freed.
let observation = unsafe {
let stats_ptr = libmimalloc_sys::mi_stats_get_json(0, std::ptr::null_mut());
if stats_ptr.is_null() {
return None;
}
let observation = CStr::from_ptr(stats_ptr).to_str().ok().and_then(parse_mimalloc_stats_json);
libmimalloc_sys::mi_free(stats_ptr.cast());
observation?
};
Some(AllocatorMemorySnapshot {
backend: crate::allocator_reclaim::allocator_backend(),
observation,
})
}
#[cfg(target_os = "windows")]
fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
None
}
fn configured_memory_observability_interval_secs() -> u64 {
rustfs_utils::get_env_u64(ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS, DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS).max(1)
}
@@ -566,6 +542,13 @@ mod tests {
assert_eq!(parse_mimalloc_stats_json(r#"{ "allocator": "unknown" }"#), None);
}
#[test]
fn read_allocator_memory_snapshot_uses_mimalloc_stats_json() {
let snapshot = super::read_allocator_memory_snapshot();
#[cfg(not(target_os = "windows"))]
assert!(snapshot.is_some(), "allocator snapshot should be available on non-Windows");
}
#[test]
fn memory_observability_snapshot_reports_disabled_when_metrics_are_disabled() {
let snapshot = build_memory_observability_status_snapshot(false, 15, false);