mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-23 04:39:04 +00:00
Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5ff5a7a06e | |||
| b6ba89d9e4 | |||
| 648d5166e2 | |||
| 84eb5aebef | |||
| c58accabe7 | |||
| 4d92835d7b | |||
| 71db3327ad | |||
| 68e95d497e | |||
| 8350d73be9 | |||
| 9a2a33a7eb | |||
| 5df26a13ba | |||
| e3a362989f | |||
| 0cd2ae20e2 | |||
| 55a7fa9f03 | |||
| 9f51f37a0d |
@@ -30,7 +30,8 @@ make build-docker BUILD_OS=ubuntu22.04
|
||||
- Crate membership: `Cargo.toml` `[workspace].members`
|
||||
- Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
|
||||
- Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)
|
||||
- CI gates: `.github/workflows/ci.yml` (source of truth; never copy its steps into docs)
|
||||
- CI workflow steps: `.github/workflows/`; event, timeout, and required-status
|
||||
matrix: [docs/testing/ci-gates.md](docs/testing/ci-gates.md)
|
||||
- Test-layer taxonomy, per-layer entry commands, serial/nextest rules, flake
|
||||
policy: [docs/testing/README.md](docs/testing/README.md)
|
||||
- Tier/ILM transition debugging (xl.meta inspection, versionId tracing):
|
||||
|
||||
@@ -70,6 +70,8 @@ make pre-pr
|
||||
|
||||
> For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
|
||||
|
||||
> For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
|
||||
|
||||
### 🔒 Automated Pre-commit Hooks
|
||||
#### What `make pre-commit` and `make pre-pr` actually run
|
||||
|
||||
|
||||
Generated
+23
-27
@@ -1858,9 +1858,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.3"
|
||||
version = "1.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
|
||||
checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -2522,12 +2522,6 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cty"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
|
||||
|
||||
[[package]]
|
||||
name = "curve25519-dalek"
|
||||
version = "4.1.3"
|
||||
@@ -5988,15 +5982,6 @@ version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
||||
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
version = "0.1.49"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cty",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.20"
|
||||
@@ -6397,14 +6382,6 @@ dependencies = [
|
||||
"synstructure 0.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mimalloc"
|
||||
version = "0.1.52"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
|
||||
dependencies = [
|
||||
"libmimalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.17"
|
||||
@@ -9162,13 +9139,11 @@ dependencies = [
|
||||
"insta",
|
||||
"jiff",
|
||||
"libc",
|
||||
"libmimalloc-sys",
|
||||
"libsystemd",
|
||||
"matchit 0.9.2",
|
||||
"md-5 0.11.0",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"mimalloc",
|
||||
"mime_guess",
|
||||
"opentelemetry",
|
||||
"opentelemetry_sdk",
|
||||
@@ -9204,6 +9179,8 @@ dependencies = [
|
||||
"rustfs-lock",
|
||||
"rustfs-log-analyzer",
|
||||
"rustfs-madmin",
|
||||
"rustfs-mimalloc",
|
||||
"rustfs-mimalloc-sys",
|
||||
"rustfs-notify",
|
||||
"rustfs-object-capacity",
|
||||
"rustfs-object-data-cache",
|
||||
@@ -9587,6 +9564,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
"sha2 0.11.0",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.20",
|
||||
@@ -9875,6 +9853,24 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-mimalloc"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a406f4aa07084301d485beec873af6dccc8e3f8762da244743df92038b1db1a6"
|
||||
dependencies = [
|
||||
"rustfs-mimalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-mimalloc-sys"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3051b819175f58445d4c369a72f0ab88149f3885ba8bea2aff3be01f53fe7cd"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-rc.3"
|
||||
|
||||
+2
-2
@@ -350,8 +350,8 @@ russh-sftp = "2.4.0"
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
|
||||
rustfs-mimalloc = { version = "0.5.0" }
|
||||
rustfs-mimalloc-sys = { version = "0.5.0" }
|
||||
hotpath = { version = "0.23.3", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
@@ -23,32 +23,21 @@
|
||||
//! unconsumed intents is the consumer's job (see `rustfs-heal`
|
||||
//! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::hash_map::RandomState;
|
||||
use std::hash::{BuildHasher, Hash};
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::{
|
||||
Arc, Mutex, OnceLock,
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Bounded capacity of the global MRF channel. Backpressure is resolved by
|
||||
/// dropping (and counting) intents, never by blocking the producer.
|
||||
const MRF_CHANNEL_CAPACITY: usize = 8192;
|
||||
const MRF_COALESCER_SHARDS: usize = 16;
|
||||
const MRF_COALESCER_MAX_KEYS: usize = 8192;
|
||||
const MRF_COALESCER_MAX_BYTES: usize = 16 * 1024 * 1024;
|
||||
const MRF_COALESCER_TTL: Duration = Duration::from_secs(60);
|
||||
const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
|
||||
|
||||
/// Why an intent was produced. Drives the heal priority mapping on the
|
||||
/// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
|
||||
/// PartialWrite -> Normal).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MrfKind {
|
||||
/// Erasure decode failed while serving a read (read path).
|
||||
DecodeFailure,
|
||||
@@ -78,52 +67,12 @@ pub struct MrfIntent {
|
||||
/// Version the intent targets, as raw UUID bytes.
|
||||
pub version_id: Option<[u8; 16]>,
|
||||
pub kind: MrfKind,
|
||||
/// Stable erasure-set scope when the producer has it. Kept optional so
|
||||
/// metadata corruption and legacy producers do not invent a scope.
|
||||
pub scope: Option<MrfScope>,
|
||||
/// Generation of the node-local ingress lease. It is not persisted in
|
||||
/// the journal; replayed records acquire a fresh lease when re-enqueued.
|
||||
pub lease: Option<MrfIngressLease>,
|
||||
pub enqueued_at_ms: u64,
|
||||
/// Times this intent has already been offered to the heal manager.
|
||||
/// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
|
||||
pub attempts: u8,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct MrfScope {
|
||||
pub pool_index: u32,
|
||||
pub set_index: u32,
|
||||
}
|
||||
|
||||
/// Opaque generation used to release exactly the admission that created an
|
||||
/// ingress entry. A generation prevents a late terminal callback from
|
||||
/// deleting a newer retry for the same identity (ABA).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct MrfIngressLease(u64);
|
||||
|
||||
impl MrfIngressLease {
|
||||
const fn new(value: u64) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MrfDropReason {
|
||||
Disabled,
|
||||
Uninitialized,
|
||||
Full,
|
||||
OversizedIdentity,
|
||||
CoalescerFull,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MrfIngressResult {
|
||||
Enqueued,
|
||||
Coalesced,
|
||||
Dropped(MrfDropReason),
|
||||
}
|
||||
|
||||
/// Consumer-side retry ceiling before an intent is given up on.
|
||||
pub const MRF_MAX_ATTEMPTS: u8 = 3;
|
||||
|
||||
@@ -138,159 +87,6 @@ impl MrfIntent {
|
||||
|
||||
static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
struct MrfIdentityKey {
|
||||
kind: MrfKind,
|
||||
bucket: Arc<str>,
|
||||
object: Arc<str>,
|
||||
version_id: Option<[u8; 16]>,
|
||||
scope: Option<MrfScope>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct IngressEntry {
|
||||
lease: MrfIngressLease,
|
||||
expires_at: Instant,
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
type MrfCoalescerShard = Mutex<HashMap<MrfIdentityKey, IngressEntry>>;
|
||||
type MrfCoalescer = Box<[MrfCoalescerShard]>;
|
||||
|
||||
static MRF_COALESCER: OnceLock<MrfCoalescer> = OnceLock::new();
|
||||
static NEXT_MRF_LEASE: AtomicU64 = AtomicU64::new(1);
|
||||
static MRF_COALESCER_COUNT: AtomicUsize = AtomicUsize::new(0);
|
||||
static MRF_COALESCER_BYTES: AtomicUsize = AtomicUsize::new(0);
|
||||
static MRF_HASH_STATE: OnceLock<RandomState> = OnceLock::new();
|
||||
|
||||
fn coalescer() -> &'static [MrfCoalescerShard] {
|
||||
MRF_COALESCER.get_or_init(|| {
|
||||
(0..MRF_COALESCER_SHARDS)
|
||||
.map(|_| Mutex::new(HashMap::new()))
|
||||
.collect::<Vec<_>>()
|
||||
.into_boxed_slice()
|
||||
})
|
||||
}
|
||||
|
||||
fn key_shard(key: &MrfIdentityKey) -> usize {
|
||||
let hash = MRF_HASH_STATE.get_or_init(RandomState::new).hash_one(key);
|
||||
usize::try_from(hash).unwrap_or(0) % MRF_COALESCER_SHARDS
|
||||
}
|
||||
|
||||
fn canonical_version(version_id: Option<Uuid>) -> Option<[u8; 16]> {
|
||||
version_id
|
||||
.filter(|version| !version.is_nil())
|
||||
.map(|version| *version.as_bytes())
|
||||
}
|
||||
|
||||
fn canonical_identity(
|
||||
kind: MrfKind,
|
||||
version_id: Option<[u8; 16]>,
|
||||
scope: Option<MrfScope>,
|
||||
) -> (Option<[u8; 16]>, Option<MrfScope>) {
|
||||
let version_id = version_id.filter(|bytes| *bytes != [0; 16]);
|
||||
match kind {
|
||||
MrfKind::MetadataCorruption => (None, None),
|
||||
MrfKind::DecodeFailure | MrfKind::PartialWrite => (version_id, scope),
|
||||
}
|
||||
}
|
||||
|
||||
fn identity_estimated_bytes(key: &MrfIdentityKey) -> usize {
|
||||
64usize
|
||||
.saturating_add(key.bucket.len())
|
||||
.saturating_add(key.object.len())
|
||||
.saturating_add(key.version_id.map_or(0, |_| 16))
|
||||
.saturating_add(key.scope.map_or(0, |_| 8))
|
||||
}
|
||||
|
||||
fn reserve(counter: &AtomicUsize, limit: usize, amount: usize) -> bool {
|
||||
let mut current = counter.load(Ordering::Relaxed);
|
||||
loop {
|
||||
let Some(next) = current.checked_add(amount) else {
|
||||
return false;
|
||||
};
|
||||
if next > limit {
|
||||
return false;
|
||||
}
|
||||
match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => return true,
|
||||
Err(observed) => current = observed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn coalescer_admit(key: MrfIdentityKey) -> Result<MrfIngressLease, MrfIngressResult> {
|
||||
let shard = key_shard(&key);
|
||||
let mut entries = coalescer()[shard]
|
||||
.lock()
|
||||
.map_err(|_| MrfIngressResult::Dropped(MrfDropReason::CoalescerFull))?;
|
||||
let now = Instant::now();
|
||||
let before = entries.len();
|
||||
let mut expired_bytes = 0usize;
|
||||
entries.retain(|_, entry| {
|
||||
if entry.expires_at > now {
|
||||
true
|
||||
} else {
|
||||
expired_bytes = expired_bytes.saturating_add(entry.bytes);
|
||||
false
|
||||
}
|
||||
});
|
||||
let evicted = before.saturating_sub(entries.len());
|
||||
if evicted > 0 {
|
||||
MRF_COALESCER_COUNT.fetch_sub(evicted, Ordering::Relaxed);
|
||||
MRF_COALESCER_BYTES.fetch_sub(expired_bytes, Ordering::Relaxed);
|
||||
let evicted = u64::try_from(evicted).unwrap_or(u64::MAX);
|
||||
metrics::counter!("rustfs_heal_mrf_coalescer_expired_total").increment(evicted);
|
||||
metrics::counter!("rustfs_heal_mrf_coalescer_evictions_total").increment(evicted);
|
||||
}
|
||||
if entries.contains_key(&key) {
|
||||
metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
|
||||
return Err(MrfIngressResult::Coalesced);
|
||||
}
|
||||
let bytes = identity_estimated_bytes(&key);
|
||||
let count_reserved = reserve(&MRF_COALESCER_COUNT, MRF_COALESCER_MAX_KEYS, 1);
|
||||
let bytes_reserved = count_reserved && reserve(&MRF_COALESCER_BYTES, MRF_COALESCER_MAX_BYTES, bytes);
|
||||
if !count_reserved || !bytes_reserved {
|
||||
if count_reserved {
|
||||
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "coalescer_full").increment(1);
|
||||
return Err(MrfIngressResult::Dropped(MrfDropReason::CoalescerFull));
|
||||
}
|
||||
let lease = MrfIngressLease::new(NEXT_MRF_LEASE.fetch_add(1, Ordering::Relaxed));
|
||||
if entries
|
||||
.insert(
|
||||
key,
|
||||
IngressEntry {
|
||||
lease,
|
||||
expires_at: now + MRF_COALESCER_TTL,
|
||||
bytes,
|
||||
},
|
||||
)
|
||||
.is_some()
|
||||
{
|
||||
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
|
||||
MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
|
||||
metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
|
||||
return Err(MrfIngressResult::Coalesced);
|
||||
}
|
||||
Ok(lease)
|
||||
}
|
||||
|
||||
fn coalescer_release(key: &MrfIdentityKey, lease: Option<MrfIngressLease>) {
|
||||
let Some(lease) = lease else {
|
||||
return;
|
||||
};
|
||||
if let Ok(mut entries) = coalescer()[key_shard(key)].lock() {
|
||||
let should_remove = entries.get(key).is_some_and(|entry| entry.lease == lease);
|
||||
if should_remove {
|
||||
let bytes = entries.remove(key).map(|entry| entry.bytes).unwrap_or(0);
|
||||
MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
|
||||
MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
|
||||
/// this before touching the channel so the disabled path stays allocation- and
|
||||
/// sync-free.
|
||||
@@ -326,90 +122,21 @@ pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
|
||||
/// This runs on IO error paths, so it stays synchronous and cheap: one
|
||||
/// bounded allocation for the two `Arc<str>` handles plus the channel slot.
|
||||
pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
|
||||
matches!(
|
||||
try_send_mrf_intent_typed(kind, bucket, object, version_id, None),
|
||||
MrfIngressResult::Enqueued
|
||||
)
|
||||
}
|
||||
|
||||
/// Typed ingress result. `Coalesced` means an equivalent in-flight channel
|
||||
/// intent already exists; it is not a second executable or durable admission.
|
||||
pub fn try_send_mrf_intent_typed(
|
||||
kind: MrfKind,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<Uuid>,
|
||||
scope: Option<MrfScope>,
|
||||
) -> MrfIngressResult {
|
||||
if !mrf_delivery_enabled() {
|
||||
return MrfIngressResult::Dropped(MrfDropReason::Disabled);
|
||||
return false;
|
||||
}
|
||||
let Some(sender) = GLOBAL_MRF_SENDER.get() else {
|
||||
return MrfIngressResult::Dropped(MrfDropReason::Uninitialized);
|
||||
};
|
||||
if bucket.len() > MRF_MAX_IDENTITY_COMPONENT || object.len() > MRF_MAX_IDENTITY_COMPONENT {
|
||||
return MrfIngressResult::Dropped(MrfDropReason::OversizedIdentity);
|
||||
}
|
||||
let (version_id, scope) = canonical_identity(kind, canonical_version(version_id), scope);
|
||||
let key = MrfIdentityKey {
|
||||
kind,
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object),
|
||||
version_id,
|
||||
scope,
|
||||
};
|
||||
let lease = match coalescer_admit(key.clone()) {
|
||||
Ok(lease) => lease,
|
||||
Err(result) => return result,
|
||||
return false;
|
||||
};
|
||||
let intent = MrfIntent {
|
||||
bucket: key.bucket.clone(),
|
||||
object: key.object.clone(),
|
||||
version_id: key.version_id,
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object),
|
||||
version_id: version_id.map(|vid| *vid.as_bytes()),
|
||||
kind,
|
||||
scope,
|
||||
lease: Some(lease),
|
||||
enqueued_at_ms: unix_now_ms(),
|
||||
attempts: 0,
|
||||
};
|
||||
match sender.try_send(intent) {
|
||||
Ok(()) => MrfIngressResult::Enqueued,
|
||||
Err(mpsc::error::TrySendError::Full(_)) => {
|
||||
coalescer_release(&key, Some(lease));
|
||||
metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "channel_full").increment(1);
|
||||
MrfIngressResult::Dropped(MrfDropReason::Full)
|
||||
}
|
||||
Err(mpsc::error::TrySendError::Closed(_)) => {
|
||||
coalescer_release(&key, Some(lease));
|
||||
MrfIngressResult::Dropped(MrfDropReason::Uninitialized)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Release the ingress key once the consumer owns the intent.
|
||||
pub fn release_mrf_intent(intent: &MrfIntent) {
|
||||
release_mrf_identity(intent.kind, &intent.bucket, &intent.object, intent.version_id, intent.scope, intent.lease);
|
||||
}
|
||||
|
||||
pub fn release_mrf_identity(
|
||||
kind: MrfKind,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<[u8; 16]>,
|
||||
scope: Option<MrfScope>,
|
||||
lease: Option<MrfIngressLease>,
|
||||
) {
|
||||
let (version_id, scope) = canonical_identity(kind, version_id, scope);
|
||||
coalescer_release(
|
||||
&MrfIdentityKey {
|
||||
kind,
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object),
|
||||
version_id,
|
||||
scope,
|
||||
},
|
||||
lease,
|
||||
);
|
||||
sender.try_send(intent).is_ok()
|
||||
}
|
||||
|
||||
fn unix_now_ms() -> u64 {
|
||||
@@ -417,8 +144,7 @@ fn unix_now_ms() -> u64 {
|
||||
// failure would be a bug rather than something to handle here.
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()
|
||||
.and_then(|d| u64::try_from(d.as_millis()).ok())
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
@@ -489,60 +215,12 @@ mod tests {
|
||||
object: Arc::from("object"),
|
||||
version_id: Some([0u8; 16]),
|
||||
kind: MrfKind::DecodeFailure,
|
||||
scope: None,
|
||||
lease: None,
|
||||
enqueued_at_ms: 0,
|
||||
attempts: 0,
|
||||
};
|
||||
assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ingress_duplicate_identity_coalesces_and_releases_for_retry() {
|
||||
let key = MrfIdentityKey {
|
||||
kind: MrfKind::DecodeFailure,
|
||||
bucket: Arc::from("ingress-test-bucket"),
|
||||
object: Arc::from("ingress-test-object"),
|
||||
version_id: Some([9; 16]),
|
||||
scope: Some(MrfScope {
|
||||
pool_index: 3,
|
||||
set_index: 4,
|
||||
}),
|
||||
};
|
||||
let lease = coalescer_admit(key.clone()).expect("first identity should be admitted");
|
||||
for _ in 0..999 {
|
||||
assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced));
|
||||
}
|
||||
coalescer_release(&key, Some(lease));
|
||||
let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry");
|
||||
coalescer_release(&key, Some(retry_lease));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ingress_identity_preserves_kind_scope_and_version_boundaries() {
|
||||
let (nil_version, nil_scope) = canonical_identity(
|
||||
MrfKind::DecodeFailure,
|
||||
Some([0; 16]),
|
||||
Some(MrfScope {
|
||||
pool_index: 1,
|
||||
set_index: 2,
|
||||
}),
|
||||
);
|
||||
assert_eq!(nil_version, None, "nil UUID is the unversioned identity");
|
||||
assert!(nil_scope.is_some());
|
||||
|
||||
let (metadata_version, metadata_scope) = canonical_identity(
|
||||
MrfKind::MetadataCorruption,
|
||||
Some([7; 16]),
|
||||
Some(MrfScope {
|
||||
pool_index: 1,
|
||||
set_index: 2,
|
||||
}),
|
||||
);
|
||||
assert_eq!(metadata_version, None);
|
||||
assert_eq!(metadata_scope, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_send_delivers_and_respects_capacity() {
|
||||
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
|
||||
@@ -552,7 +230,6 @@ mod tests {
|
||||
let intent = receiver.recv().await.expect("intent should arrive");
|
||||
assert_eq!(intent.kind, MrfKind::DecodeFailure);
|
||||
assert_eq!(intent.bucket.as_ref(), "b");
|
||||
release_mrf_intent(&intent);
|
||||
|
||||
// Disable delivery: producers become no-ops.
|
||||
set_mrf_delivery_enabled(false);
|
||||
@@ -562,8 +239,8 @@ mod tests {
|
||||
// Fill the bounded channel past capacity: excess intents are dropped,
|
||||
// never blocking.
|
||||
let mut accepted = 0;
|
||||
for index in 0..(MRF_CHANNEL_CAPACITY + 64) {
|
||||
if try_send_mrf_intent(MrfKind::PartialWrite, "b", &format!("o-{index}"), None) {
|
||||
for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
|
||||
if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
|
||||
accepted += 1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8000,10 +8000,15 @@ impl DiskAPI for LocalDisk {
|
||||
use std::io::Write as _;
|
||||
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
let lock_path = file_path.with_extension("rustfs-cas.lock");
|
||||
let path = path.to_string();
|
||||
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
|
||||
return Ok(tokio::task::spawn_blocking(move || {
|
||||
// A persistent directory lock bounds metadata growth. Removing
|
||||
// per-target lock files can split flock ownership across inodes.
|
||||
let lock_path = file_path
|
||||
.parent()
|
||||
.ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "conditional file has no parent"))?
|
||||
.join(".rustfs-cas.lock");
|
||||
let lock = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
@@ -8068,7 +8073,25 @@ impl DiskAPI for LocalDisk {
|
||||
.map_err(DiskError::from)??);
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
let sync_metadata = effective_durability(volume).syncs_commit_metadata();
|
||||
let publication_root = self.publication_root.clone();
|
||||
return Ok(tokio::task::spawn_blocking(move || {
|
||||
os::compare_and_update_control_file(
|
||||
&file_path,
|
||||
expected.as_deref(),
|
||||
replacement.as_deref(),
|
||||
sync_metadata,
|
||||
&publication_root,
|
||||
)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)??);
|
||||
}
|
||||
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
let _ = (volume, path, expected, replacement);
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
@@ -21823,9 +21846,9 @@ mod test {
|
||||
assert!(matches!(results[1].as_ref().unwrap_err(), DiskError::Io(_)));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[cfg(any(unix, windows))]
|
||||
#[tokio::test]
|
||||
async fn conditional_file_update_never_deletes_a_new_owner() {
|
||||
async fn windows_and_unix_conditional_file_update_never_deletes_a_new_owner() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
@@ -21856,8 +21879,18 @@ mod test {
|
||||
disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
|
||||
.await
|
||||
.expect("new owner marker should remain"),
|
||||
owner_b
|
||||
owner_b.clone()
|
||||
);
|
||||
assert_eq!(
|
||||
disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Some(owner_b), None)
|
||||
.await
|
||||
.expect("current owner should remove marker"),
|
||||
ConditionalFileUpdate::Updated
|
||||
);
|
||||
assert!(matches!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH).await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
@@ -21872,7 +21905,10 @@ mod test {
|
||||
let marker_path = disk
|
||||
.get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
|
||||
.expect("marker path should resolve");
|
||||
let lock_path = marker_path.with_extension("rustfs-cas.lock");
|
||||
let lock_path = marker_path
|
||||
.parent()
|
||||
.expect("marker path should have a parent")
|
||||
.join(".rustfs-cas.lock");
|
||||
let lock = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
@@ -21893,6 +21929,40 @@ mod test {
|
||||
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_conditional_file_update_returns_would_block_when_marker_lock_is_contended() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
ensure_test_volume(&disk, RUSTFS_META_BUCKET).await;
|
||||
let marker_path = disk
|
||||
.get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
|
||||
.expect("marker path should resolve");
|
||||
let lock_path = marker_path
|
||||
.parent()
|
||||
.expect("marker path should have a parent")
|
||||
.join(".rustfs-cas.lock");
|
||||
let lock = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(lock_path)
|
||||
.expect("marker lock should open");
|
||||
lock.try_lock().expect("marker lock should be held");
|
||||
|
||||
let err = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, None, Some(Bytes::from_static(b"owner"))),
|
||||
)
|
||||
.await
|
||||
.expect("contended conditional update must not block")
|
||||
.expect_err("contended conditional update must retry");
|
||||
|
||||
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn replacement_io_paths_stay_under_the_mount_lease() {
|
||||
|
||||
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(windows)]
|
||||
use crate::disk::ConditionalFileUpdate;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::error::Result;
|
||||
use crate::disk::error_conv::to_file_error;
|
||||
@@ -3458,6 +3460,89 @@ fn read_windows_relative_file(file_path: &Path, parent_guard: &ExistingBaseDirec
|
||||
Ok(Some(data))
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) fn compare_and_update_control_file(
|
||||
file_path: &Path,
|
||||
expected: Option<&[u8]>,
|
||||
replacement: Option<&[u8]>,
|
||||
sync_metadata: bool,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<ConditionalFileUpdate> {
|
||||
use windows_sys::{
|
||||
Wdk::Storage::FileSystem::{
|
||||
FILE_NON_DIRECTORY_FILE, FILE_OPEN, FILE_OPEN_IF, FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT,
|
||||
},
|
||||
Win32::Storage::FileSystem::{
|
||||
DELETE, FILE_ATTRIBUTE_NORMAL, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_WRITE_DATA, SYNCHRONIZE,
|
||||
},
|
||||
};
|
||||
|
||||
let parent = file_path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file has no parent"))?;
|
||||
let parent_guard = lock_windows_directory_tree(parent, Some(parent), publication_root)?;
|
||||
let lock = open_windows_relative(
|
||||
parent_guard.last_handle()?,
|
||||
std::ffi::OsStr::new(".rustfs-cas.lock"),
|
||||
SYNCHRONIZE | FILE_READ_ATTRIBUTES | FILE_WRITE_DATA,
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE,
|
||||
FILE_OPEN_IF,
|
||||
FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
|
||||
FILE_ATTRIBUTE_NORMAL,
|
||||
true,
|
||||
)?;
|
||||
validate_windows_owned_file(&lock)?;
|
||||
match lock.as_file().try_lock() {
|
||||
Ok(()) => {}
|
||||
Err(std::fs::TryLockError::WouldBlock) => return Err(io::Error::from(io::ErrorKind::WouldBlock)),
|
||||
Err(std::fs::TryLockError::Error(err)) => return Err(err),
|
||||
}
|
||||
|
||||
let current = read_windows_relative_file(file_path, &parent_guard)?;
|
||||
let matches = match (¤t, expected) {
|
||||
(None, None) => true,
|
||||
(Some(current), Some(expected)) => current.as_slice() == expected,
|
||||
_ => false,
|
||||
};
|
||||
if !matches {
|
||||
return Ok(match current {
|
||||
None => ConditionalFileUpdate::Missing,
|
||||
Some(_) => ConditionalFileUpdate::Mismatch,
|
||||
});
|
||||
}
|
||||
|
||||
match replacement {
|
||||
Some(replacement) => RenameDestinationPathGuard {
|
||||
directory: parent.to_path_buf(),
|
||||
_directory_guard: parent_guard,
|
||||
}
|
||||
.write_file_for_path_access(file_path, replacement, sync_metadata, sync_metadata)?,
|
||||
None => {
|
||||
let file_name = file_path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file must have a name"))?;
|
||||
let file = open_windows_relative(
|
||||
parent_guard.last_handle()?,
|
||||
file_name,
|
||||
DELETE | SYNCHRONIZE | FILE_READ_ATTRIBUTES,
|
||||
FILE_SHARE_READ,
|
||||
FILE_OPEN,
|
||||
FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
|
||||
0,
|
||||
true,
|
||||
)?;
|
||||
validate_windows_owned_file(&file)?;
|
||||
set_windows_file_delete_on_close(&file, true)?;
|
||||
drop(file);
|
||||
if sync_metadata {
|
||||
fsync_dir_std(parent)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ConditionalFileUpdate::Updated)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn open_windows_directory_component(
|
||||
parent: &WindowsDirectoryHandle,
|
||||
|
||||
@@ -1412,11 +1412,8 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
|
||||
|
||||
// Reservation won: this sighting owns the repair records for the object,
|
||||
// including the durable journal intent when the caller asked for one.
|
||||
if let Some((kind, version_uuid)) = mrf_intent
|
||||
&& let (Ok(pool_index), Ok(set_index)) = (u32::try_from(pool_index), u32::try_from(set_index))
|
||||
{
|
||||
let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
|
||||
let _ = rustfs_common::mrf_channel::try_send_mrf_intent_typed(kind, bucket, object, version_uuid, Some(scope));
|
||||
if let Some((kind, version_uuid)) = mrf_intent {
|
||||
rustfs_common::mrf_channel::try_send_mrf_intent(kind, bucket, object, version_uuid);
|
||||
}
|
||||
|
||||
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
|
||||
@@ -2124,26 +2124,13 @@ impl SetDisks {
|
||||
|
||||
let put_object_size = known_put_object_storage_size(data.size());
|
||||
let shard_file_size_raw = erasure.shard_file_size(put_object_size);
|
||||
let is_inline_buffer =
|
||||
storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
|
||||
let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
|
||||
|
||||
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
|
||||
let shard_file_size = shard_file_size_raw;
|
||||
let shard_size = erasure.shard_size();
|
||||
let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
|
||||
let direct_inline_commit = matches!(write_path, SmallWritePath::Inline);
|
||||
{
|
||||
use std::io::Write;
|
||||
let msg = format!(
|
||||
"INLINE_DEBUG: bucket={} obj={} size={} shard_fs={} ds={} bs={} inline={} direct={} path={} iblock={} ver={}\n",
|
||||
bucket, object, put_object_size, shard_file_size_raw, erasure.data_shards, fi.erasure.block_size,
|
||||
is_inline_buffer, direct_inline_commit, write_path.metric_label(), storage_class_config.inline_block(), opts.versioned
|
||||
);
|
||||
if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open("/tmp/rustfs_inline_debug.log") {
|
||||
let _ = f.write_all(msg.as_bytes());
|
||||
}
|
||||
let _ = std::io::stderr().write_all(msg.as_bytes());
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_path(write_path.metric_label());
|
||||
let writer_setup_stage_start = collect_stage_timing.then(Instant::now);
|
||||
let (mut writers, errors) = if direct_inline_commit {
|
||||
@@ -6650,23 +6637,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
|
||||
// MRF journal intent: partial-write recovery must survive a restart
|
||||
// (HS-01); the heal request below remains the in-memory fast path.
|
||||
let version_uuid = if version_id.is_empty() {
|
||||
Some(None)
|
||||
} else {
|
||||
uuid::Uuid::try_parse(version_id).ok().map(Some)
|
||||
};
|
||||
if let Some(version_uuid) = version_uuid
|
||||
&& let (Ok(pool_index), Ok(set_index)) = (u32::try_from(self.pool_index), u32::try_from(self.set_index))
|
||||
{
|
||||
let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index };
|
||||
let _ = rustfs_common::mrf_channel::try_send_mrf_intent_typed(
|
||||
rustfs_common::mrf_channel::MrfKind::PartialWrite,
|
||||
bucket,
|
||||
object,
|
||||
version_uuid,
|
||||
Some(scope),
|
||||
);
|
||||
}
|
||||
rustfs_common::mrf_channel::try_send_mrf_intent(
|
||||
rustfs_common::mrf_channel::MrfKind::PartialWrite,
|
||||
bucket,
|
||||
object,
|
||||
uuid::Uuid::try_parse(version_id).ok(),
|
||||
);
|
||||
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
|
||||
@@ -3194,7 +3194,7 @@ impl ECStore {
|
||||
|
||||
// Default return value
|
||||
let mut del_objects = vec![DeletedObject::default(); objects.len()];
|
||||
let mut accounting = vec![None; objects.len()];
|
||||
let accounting = vec![None; objects.len()];
|
||||
|
||||
let mut del_errs = Vec::with_capacity(objects.len());
|
||||
for _ in 0..objects.len() {
|
||||
|
||||
@@ -271,7 +271,7 @@ pub(super) fn resolve_latest_object_info_candidates(
|
||||
.filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx));
|
||||
latest_candidates.sort_by_key(|candidate| std::cmp::Reverse(candidate.idx));
|
||||
|
||||
let Some(winner) = latest_candidates.first() else {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
|
||||
@@ -91,6 +91,7 @@ metrics = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
crc-fast = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
|
||||
@@ -373,6 +373,11 @@ impl ErasureSetHealer {
|
||||
set_disk_id: &str,
|
||||
buckets: &[String],
|
||||
) -> Result<(ResumeManager, CheckpointManager)> {
|
||||
if self.replacement_task_id.is_none() && CheckpointManager::is_blocked(&self.disk, task_id).await {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Resume task {task_id} has a blocked checkpoint"),
|
||||
});
|
||||
}
|
||||
// check if resume state exists
|
||||
let has_resume_state = if self.replacement_task_id.is_some() {
|
||||
ResumeManager::has_replacement_intent(&self.disk, task_id).await
|
||||
|
||||
@@ -119,9 +119,6 @@ struct MrfRepairNoticeTarget {
|
||||
bucket: Arc<str>,
|
||||
object: Arc<str>,
|
||||
version_id: Option<[u8; 16]>,
|
||||
kind: rustfs_common::mrf_channel::MrfKind,
|
||||
scope: Option<rustfs_common::mrf_channel::MrfScope>,
|
||||
lease: Option<rustfs_common::mrf_channel::MrfIngressLease>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -892,19 +889,7 @@ impl HealManager {
|
||||
}
|
||||
|
||||
fn remove_mrf_repair_notice_targets_for_task(&self, task_id: &str) {
|
||||
let targets = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(task_id);
|
||||
if let Some(targets) = targets {
|
||||
for target in targets {
|
||||
rustfs_common::mrf_channel::release_mrf_identity(
|
||||
target.kind,
|
||||
&target.bucket,
|
||||
&target.object,
|
||||
target.version_id,
|
||||
target.scope,
|
||||
target.lease,
|
||||
);
|
||||
}
|
||||
}
|
||||
lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(task_id);
|
||||
}
|
||||
|
||||
fn insert_mrf_repair_notice_target(
|
||||
@@ -1294,20 +1279,7 @@ impl HealManager {
|
||||
}
|
||||
self.task_aliases.lock().await.clear();
|
||||
self.retrying_heals.lock().await.clear();
|
||||
let mrf_targets = {
|
||||
let mut registry = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets);
|
||||
registry.drain().flat_map(|(_, targets)| targets).collect::<Vec<_>>()
|
||||
};
|
||||
for target in mrf_targets {
|
||||
rustfs_common::mrf_channel::release_mrf_identity(
|
||||
target.kind,
|
||||
&target.bucket,
|
||||
&target.object,
|
||||
target.version_id,
|
||||
target.scope,
|
||||
target.lease,
|
||||
);
|
||||
}
|
||||
lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).clear();
|
||||
crate::set_heal_queue_length(0);
|
||||
|
||||
// update state
|
||||
@@ -1339,32 +1311,12 @@ impl HealManager {
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn submit_mrf_heal_request_with_receipt(
|
||||
&self,
|
||||
request: HealRequest,
|
||||
bucket: Arc<str>,
|
||||
object: Arc<str>,
|
||||
version_id: Option<[u8; 16]>,
|
||||
) -> Result<HealAdmissionReceipt> {
|
||||
let kind = match &request.heal_type {
|
||||
HealType::Metadata { .. } => rustfs_common::mrf_channel::MrfKind::MetadataCorruption,
|
||||
HealType::ECDecode { .. } => rustfs_common::mrf_channel::MrfKind::DecodeFailure,
|
||||
_ => rustfs_common::mrf_channel::MrfKind::PartialWrite,
|
||||
};
|
||||
self.submit_mrf_heal_request_with_receipt_and_identity(request, bucket, object, version_id, kind, None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn submit_mrf_heal_request_with_receipt_and_identity(
|
||||
&self,
|
||||
request: HealRequest,
|
||||
bucket: Arc<str>,
|
||||
object: Arc<str>,
|
||||
version_id: Option<[u8; 16]>,
|
||||
kind: rustfs_common::mrf_channel::MrfKind,
|
||||
scope: Option<rustfs_common::mrf_channel::MrfScope>,
|
||||
lease: Option<rustfs_common::mrf_channel::MrfIngressLease>,
|
||||
) -> Result<HealAdmissionReceipt> {
|
||||
self.submit_heal_request_with_receipt_alias_and_mrf_notice(
|
||||
request,
|
||||
@@ -1373,9 +1325,6 @@ impl HealManager {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
kind,
|
||||
scope,
|
||||
lease,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
@@ -1590,7 +1539,7 @@ impl HealManager {
|
||||
Self::insert_mrf_repair_notice_target(&mut targets, &task_id, target);
|
||||
}
|
||||
if let Some(displaced_task_id) = &displaced_task_id {
|
||||
self.remove_mrf_repair_notice_targets_for_task(displaced_task_id);
|
||||
lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(displaced_task_id);
|
||||
}
|
||||
drop(retrying_heals);
|
||||
drop(queue);
|
||||
|
||||
@@ -506,18 +506,7 @@ impl HealManager {
|
||||
&displaced_terminal,
|
||||
)
|
||||
.await;
|
||||
if let Some(targets) = lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id) {
|
||||
for target in targets {
|
||||
rustfs_common::mrf_channel::release_mrf_identity(
|
||||
target.kind,
|
||||
&target.bucket,
|
||||
&target.object,
|
||||
target.version_id,
|
||||
target.scope,
|
||||
target.lease,
|
||||
);
|
||||
}
|
||||
}
|
||||
lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id);
|
||||
}
|
||||
if matches!(admission, HealAdmissionResult::Accepted) {
|
||||
if should_notify {
|
||||
|
||||
@@ -299,11 +299,7 @@ impl PriorityHealQueue {
|
||||
|
||||
/// Create a deduplication key from a heal request
|
||||
pub(super) fn make_dedup_key(request: &HealRequest) -> String {
|
||||
let base = Self::make_dedup_key_for_type(&request.heal_type);
|
||||
match (&request.heal_type, request.options.set_key()) {
|
||||
(HealType::Object { .. } | HealType::ECDecode { .. }, Some(scope)) => format!("{base}:scope:{scope}"),
|
||||
_ => base,
|
||||
}
|
||||
Self::make_dedup_key_for_type(&request.heal_type)
|
||||
}
|
||||
|
||||
pub(super) fn make_dedup_key_for_type(heal_type: &HealType) -> String {
|
||||
|
||||
@@ -349,8 +349,6 @@ impl HealManager {
|
||||
let notice_targets = take_mrf_repair_notice_targets(&mrf_repair_notice_targets_clone, &task_id);
|
||||
if successful_completion {
|
||||
emit_mrf_repaired_events(notice_targets);
|
||||
} else {
|
||||
release_mrf_repair_notice_targets(notice_targets);
|
||||
}
|
||||
task_aliases_clone
|
||||
.lock()
|
||||
@@ -640,19 +638,7 @@ pub(super) fn running_heal_set_counts(active_heals: &HashMap<String, Arc<HealTas
|
||||
}
|
||||
|
||||
fn remove_mrf_repair_notice_targets(registry: &Arc<StdMutex<HashMap<String, Vec<MrfRepairNoticeTarget>>>>, task_id: &str) {
|
||||
let targets = lock_mrf_repair_notice_targets(registry).remove(task_id);
|
||||
if let Some(targets) = targets {
|
||||
for target in targets {
|
||||
rustfs_common::mrf_channel::release_mrf_identity(
|
||||
target.kind,
|
||||
&target.bucket,
|
||||
&target.object,
|
||||
target.version_id,
|
||||
target.scope,
|
||||
target.lease,
|
||||
);
|
||||
}
|
||||
}
|
||||
lock_mrf_repair_notice_targets(registry).remove(task_id);
|
||||
}
|
||||
|
||||
fn take_mrf_repair_notice_targets(
|
||||
@@ -685,27 +671,6 @@ fn move_mrf_repair_notice_targets(
|
||||
fn emit_mrf_repaired_events(targets: Vec<MrfRepairNoticeTarget>) {
|
||||
for target in targets {
|
||||
rustfs_common::mrf_channel::note_mrf_repaired(&target.bucket, &target.object, target.version_id);
|
||||
rustfs_common::mrf_channel::release_mrf_identity(
|
||||
target.kind,
|
||||
&target.bucket,
|
||||
&target.object,
|
||||
target.version_id,
|
||||
target.scope,
|
||||
target.lease,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn release_mrf_repair_notice_targets(targets: Vec<MrfRepairNoticeTarget>) {
|
||||
for target in targets {
|
||||
rustfs_common::mrf_channel::release_mrf_identity(
|
||||
target.kind,
|
||||
&target.bucket,
|
||||
&target.object,
|
||||
target.version_id,
|
||||
target.scope,
|
||||
target.lease,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ use crate::heal::manager::HealManager;
|
||||
use metrics::{counter, gauge};
|
||||
use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult};
|
||||
use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfIntent};
|
||||
use std::collections::{HashSet, VecDeque};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
@@ -48,27 +48,15 @@ use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
|
||||
/// Journal location inside the metadata bucket, following the resume-state
|
||||
/// layout.
|
||||
pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin";
|
||||
/// The scoped path is the authoritative snapshot for new readers and carries
|
||||
/// both v1 and v2 records. The legacy path is only a v1 compatibility mirror;
|
||||
/// older readers ignore the authoritative path, while new readers never merge
|
||||
/// the two files. This prevents a partial two-file flush from fabricating a
|
||||
/// mixed epoch.
|
||||
pub(crate) const MRF_SCOPED_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal-scoped.bin";
|
||||
|
||||
/// Record format tag.
|
||||
const MRF_JOURNAL_FORMAT: u8 = 1;
|
||||
/// Record layout version.
|
||||
const MRF_JOURNAL_VERSION: u8 = 1;
|
||||
const MRF_JOURNAL_VERSION_SCOPED: u8 = 2;
|
||||
|
||||
/// Fixed header size: format, version, kind, attempts, enqueued_at_ms,
|
||||
/// has_version flag.
|
||||
const MRF_RECORD_FIXED_HEAD: usize = 1 + 1 + 1 + 1 + 8 + 1;
|
||||
const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;
|
||||
|
||||
fn metric_f64(value: usize) -> f64 {
|
||||
f64::from(u32::try_from(value).unwrap_or(u32::MAX))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct MrfConsumerConfig {
|
||||
@@ -113,90 +101,40 @@ impl Default for MrfConsumerConfig {
|
||||
/// incoming intent (never a resident one) and counts the loss.
|
||||
pub(crate) struct MrfQueue {
|
||||
pending: VecDeque<MrfIntent>,
|
||||
pending_keys: HashSet<MrfQueueKey>,
|
||||
bytes: usize,
|
||||
capacity: usize,
|
||||
byte_budget: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
struct MrfQueueKey {
|
||||
kind: rustfs_common::mrf_channel::MrfKind,
|
||||
bucket: Arc<str>,
|
||||
object: Arc<str>,
|
||||
version_id: Option<[u8; 16]>,
|
||||
scope: Option<rustfs_common::mrf_channel::MrfScope>,
|
||||
}
|
||||
|
||||
fn queue_key(intent: &MrfIntent) -> MrfQueueKey {
|
||||
let version_id = intent.version_id.filter(|bytes| *bytes != [0; 16]);
|
||||
let scope = (!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption))
|
||||
.then_some(intent.scope)
|
||||
.flatten();
|
||||
MrfQueueKey {
|
||||
kind: intent.kind,
|
||||
bucket: intent.bucket.clone(),
|
||||
object: intent.object.clone(),
|
||||
version_id,
|
||||
scope,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum MrfQueuePushResult {
|
||||
Enqueued,
|
||||
Coalesced,
|
||||
Rejected,
|
||||
}
|
||||
|
||||
impl MrfQueue {
|
||||
pub(crate) fn new(capacity: usize, byte_budget: usize) -> Self {
|
||||
Self {
|
||||
pending: VecDeque::new(),
|
||||
pending_keys: HashSet::new(),
|
||||
bytes: 0,
|
||||
capacity,
|
||||
byte_budget,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn try_push_typed(&mut self, intent: MrfIntent) -> MrfQueuePushResult {
|
||||
if intent.bucket.len() > MRF_MAX_IDENTITY_COMPONENT || intent.object.len() > MRF_MAX_IDENTITY_COMPONENT {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "identity_oversized").increment(1);
|
||||
return MrfQueuePushResult::Rejected;
|
||||
}
|
||||
let key = queue_key(&intent);
|
||||
if self.pending_keys.contains(&key) {
|
||||
counter!("rustfs_heal_mrf_coalesced_total", "layer" => "queue").increment(1);
|
||||
return MrfQueuePushResult::Coalesced;
|
||||
}
|
||||
/// Returns `false` (after counting) when either ceiling would be crossed.
|
||||
pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool {
|
||||
let cost = intent.estimated_bytes();
|
||||
if self.pending.len() >= self.capacity || self.bytes + cost > self.byte_budget {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1);
|
||||
return MrfQueuePushResult::Rejected;
|
||||
return false;
|
||||
}
|
||||
self.bytes += cost;
|
||||
self.pending_keys.insert(key);
|
||||
self.pending.push_back(intent);
|
||||
MrfQueuePushResult::Enqueued
|
||||
}
|
||||
|
||||
/// Bool compatibility adapter: only a newly executable queue item is
|
||||
/// reported as accepted; a coalesced duplicate is not durable admission.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool {
|
||||
matches!(self.try_push_typed(intent), MrfQueuePushResult::Enqueued)
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn pop_front(&mut self) -> Option<MrfIntent> {
|
||||
let intent = self.pending.pop_front()?;
|
||||
self.pending_keys.remove(&queue_key(&intent));
|
||||
self.bytes = self.bytes.saturating_sub(intent.estimated_bytes());
|
||||
Some(intent)
|
||||
}
|
||||
|
||||
pub(crate) fn push_back(&mut self, intent: MrfIntent) {
|
||||
self.pending_keys.insert(queue_key(&intent));
|
||||
self.bytes += intent.estimated_bytes();
|
||||
self.pending.push_back(intent);
|
||||
}
|
||||
@@ -219,24 +157,10 @@ impl MrfQueue {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Append one encoded record to `out`.
|
||||
pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) -> bool {
|
||||
let Ok(bucket_len) = u32::try_from(intent.bucket.len()) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(object_len) = u32::try_from(intent.object.len()) else {
|
||||
return false;
|
||||
};
|
||||
let scope = (!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption))
|
||||
.then_some(intent.scope)
|
||||
.flatten();
|
||||
let version_id = intent.version_id.filter(|bytes| *bytes != [0; 16]);
|
||||
pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) {
|
||||
let start = out.len();
|
||||
out.push(MRF_JOURNAL_FORMAT);
|
||||
out.push(if scope.is_some() {
|
||||
MRF_JOURNAL_VERSION_SCOPED
|
||||
} else {
|
||||
MRF_JOURNAL_VERSION
|
||||
});
|
||||
out.push(MRF_JOURNAL_VERSION);
|
||||
out.push(match intent.kind {
|
||||
rustfs_common::mrf_channel::MrfKind::DecodeFailure => 1,
|
||||
rustfs_common::mrf_channel::MrfKind::MetadataCorruption => 2,
|
||||
@@ -244,36 +168,27 @@ pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec<u8>) -> bool {
|
||||
});
|
||||
out.push(intent.attempts);
|
||||
out.extend_from_slice(&intent.enqueued_at_ms.to_le_bytes());
|
||||
match version_id {
|
||||
match intent.version_id {
|
||||
Some(bytes) => {
|
||||
out.push(1);
|
||||
out.extend_from_slice(&bytes);
|
||||
}
|
||||
None => out.push(0),
|
||||
}
|
||||
if let Some(scope) = scope {
|
||||
out.extend_from_slice(&scope.pool_index.to_le_bytes());
|
||||
out.extend_from_slice(&scope.set_index.to_le_bytes());
|
||||
}
|
||||
out.extend_from_slice(&bucket_len.to_le_bytes());
|
||||
out.extend_from_slice(&object_len.to_le_bytes());
|
||||
out.extend_from_slice(&(intent.bucket.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&(intent.object.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(intent.bucket.as_bytes());
|
||||
out.extend_from_slice(intent.object.as_bytes());
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(&out[start..]);
|
||||
let Ok(checksum) = u32::try_from(hasher.finalize()) else {
|
||||
out.truncate(start);
|
||||
return false;
|
||||
};
|
||||
out.extend_from_slice(&checksum.to_le_bytes());
|
||||
true
|
||||
out.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes());
|
||||
}
|
||||
|
||||
fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
|
||||
if data.len() < MRF_RECORD_FIXED_HEAD + 8 {
|
||||
return None;
|
||||
}
|
||||
if data[0] != MRF_JOURNAL_FORMAT || !matches!(data[1], MRF_JOURNAL_VERSION | MRF_JOURNAL_VERSION_SCOPED) {
|
||||
if data[0] != MRF_JOURNAL_FORMAT || data[1] != MRF_JOURNAL_VERSION {
|
||||
return None;
|
||||
}
|
||||
let kind = match data[2] {
|
||||
@@ -283,38 +198,24 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
|
||||
_ => return None,
|
||||
};
|
||||
let attempts = data[3];
|
||||
let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().ok()?);
|
||||
let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().expect("slice length checked"));
|
||||
let has_version = data[12] != 0;
|
||||
let mut cursor = MRF_RECORD_FIXED_HEAD;
|
||||
let version_id = if has_version {
|
||||
if data.len() < cursor + 16 {
|
||||
return None;
|
||||
}
|
||||
let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().ok()?;
|
||||
let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().expect("slice length checked");
|
||||
cursor += 16;
|
||||
Some(bytes)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let scope = if data[1] == MRF_JOURNAL_VERSION_SCOPED {
|
||||
if data.len() < cursor + 8 {
|
||||
return None;
|
||||
}
|
||||
let pool_index = u32::from_le_bytes(data[cursor..cursor + 4].try_into().ok()?);
|
||||
let set_index = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().ok()?);
|
||||
cursor += 8;
|
||||
Some(rustfs_common::mrf_channel::MrfScope { pool_index, set_index })
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if data.len() < cursor + 8 {
|
||||
return None;
|
||||
}
|
||||
let bucket_len = usize::try_from(u32::from_le_bytes(data[cursor..cursor + 4].try_into().ok()?)).ok()?;
|
||||
let object_len = usize::try_from(u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().ok()?)).ok()?;
|
||||
if bucket_len > MRF_MAX_IDENTITY_COMPONENT || object_len > MRF_MAX_IDENTITY_COMPONENT {
|
||||
return None;
|
||||
}
|
||||
let bucket_len = u32::from_le_bytes(data[cursor..cursor + 4].try_into().expect("slice length checked")) as usize;
|
||||
let object_len = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().expect("slice length checked")) as usize;
|
||||
cursor += 8;
|
||||
let body_end = cursor.checked_add(bucket_len)?.checked_add(object_len)?;
|
||||
let record_end = body_end.checked_add(4)?;
|
||||
@@ -323,7 +224,7 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
|
||||
}
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(&data[..body_end]);
|
||||
if u32::try_from(hasher.finalize()).ok()? != u32::from_le_bytes(data[body_end..record_end].try_into().ok()?) {
|
||||
if (hasher.finalize() as u32) != u32::from_le_bytes(data[body_end..record_end].try_into().expect("slice length checked")) {
|
||||
return None;
|
||||
}
|
||||
let bucket = std::sync::Arc::from(std::str::from_utf8(&data[cursor..cursor + bucket_len]).ok()?);
|
||||
@@ -334,12 +235,6 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> {
|
||||
object,
|
||||
version_id,
|
||||
kind,
|
||||
scope: if matches!(kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) {
|
||||
None
|
||||
} else {
|
||||
scope
|
||||
},
|
||||
lease: None,
|
||||
enqueued_at_ms,
|
||||
attempts,
|
||||
},
|
||||
@@ -374,9 +269,9 @@ async fn journal_disks() -> Vec<DiskStore> {
|
||||
map.values().flatten().cloned().collect()
|
||||
}
|
||||
|
||||
async fn read_journal(path: &str) -> Option<Vec<u8>> {
|
||||
async fn read_journal() -> Option<Vec<u8>> {
|
||||
for disk in journal_disks().await {
|
||||
match disk.read_all(super::RUSTFS_META_BUCKET, path).await {
|
||||
match disk.read_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH).await {
|
||||
Ok(bytes) => return Some(bytes.to_vec()),
|
||||
Err(_) => continue,
|
||||
}
|
||||
@@ -387,51 +282,35 @@ async fn read_journal(path: &str) -> Option<Vec<u8>> {
|
||||
/// Write the snapshot to every local disk; returns true when at least one
|
||||
/// disk accepted it, so a total write failure keeps the runtime dirty and
|
||||
/// the next tick retries the persist.
|
||||
async fn write_journal(path: &str, data: &[u8]) -> bool {
|
||||
async fn write_journal(data: &[u8]) -> bool {
|
||||
let payload = bytes::Bytes::copy_from_slice(data);
|
||||
let mut any_persisted = false;
|
||||
for disk in journal_disks().await {
|
||||
match disk.write_all(super::RUSTFS_META_BUCKET, path, payload.clone()).await {
|
||||
match disk
|
||||
.write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone())
|
||||
.await
|
||||
{
|
||||
Ok(()) => any_persisted = true,
|
||||
Err(err) => warn_mrf_journal_write(&err),
|
||||
}
|
||||
}
|
||||
if !data.is_empty() {
|
||||
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64);
|
||||
any_persisted
|
||||
}
|
||||
|
||||
async fn delete_journal(path: &str) -> bool {
|
||||
let disks = journal_disks().await;
|
||||
if disks.is_empty() {
|
||||
counter!("rustfs_heal_mrf_journal_delete_failures_total").increment(1);
|
||||
return false;
|
||||
}
|
||||
let mut all_deleted = true;
|
||||
for disk in disks {
|
||||
let result = disk
|
||||
async fn delete_journal() {
|
||||
for disk in journal_disks().await {
|
||||
let _ = disk
|
||||
.delete(
|
||||
super::RUSTFS_META_BUCKET,
|
||||
path,
|
||||
MRF_JOURNAL_PATH,
|
||||
crate::heal::storage_api::owner::EcstoreDeleteOptions::default(),
|
||||
)
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
// Delete is idempotent: a compatibility mirror that was never
|
||||
// written (or was already removed) is clean, not a retry state.
|
||||
if !matches!(err, super::DiskError::FileNotFound | super::DiskError::VolumeNotFound) {
|
||||
all_deleted = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !all_deleted {
|
||||
counter!("rustfs_heal_mrf_journal_delete_failures_total").increment(1);
|
||||
}
|
||||
all_deleted
|
||||
}
|
||||
|
||||
async fn delete_journals() -> bool {
|
||||
let authoritative_deleted = delete_journal(MRF_SCOPED_JOURNAL_PATH).await;
|
||||
let legacy_deleted = delete_journal(MRF_JOURNAL_PATH).await;
|
||||
authoritative_deleted && legacy_deleted
|
||||
}
|
||||
|
||||
fn warn_mrf_journal_write(err: &super::DiskError) {
|
||||
@@ -452,10 +331,7 @@ fn warn_mrf_journal_write(err: &super::DiskError) {
|
||||
pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
|
||||
let bucket = intent.bucket.to_string();
|
||||
let object = intent.object.to_string();
|
||||
let version_id = intent
|
||||
.version_id
|
||||
.filter(|bytes| *bytes != [0; 16])
|
||||
.map(|bytes| Uuid::from_bytes(bytes).to_string());
|
||||
let version_id = intent.version_id.map(|bytes| Uuid::from_bytes(bytes).to_string());
|
||||
let (heal_type, priority) = match intent.kind {
|
||||
rustfs_common::mrf_channel::MrfKind::DecodeFailure => (
|
||||
HealType::ECDecode {
|
||||
@@ -475,28 +351,18 @@ pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
|
||||
HealPriority::Normal,
|
||||
),
|
||||
};
|
||||
let mut options = HealOptions::default();
|
||||
if !matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption)
|
||||
&& let Some(scope) = intent.scope
|
||||
{
|
||||
options.pool_index = usize::try_from(scope.pool_index).ok();
|
||||
options.set_index = usize::try_from(scope.set_index).ok();
|
||||
}
|
||||
let mut request = HealRequest::new(heal_type, options, priority);
|
||||
let mut request = HealRequest::new(heal_type, HealOptions::default(), priority);
|
||||
request.source = rustfs_common::heal_channel::HealRequestSource::Mrf;
|
||||
request
|
||||
}
|
||||
|
||||
async fn submit_mrf_heal_request(manager: &HealManager, intent: &MrfIntent) -> crate::Result<HealAdmissionResult> {
|
||||
let receipt = manager
|
||||
.submit_mrf_heal_request_with_receipt_and_identity(
|
||||
.submit_mrf_heal_request_with_receipt(
|
||||
build_heal_request(intent),
|
||||
intent.bucket.clone(),
|
||||
intent.object.clone(),
|
||||
intent.version_id,
|
||||
intent.kind,
|
||||
intent.scope,
|
||||
intent.lease,
|
||||
)
|
||||
.await?;
|
||||
Ok(receipt.result)
|
||||
@@ -521,38 +387,16 @@ struct MrfRuntime {
|
||||
}
|
||||
|
||||
impl MrfRuntime {
|
||||
fn snapshot(&self) -> (Vec<u8>, Vec<u8>) {
|
||||
let mut authoritative = Vec::new();
|
||||
let mut legacy = Vec::new();
|
||||
fn snapshot(&self) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
for intent in self.queue.intents() {
|
||||
let scoped_identity =
|
||||
!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) && intent.scope.is_some();
|
||||
if !encode_intent(intent, &mut authoritative) {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "journal_identity_oversized").increment(1);
|
||||
}
|
||||
if !scoped_identity && !encode_intent(intent, &mut legacy) {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "journal_identity_oversized").increment(1);
|
||||
}
|
||||
encode_intent(intent, &mut buf);
|
||||
}
|
||||
(authoritative, legacy)
|
||||
buf
|
||||
}
|
||||
|
||||
async fn flush(&mut self) {
|
||||
let (authoritative, legacy) = self.snapshot();
|
||||
let authoritative_persisted = write_journal(MRF_SCOPED_JOURNAL_PATH, &authoritative).await;
|
||||
if !authoritative.is_empty() {
|
||||
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(metric_f64(authoritative.len()));
|
||||
// Publish the compatibility mirror only after the authoritative
|
||||
// snapshot has reached at least one disk. This ordering prevents an
|
||||
// old reader from observing a newer epoch that a new reader cannot
|
||||
// see when the canonical write is unavailable.
|
||||
let legacy_persisted = authoritative_persisted && write_journal(MRF_JOURNAL_PATH, &legacy).await;
|
||||
// Keep dirty until both the authoritative snapshot and its
|
||||
// compatibility mirror have been accepted; otherwise a one-sided
|
||||
// failure would never retry the missing file.
|
||||
let persisted = authoritative_persisted && legacy_persisted;
|
||||
let persisted = write_journal(&self.snapshot()).await;
|
||||
self.new_since_flush = 0;
|
||||
// Keep the dirty flag when every disk write failed: a clean backlog
|
||||
// would otherwise never rewrite, losing the periodic persist retry a
|
||||
@@ -560,7 +404,7 @@ impl MrfRuntime {
|
||||
if persisted {
|
||||
self.dirty = false;
|
||||
}
|
||||
self.journal_on_disk |= authoritative_persisted || legacy_persisted;
|
||||
self.journal_on_disk = true;
|
||||
}
|
||||
|
||||
/// Drain pending intents into the heal manager until it is full, the
|
||||
@@ -586,7 +430,6 @@ impl MrfRuntime {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
if intent.attempts >= MRF_MAX_ATTEMPTS {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1);
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
continue;
|
||||
}
|
||||
self.queue.push_back(intent);
|
||||
@@ -595,13 +438,11 @@ impl MrfRuntime {
|
||||
}
|
||||
Ok(HealAdmissionResult::Dropped(_)) => {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "admission_policy").increment(1);
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
}
|
||||
Err(_) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
if intent.attempts >= MRF_MAX_ATTEMPTS {
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1);
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
continue;
|
||||
}
|
||||
self.queue.push_back(intent);
|
||||
@@ -610,8 +451,8 @@ impl MrfRuntime {
|
||||
}
|
||||
}
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(self.queue.depth()));
|
||||
gauge!("rustfs_heal_mrf_queue_bytes").set(metric_f64(self.queue.bytes()));
|
||||
gauge!("rustfs_heal_mrf_queue_depth").set(self.queue.depth() as f64);
|
||||
gauge!("rustfs_heal_mrf_queue_bytes").set(self.queue.bytes() as f64);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -655,12 +496,7 @@ pub async fn replay_journal_once(manager: &Arc<HealManager>) -> usize {
|
||||
let config = MrfConsumerConfig::default();
|
||||
let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes);
|
||||
let mut backoff_until: Option<tokio::time::Instant> = None;
|
||||
replay_into(manager, &mut queue, &mut backoff_until).await.replayed
|
||||
}
|
||||
|
||||
struct ReplayOutcome {
|
||||
replayed: usize,
|
||||
journal_on_disk: bool,
|
||||
replay_into(manager, &mut queue, &mut backoff_until).await
|
||||
}
|
||||
|
||||
/// Shared replay core: read + decode + re-arm + delete, then drain what fits.
|
||||
@@ -668,25 +504,11 @@ async fn replay_into(
|
||||
manager: &Arc<HealManager>,
|
||||
queue: &mut MrfQueue,
|
||||
backoff_until: &mut Option<tokio::time::Instant>,
|
||||
) -> ReplayOutcome {
|
||||
// The scoped file is a complete authoritative snapshot. Fall back to the
|
||||
// legacy mirror only when the authoritative path is unavailable; merging
|
||||
// both files could combine records from different flush epochs.
|
||||
let data = match read_journal(MRF_SCOPED_JOURNAL_PATH).await {
|
||||
Some(data) => data,
|
||||
None => match read_journal(MRF_JOURNAL_PATH).await {
|
||||
Some(data) => data,
|
||||
None => {
|
||||
return ReplayOutcome {
|
||||
replayed: 0,
|
||||
journal_on_disk: false,
|
||||
};
|
||||
}
|
||||
},
|
||||
) -> usize {
|
||||
let Some(data) = read_journal().await else {
|
||||
return 0;
|
||||
};
|
||||
let mut intents = Vec::new();
|
||||
let (decoded, truncated) = decode_journal(&data);
|
||||
intents.extend(decoded);
|
||||
let (intents, truncated) = decode_journal(&data);
|
||||
if truncated > 0 {
|
||||
tracing::warn!(
|
||||
target: "rustfs::heal::mrf",
|
||||
@@ -694,15 +516,12 @@ async fn replay_into(
|
||||
"MRF journal had a torn tail; truncated records were discarded"
|
||||
);
|
||||
}
|
||||
counter!("rustfs_heal_mrf_replayed_total").increment(u64::try_from(intents.len()).unwrap_or(u64::MAX));
|
||||
counter!("rustfs_heal_mrf_replayed_total").increment(intents.len() as u64);
|
||||
let replayed = intents.len();
|
||||
for intent in intents {
|
||||
let result = queue.try_push_typed(intent.clone());
|
||||
if !matches!(result, MrfQueuePushResult::Enqueued) {
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
}
|
||||
queue.try_push(intent);
|
||||
}
|
||||
let journal_on_disk = !delete_journals().await;
|
||||
delete_journal().await;
|
||||
|
||||
// Drain the replayed intents immediately; whatever the manager refuses
|
||||
// stays armed in `queue` for the consumer's retry loop.
|
||||
@@ -722,10 +541,7 @@ async fn replay_into(
|
||||
}
|
||||
}
|
||||
}
|
||||
ReplayOutcome {
|
||||
replayed,
|
||||
journal_on_disk,
|
||||
}
|
||||
replayed
|
||||
}
|
||||
|
||||
/// Replay the journal, then keep draining the channel into the heal manager
|
||||
@@ -743,8 +559,7 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
|
||||
// Replay: read the journal, re-arm intents (duplicates are merged by the
|
||||
// manager's dedup key), then drop the file so the next flush starts clean.
|
||||
let replay = replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
|
||||
runtime.journal_on_disk = replay.journal_on_disk;
|
||||
replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
|
||||
// The replay deleted the journal file; anything still pending (e.g. the
|
||||
// manager was full and backoff armed) must be re-persisted by the next
|
||||
// flush or a crash before it would lose those intents.
|
||||
@@ -772,14 +587,9 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
return;
|
||||
}
|
||||
for intent in batch.drain(..) {
|
||||
match runtime.queue.try_push_typed(intent.clone()) {
|
||||
MrfQueuePushResult::Enqueued => {
|
||||
runtime.new_since_flush += 1;
|
||||
runtime.dirty = true;
|
||||
}
|
||||
MrfQueuePushResult::Coalesced | MrfQueuePushResult::Rejected => {
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
}
|
||||
if runtime.queue.try_push(intent) {
|
||||
runtime.new_since_flush += 1;
|
||||
runtime.dirty = true;
|
||||
}
|
||||
}
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
@@ -803,14 +613,13 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
TickAction::DeleteJournal => {
|
||||
// All intents consumed: remove the journal so a restart
|
||||
// replays nothing (mirrors MinIO's post-replay unlink).
|
||||
if delete_journals().await {
|
||||
runtime.journal_on_disk = false;
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
|
||||
}
|
||||
delete_journal().await;
|
||||
runtime.journal_on_disk = false;
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
|
||||
}
|
||||
TickAction::Idle => {}
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(runtime.queue.depth()));
|
||||
gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -855,8 +664,6 @@ mod tests {
|
||||
object: StdArc::from(object),
|
||||
version_id: Some([7u8; 16]),
|
||||
kind: MrfKind::DecodeFailure,
|
||||
scope: None,
|
||||
lease: None,
|
||||
enqueued_at_ms: 1_700_000_000_000,
|
||||
attempts,
|
||||
}
|
||||
@@ -887,101 +694,17 @@ mod tests {
|
||||
fn queue_enforces_count_and_byte_ceilings() {
|
||||
let mut queue = MrfQueue::new(2, usize::MAX);
|
||||
assert!(queue.try_push(intent("b", "o", 0)));
|
||||
assert!(queue.try_push(intent("b", "o2", 0)));
|
||||
assert!(!queue.try_push(intent("b", "o3", 0)), "count ceiling must drop");
|
||||
assert!(queue.try_push(intent("b", "o", 0)));
|
||||
assert!(!queue.try_push(intent("b", "o", 0)), "count ceiling must drop");
|
||||
|
||||
let mut tiny = MrfQueue::new(usize::MAX, intent("bucket", "object", 0).estimated_bytes());
|
||||
assert!(tiny.try_push(intent("bucket", "object", 0)));
|
||||
assert!(
|
||||
!tiny.try_push(intent("bucket", "object2", 0)),
|
||||
!tiny.try_push(intent("bucket", "object", 0)),
|
||||
"byte budget must drop before the second intent fits"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_mrf_intents_coalesce_to_one_execution() {
|
||||
let mut queue = MrfQueue::new(1000, usize::MAX);
|
||||
let mut enqueued = 0;
|
||||
let mut coalesced = 0;
|
||||
assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
|
||||
enqueued += 1;
|
||||
for _ in 0..999 {
|
||||
match queue.try_push_typed(intent("bucket", "object", 0)) {
|
||||
MrfQueuePushResult::Coalesced => coalesced += 1,
|
||||
other => panic!("duplicate intent was not coalesced: {other:?}"),
|
||||
}
|
||||
}
|
||||
assert_eq!(enqueued, 1);
|
||||
assert_eq!(coalesced, 999);
|
||||
assert_eq!(queue.depth(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_dedupe_does_not_merge_adjacent_version_pool_or_kind() {
|
||||
let mut queue = MrfQueue::new(8, usize::MAX);
|
||||
let mut first = intent("bucket", "object", 0);
|
||||
first.kind = MrfKind::PartialWrite;
|
||||
first.scope = Some(rustfs_common::mrf_channel::MrfScope {
|
||||
pool_index: 1,
|
||||
set_index: 1,
|
||||
});
|
||||
assert!(queue.try_push(first.clone()));
|
||||
first.version_id = Some([8u8; 16]);
|
||||
assert!(queue.try_push(first));
|
||||
let mut other_scope = intent("bucket", "object", 0);
|
||||
other_scope.kind = MrfKind::PartialWrite;
|
||||
other_scope.scope = Some(rustfs_common::mrf_channel::MrfScope {
|
||||
pool_index: 2,
|
||||
set_index: 1,
|
||||
});
|
||||
assert!(queue.try_push(other_scope));
|
||||
let mut other_kind = intent("bucket", "object", 0);
|
||||
other_kind.kind = MrfKind::DecodeFailure;
|
||||
other_kind.scope = None;
|
||||
assert!(queue.try_push(other_kind));
|
||||
assert_eq!(queue.depth(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_dedupe_full_returns_rejected_with_durable_pending() {
|
||||
let mut queue = MrfQueue::new(1, usize::MAX);
|
||||
assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
|
||||
assert_eq!(queue.try_push_typed(intent("bucket", "other", 0)), MrfQueuePushResult::Rejected);
|
||||
assert_eq!(queue.depth(), 1);
|
||||
let mut snapshot = Vec::new();
|
||||
assert!(encode_intent(queue.intents().next().expect("resident intent"), &mut snapshot));
|
||||
assert!(!snapshot.is_empty(), "the resident intent remains journalable after rejection");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_dedupe_failure_releases_key_for_retry() {
|
||||
let mut queue = MrfQueue::new(1, usize::MAX);
|
||||
assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
|
||||
let _failed = queue.pop_front().expect("queued intent");
|
||||
assert_eq!(queue.try_push_typed(intent("bucket", "object", 1)), MrfQueuePushResult::Enqueued);
|
||||
assert_eq!(queue.depth(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_dedupe_key_and_map_are_bounded() {
|
||||
let mut queue = MrfQueue::new(2, usize::MAX);
|
||||
assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
|
||||
assert_eq!(queue.try_push_typed(intent("bucket", "other", 0)), MrfQueuePushResult::Enqueued);
|
||||
assert_eq!(queue.pending_keys.len(), 2);
|
||||
assert_eq!(queue.try_push_typed(intent("bucket", "third", 0)), MrfQueuePushResult::Rejected);
|
||||
assert_eq!(queue.depth(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cross_node_duplicate_execution_remains_idempotent() {
|
||||
// Node-local ingress maps intentionally do not merge across nodes;
|
||||
// the manager's existing identity key absorbs the duplicate later.
|
||||
let mut node_a = MrfQueue::new(8, usize::MAX);
|
||||
let mut node_b = MrfQueue::new(8, usize::MAX);
|
||||
assert_eq!(node_a.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
|
||||
assert_eq!(node_b.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn journal_roundtrip_preserves_intents() {
|
||||
let intents = vec![
|
||||
@@ -992,8 +715,6 @@ mod tests {
|
||||
object: StdArc::from("object/c"),
|
||||
version_id: None,
|
||||
kind: MrfKind::MetadataCorruption,
|
||||
scope: None,
|
||||
lease: None,
|
||||
enqueued_at_ms: 5,
|
||||
attempts: 1,
|
||||
},
|
||||
@@ -1045,8 +766,6 @@ mod tests {
|
||||
object: StdArc::from("o"),
|
||||
version_id: None,
|
||||
kind: MrfKind::MetadataCorruption,
|
||||
scope: None,
|
||||
lease: None,
|
||||
enqueued_at_ms: 0,
|
||||
attempts: 0,
|
||||
});
|
||||
@@ -1058,8 +777,6 @@ mod tests {
|
||||
object: StdArc::from("o"),
|
||||
version_id: None,
|
||||
kind: MrfKind::PartialWrite,
|
||||
scope: None,
|
||||
lease: None,
|
||||
enqueued_at_ms: 0,
|
||||
attempts: 0,
|
||||
});
|
||||
|
||||
@@ -51,6 +51,7 @@ const RESUME_STATE_FILE: &str = "ahm_resume_state.json";
|
||||
const REPLACEMENT_INTENT_FILE: &str = "ahm_replacement_intent.json";
|
||||
const RESUME_PROGRESS_FILE: &str = "ahm_progress.json";
|
||||
pub(super) const RESUME_CHECKPOINT_FILE: &str = "ahm_checkpoint.json";
|
||||
pub(super) const RESUME_CHECKPOINT_BLOCKED_FILE: &str = "ahm_checkpoint.blocked";
|
||||
const REPLACEMENT_COMPLETION_PROOF_FILE: &str = "ahm_replacement_completion_proof.json";
|
||||
const REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
|
||||
const REPLACEMENT_INTENT_SEAL_FILE: &str = "ahm_replacement_intent_seal";
|
||||
|
||||
@@ -13,26 +13,31 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{Error, Result};
|
||||
use base64::Engine as _;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::sync::{Mutex as AsyncMutex, RwLock};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
|
||||
use super::super::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes};
|
||||
use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt, RUSTFS_META_BUCKET};
|
||||
use super::{
|
||||
LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_FILE, delete_resume_file, path_to_str,
|
||||
validate_resume_task_id,
|
||||
LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_BLOCKED_FILE, RESUME_CHECKPOINT_FILE,
|
||||
delete_resume_file, path_to_str, validate_resume_task_id,
|
||||
};
|
||||
|
||||
const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state";
|
||||
const RESUME_CHECKPOINT_DIGEST_FILE: &str = "ahm_checkpoint.sha256";
|
||||
const CHECKPOINT_PER_VERSION_SCHEMA: u32 = 5;
|
||||
|
||||
/// Current on-disk schema version for `ResumeCheckpoint`. Same rationale as
|
||||
/// `CURRENT_RESUME_SCHEMA`: pre-per-version dedup identities are not comparable
|
||||
/// to the new `compose_key` identities, so a stale checkpoint is discarded.
|
||||
pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 5;
|
||||
pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 6;
|
||||
|
||||
/// resume checkpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -57,6 +62,11 @@ pub struct ResumeCheckpoint {
|
||||
pub failed_objects: HashSet<String>,
|
||||
/// skipped objects
|
||||
pub skipped_objects: HashSet<String>,
|
||||
/// Integrity digest over the checkpoint with this field set to `None`.
|
||||
/// Keeping it in the checkpoint makes the payload and its authentication
|
||||
/// record one CAS generation instead of two independently-written files.
|
||||
#[serde(default)]
|
||||
pub integrity_digest: Option<String>,
|
||||
}
|
||||
|
||||
impl ResumeCheckpoint {
|
||||
@@ -70,6 +80,7 @@ impl ResumeCheckpoint {
|
||||
processed_objects: HashSet::new(),
|
||||
failed_objects: HashSet::new(),
|
||||
skipped_objects: HashSet::new(),
|
||||
integrity_digest: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,17 +127,111 @@ pub struct CheckpointManager {
|
||||
disk: DiskStore,
|
||||
checkpoint: Arc<RwLock<ResumeCheckpoint>>,
|
||||
throttle: Mutex<PersistThrottle>,
|
||||
save_lock: AsyncMutex<()>,
|
||||
last_saved: Mutex<Option<EcstoreDiskBytes>>,
|
||||
}
|
||||
|
||||
impl CheckpointManager {
|
||||
fn blocked_path(task_id: &str) -> std::path::PathBuf {
|
||||
Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}"))
|
||||
}
|
||||
|
||||
/// Return whether a checkpoint was permanently isolated after a malformed
|
||||
/// or unsupported snapshot was observed.
|
||||
pub(crate) async fn is_blocked(disk: &DiskStore, task_id: &str) -> bool {
|
||||
if validate_resume_task_id(task_id).is_err() {
|
||||
return false;
|
||||
}
|
||||
let blocked_path = Self::blocked_path(task_id);
|
||||
let Ok(path) = path_to_str(&blocked_path) else {
|
||||
return false;
|
||||
};
|
||||
match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
|
||||
Ok(_) => true,
|
||||
Err(crate::heal::DiskError::FileNotFound) => false,
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate the checkpoint while enumerating resumable state. This reads
|
||||
/// the checkpoint once and also isolates malformed or unsupported data.
|
||||
pub(crate) async fn is_resumable(disk: &DiskStore, task_id: &str) -> Result<bool> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
if Self::is_blocked(disk, task_id).await {
|
||||
return Err(Error::InvalidCheckpoint(format!("Resume task {task_id} has a blocked checkpoint")));
|
||||
}
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
let Ok(path) = path_to_str(&file_path) else {
|
||||
return Err(Error::InvalidCheckpoint("Resume checkpoint path is not valid UTF-8".to_string()));
|
||||
};
|
||||
match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await {
|
||||
Ok(bytes) if bytes.is_empty() => Ok(true),
|
||||
Ok(bytes) => Self::load_from_data(disk.clone(), task_id, bytes.to_vec())
|
||||
.await
|
||||
.map(|_| true),
|
||||
Err(crate::heal::DiskError::FileNotFound) => Ok(true),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn block_invalid_snapshot(disk: &DiskStore, task_id: &str) {
|
||||
// This marker is intentionally version-agnostic: an unsupported reader
|
||||
// must stop selector retries until an operator cleans up the snapshot.
|
||||
let blocked_path = Self::blocked_path(task_id);
|
||||
let Ok(path) = path_to_str(&blocked_path) else {
|
||||
return;
|
||||
};
|
||||
let result = EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
None,
|
||||
Some(EcstoreDiskBytes::from_static(b"blocked")),
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(EcstoreConditionalFileUpdate::Updated | EcstoreConditionalFileUpdate::Mismatch) => {}
|
||||
Ok(EcstoreConditionalFileUpdate::Missing) => warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
state = "blocked_marker_write_failed",
|
||||
error = "marker target disappeared",
|
||||
"Heal checkpoint could not persist its blocked marker"
|
||||
),
|
||||
Err(error) => warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
state = "blocked_marker_write_failed",
|
||||
error = %error,
|
||||
"Heal checkpoint could not persist its blocked marker"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// create new checkpoint manager
|
||||
pub async fn new(disk: DiskStore, task_id: String) -> Result<Self> {
|
||||
validate_resume_task_id(&task_id)?;
|
||||
let checkpoint_volume = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}");
|
||||
if let Err(error) = EcstoreDiskAPI::make_volume(disk.as_ref(), &checkpoint_volume).await
|
||||
&& error != crate::heal::DiskError::VolumeExists
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to create checkpoint volume: {error}"),
|
||||
});
|
||||
}
|
||||
let checkpoint = ResumeCheckpoint::new(task_id);
|
||||
let manager = Self {
|
||||
disk,
|
||||
checkpoint: Arc::new(RwLock::new(checkpoint)),
|
||||
throttle: Mutex::new(PersistThrottle::new()),
|
||||
save_lock: AsyncMutex::new(()),
|
||||
last_saved: Mutex::new(None),
|
||||
};
|
||||
|
||||
// save initial checkpoint
|
||||
@@ -140,6 +245,7 @@ impl CheckpointManager {
|
||||
error = %e,
|
||||
"Heal checkpoint persistence failed"
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
Ok(manager)
|
||||
}
|
||||
@@ -148,11 +254,22 @@ impl CheckpointManager {
|
||||
pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result<Self> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?;
|
||||
let mut checkpoint: ResumeCheckpoint =
|
||||
serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to deserialize checkpoint: {e}"),
|
||||
})?;
|
||||
Self::load_from_data(disk, task_id, checkpoint_data).await
|
||||
}
|
||||
|
||||
async fn load_from_data(disk: DiskStore, task_id: &str, checkpoint_data: Vec<u8>) -> Result<Self> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
let mut checkpoint: ResumeCheckpoint = match serde_json::from_slice(&checkpoint_data) {
|
||||
Ok(checkpoint) => checkpoint,
|
||||
Err(error) => {
|
||||
Self::block_invalid_snapshot(&disk, task_id).await;
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to deserialize checkpoint: {error}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
if checkpoint.task_id != task_id {
|
||||
Self::block_invalid_snapshot(&disk, task_id).await;
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Resume checkpoint task id does not match filename".to_string(),
|
||||
});
|
||||
@@ -163,6 +280,7 @@ impl CheckpointManager {
|
||||
// identities. Discard the stale sets and position, then stamp the
|
||||
// current schema so the scan restarts cleanly.
|
||||
if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA {
|
||||
Self::block_invalid_snapshot(&disk, task_id).await;
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!(
|
||||
"Checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}",
|
||||
@@ -170,7 +288,45 @@ impl CheckpointManager {
|
||||
),
|
||||
});
|
||||
}
|
||||
if checkpoint.schema_version < CURRENT_CHECKPOINT_SCHEMA {
|
||||
|
||||
let integrity_verified = if let Some(expected) = checkpoint.integrity_digest.as_deref() {
|
||||
let actual = Self::checkpoint_digest(&Self::serialize_without_digest(&checkpoint)?);
|
||||
if expected != actual {
|
||||
Self::block_invalid_snapshot(&disk, task_id).await;
|
||||
return Err(Error::InvalidCheckpoint(format!(
|
||||
"Resume checkpoint digest does not match task {task_id}"
|
||||
)));
|
||||
}
|
||||
true
|
||||
} else if checkpoint.schema_version >= CURRENT_CHECKPOINT_SCHEMA {
|
||||
Self::block_invalid_snapshot(&disk, task_id).await;
|
||||
return Err(Error::InvalidCheckpoint(format!(
|
||||
"Resume checkpoint digest is missing for task {task_id}"
|
||||
)));
|
||||
} else {
|
||||
let digest_path = Self::digest_path(task_id);
|
||||
let digest_path = path_to_str(&digest_path)?;
|
||||
match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, digest_path).await {
|
||||
Ok(expected) => {
|
||||
let actual = Self::checkpoint_digest(&checkpoint_data);
|
||||
if expected.as_ref() != actual.as_bytes() {
|
||||
Self::block_invalid_snapshot(&disk, task_id).await;
|
||||
return Err(Error::InvalidCheckpoint(format!(
|
||||
"Resume checkpoint digest does not match task {task_id}"
|
||||
)));
|
||||
}
|
||||
true
|
||||
}
|
||||
Err(crate::heal::DiskError::FileNotFound) => false,
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read checkpoint digest: {error}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if checkpoint.schema_version < CHECKPOINT_PER_VERSION_SCHEMA || !integrity_verified {
|
||||
warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
@@ -187,13 +343,15 @@ impl CheckpointManager {
|
||||
checkpoint.skipped_objects.clear();
|
||||
checkpoint.current_bucket_index = 0;
|
||||
checkpoint.current_object_index = 0;
|
||||
checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA;
|
||||
}
|
||||
checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA;
|
||||
|
||||
Ok(Self {
|
||||
disk,
|
||||
checkpoint: Arc::new(RwLock::new(checkpoint)),
|
||||
throttle: Mutex::new(PersistThrottle::new()),
|
||||
save_lock: AsyncMutex::new(()),
|
||||
last_saved: Mutex::new(Some(EcstoreDiskBytes::from(checkpoint_data))),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -204,7 +362,7 @@ impl CheckpointManager {
|
||||
}
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
match path_to_str(&file_path) {
|
||||
Ok(path_str) => match disk.read_all(RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(path_str) => match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(data) => !data.is_empty(),
|
||||
Err(_) => false,
|
||||
},
|
||||
@@ -292,6 +450,8 @@ impl CheckpointManager {
|
||||
|
||||
let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
delete_resume_file(&self.disk, &checkpoint_file).await?;
|
||||
delete_resume_file(&self.disk, &Self::digest_path(&task_id)).await?;
|
||||
delete_resume_file(&self.disk, &Self::blocked_path(&task_id)).await?;
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
@@ -307,21 +467,130 @@ impl CheckpointManager {
|
||||
|
||||
/// save checkpoint to disk
|
||||
async fn save_checkpoint(&self) -> Result<()> {
|
||||
let checkpoint = self.checkpoint.read().await;
|
||||
// Serialize saves and take the snapshot only after acquiring the lock:
|
||||
// a slower writer must not publish a snapshot taken before a newer one.
|
||||
let _save_guard = self.save_lock.lock().await;
|
||||
let checkpoint = self.checkpoint.read().await.clone();
|
||||
validate_resume_task_id(&checkpoint.task_id)?;
|
||||
let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to serialize checkpoint: {e}"),
|
||||
})?;
|
||||
let unsigned_checkpoint_data = Self::serialize_without_digest(&checkpoint)?;
|
||||
let digest = Self::checkpoint_digest(&unsigned_checkpoint_data);
|
||||
let mut persisted_checkpoint = checkpoint.clone();
|
||||
persisted_checkpoint.integrity_digest = Some(digest);
|
||||
let checkpoint_data =
|
||||
EcstoreDiskBytes::from(serde_json::to_vec(&persisted_checkpoint).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to serialize checkpoint: {e}"),
|
||||
})?);
|
||||
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", checkpoint.task_id, RESUME_CHECKPOINT_FILE));
|
||||
|
||||
let path_str = path_to_str(&file_path)?;
|
||||
self.disk
|
||||
.write_all(RUSTFS_META_BUCKET, path_str, checkpoint_data.into())
|
||||
let last_saved = self
|
||||
.last_saved
|
||||
.lock()
|
||||
.map_err(|_| Error::TaskExecutionFailed {
|
||||
message: "Checkpoint save state lock is poisoned; refusing to save".to_string(),
|
||||
})?
|
||||
.clone();
|
||||
let update = EcstoreDiskAPI::compare_and_update_file(
|
||||
self.disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path_str,
|
||||
last_saved.clone(),
|
||||
Some(checkpoint_data.clone()),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save checkpoint: {e}"),
|
||||
})?;
|
||||
|
||||
let expected = match update {
|
||||
EcstoreConditionalFileUpdate::Updated => None,
|
||||
EcstoreConditionalFileUpdate::Missing => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Checkpoint was removed after this manager saved it; refusing to recreate it".to_string(),
|
||||
});
|
||||
}
|
||||
EcstoreConditionalFileUpdate::Mismatch => {
|
||||
// A healthy manager normally completes the CAS above without
|
||||
// another read or JSON parse. Inspect only after a mismatch so
|
||||
// corruption and future schemas cannot be overwritten blindly.
|
||||
let existing = match HealDiskExt::read_all(self.disk.as_ref(), RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(existing) => existing,
|
||||
Err(crate::heal::DiskError::FileNotFound) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Checkpoint was removed after this manager saved it; refusing to recreate it".to_string(),
|
||||
});
|
||||
}
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to inspect checkpoint after CAS mismatch: {error}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
if existing.is_empty() && last_saved.is_none() {
|
||||
Some(existing)
|
||||
} else {
|
||||
let current: ResumeCheckpoint = match serde_json::from_slice(&existing) {
|
||||
Ok(current) => current,
|
||||
Err(error) => {
|
||||
Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Existing checkpoint is corrupt: {error}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
if current.task_id != checkpoint.task_id {
|
||||
Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Existing checkpoint task id does not match filename".to_string(),
|
||||
});
|
||||
}
|
||||
if current.schema_version > CURRENT_CHECKPOINT_SCHEMA {
|
||||
Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await;
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!(
|
||||
"Existing checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}",
|
||||
current.schema_version
|
||||
),
|
||||
});
|
||||
}
|
||||
if last_saved.as_ref().is_none_or(|saved| saved.as_ref() != existing.as_ref()) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Checkpoint changed since this manager loaded it; refusing to overwrite newer progress"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
Some(existing)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(expected) = expected {
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
self.disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path_str,
|
||||
Some(expected),
|
||||
Some(checkpoint_data.clone()),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save checkpoint: {e}"),
|
||||
})?;
|
||||
message: format!("Failed to save checkpoint after CAS mismatch: {e}"),
|
||||
})? {
|
||||
EcstoreConditionalFileUpdate::Updated => {}
|
||||
EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Checkpoint changed while saving; refusing to overwrite newer progress".to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut last_saved = self.last_saved.lock().map_err(|_| Error::TaskExecutionFailed {
|
||||
message: "Checkpoint save state lock is poisoned after save".to_string(),
|
||||
})?;
|
||||
*last_saved = Some(checkpoint_data);
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
@@ -341,11 +610,38 @@ impl CheckpointManager {
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
|
||||
let path_str = path_to_str(&file_path)?;
|
||||
disk.read_all(RUSTFS_META_BUCKET, path_str)
|
||||
HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path_str)
|
||||
.await
|
||||
.map(|bytes| bytes.to_vec())
|
||||
.map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read checkpoint file: {e}"),
|
||||
})
|
||||
}
|
||||
|
||||
fn serialize_without_digest(checkpoint: &ResumeCheckpoint) -> Result<Vec<u8>> {
|
||||
let mut unsigned = checkpoint.clone();
|
||||
unsigned.integrity_digest = None;
|
||||
let mut value = serde_json::to_value(&unsigned).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to serialize checkpoint: {e}"),
|
||||
})?;
|
||||
for field in ["processed_objects", "failed_objects", "skipped_objects"] {
|
||||
let Some(values) = value.get_mut(field).and_then(serde_json::Value::as_array_mut) else {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to canonicalize checkpoint field: {field}"),
|
||||
});
|
||||
};
|
||||
values.sort_by(|left, right| left.as_str().cmp(&right.as_str()));
|
||||
}
|
||||
serde_json::to_vec(&value).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to serialize checkpoint: {e}"),
|
||||
})
|
||||
}
|
||||
|
||||
fn checkpoint_digest(checkpoint_data: &[u8]) -> String {
|
||||
base64::engine::general_purpose::STANDARD.encode(Sha256::digest(checkpoint_data))
|
||||
}
|
||||
|
||||
fn digest_path(task_id: &str) -> std::path::PathBuf {
|
||||
Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_DIGEST_FILE}"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1600,6 +1600,32 @@ async fn test_checkpoint_schema_v4_discarded_on_load() {
|
||||
temp_dir.close().expect("remove schema test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn downgraded_unsigned_checkpoint_resets_untrusted_progress() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
|
||||
manager.add_processed_object("victim-a".to_string()).await.unwrap();
|
||||
manager.update_position(2, 500).await.unwrap();
|
||||
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
|
||||
let bytes = disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path).await.unwrap();
|
||||
let mut downgraded: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
|
||||
downgraded["schema_version"] = serde_json::json!(CURRENT_CHECKPOINT_SCHEMA - 1);
|
||||
downgraded.as_object_mut().unwrap().remove("integrity_digest");
|
||||
downgraded["processed_objects"] = serde_json::json!(["victim-b"]);
|
||||
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, serde_json::to_vec(&downgraded).unwrap().into())
|
||||
.await
|
||||
.expect("write downgraded checkpoint");
|
||||
|
||||
let manager = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap();
|
||||
let checkpoint = manager.get_checkpoint().await;
|
||||
assert_eq!(checkpoint.schema_version, CURRENT_CHECKPOINT_SCHEMA);
|
||||
assert_eq!(checkpoint.current_bucket_index, 0);
|
||||
assert_eq!(checkpoint.current_object_index, 0);
|
||||
assert!(checkpoint.processed_objects.is_empty());
|
||||
temp_dir.close().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_normal_resume_schema_preserves_progress() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
@@ -1675,6 +1701,369 @@ async fn future_resume_and_checkpoint_schemas_are_rejected() {
|
||||
temp_dir.close().expect("remove schema test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn checkpoint_save_does_not_replace_a_non_empty_truncated_snapshot() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create checkpoint manager");
|
||||
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
|
||||
let truncated = b"{\"schema_version\":5,\"task_id\":";
|
||||
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, truncated.as_slice().into())
|
||||
.await
|
||||
.expect("write truncated checkpoint fixture");
|
||||
|
||||
let error = manager
|
||||
.update_position(2, 7)
|
||||
.await
|
||||
.expect_err("a truncated checkpoint must fail closed during save");
|
||||
assert!(error.to_string().contains("Existing checkpoint is corrupt"));
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
|
||||
.await
|
||||
.expect("read truncated checkpoint fixture"),
|
||||
truncated.as_slice()
|
||||
);
|
||||
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
|
||||
temp_dir.close().expect("remove checkpoint save test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn checkpoint_save_does_not_replace_a_future_schema_snapshot() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create checkpoint manager");
|
||||
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
|
||||
let mut future = ResumeCheckpoint::new(task_id.clone());
|
||||
future.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1;
|
||||
let future_bytes = serde_json::to_vec(&future).expect("serialize future checkpoint fixture");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, future_bytes.clone().into())
|
||||
.await
|
||||
.expect("write future checkpoint fixture");
|
||||
|
||||
let error = manager
|
||||
.update_position(2, 7)
|
||||
.await
|
||||
.expect_err("a future schema must fail closed during save");
|
||||
assert!(error.to_string().contains("Existing checkpoint schema"));
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
|
||||
.await
|
||||
.expect("read future checkpoint fixture"),
|
||||
future_bytes
|
||||
);
|
||||
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
|
||||
temp_dir.close().expect("remove future schema test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn checkpoint_digest_rejects_same_length_progress_tampering() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create checkpoint manager");
|
||||
manager
|
||||
.add_processed_object("victim-a".to_string())
|
||||
.await
|
||||
.expect("persist checkpoint progress");
|
||||
manager.update_position(1, 1).await.expect("flush checkpoint progress");
|
||||
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
|
||||
let original = disk
|
||||
.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
|
||||
.await
|
||||
.expect("read checkpoint fixture");
|
||||
let tampered = original
|
||||
.windows(b"victim-a".len())
|
||||
.position(|window| window == b"victim-a")
|
||||
.map(|index| {
|
||||
let mut bytes = original.to_vec();
|
||||
bytes[index..index + b"victim-a".len()].copy_from_slice(b"victim-b");
|
||||
bytes
|
||||
})
|
||||
.expect("checkpoint should contain the processed object");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, tampered.into())
|
||||
.await
|
||||
.expect("write tampered checkpoint fixture");
|
||||
|
||||
assert!(CheckpointManager::load_from_disk(disk.clone(), &task_id).await.is_err());
|
||||
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
|
||||
temp_dir.close().expect("remove digest test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn checkpoint_integrity_survives_missing_legacy_sidecar() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
|
||||
manager.update_position(2, 9).await.unwrap();
|
||||
|
||||
let digest_path = format!("{BUCKET_META_PREFIX}/{task_id}_ahm_checkpoint.sha256");
|
||||
delete_resume_file(&disk, Path::new(&digest_path)).await.unwrap();
|
||||
|
||||
let restored = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap();
|
||||
let checkpoint = restored.get_checkpoint().await;
|
||||
assert_eq!(checkpoint.current_bucket_index, 2);
|
||||
assert_eq!(checkpoint.current_object_index, 9);
|
||||
assert!(checkpoint.integrity_digest.is_some());
|
||||
temp_dir.close().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn checkpoint_integrity_survives_multi_object_reload() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
|
||||
for index in 0..32 {
|
||||
manager.add_processed_object(format!("processed-{index}")).await.unwrap();
|
||||
manager.add_failed_object(format!("failed-{index}")).await.unwrap();
|
||||
manager.add_skipped_object(format!("skipped-{index}")).await.unwrap();
|
||||
}
|
||||
manager.update_position(2, 9).await.unwrap();
|
||||
|
||||
CheckpointManager::load_from_disk(disk, &task_id)
|
||||
.await
|
||||
.expect("a healthy multi-object checkpoint must survive reload");
|
||||
temp_dir.close().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn checkpoint_integrity_rejects_a_removed_embedded_digest() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
|
||||
manager.update_position(2, 9).await.unwrap();
|
||||
|
||||
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
|
||||
let bytes = disk
|
||||
.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
|
||||
.await
|
||||
.expect("read checkpoint fixture");
|
||||
let mut value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
|
||||
value["current_object_index"] = serde_json::json!(10);
|
||||
value.as_object_mut().unwrap().remove("integrity_digest");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, serde_json::to_vec(&value).unwrap().into())
|
||||
.await
|
||||
.expect("write tampered checkpoint fixture");
|
||||
|
||||
assert!(
|
||||
CheckpointManager::load_from_disk(disk.clone(), &task_id).await.is_err(),
|
||||
"a current checkpoint without its embedded digest must fail closed"
|
||||
);
|
||||
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
|
||||
temp_dir.close().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_checkpoint_manager_rebuilds_an_empty_snapshot() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, EcstoreDiskBytes::new())
|
||||
.await
|
||||
.expect("write empty checkpoint fixture");
|
||||
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("a new manager must rebuild an empty checkpoint");
|
||||
manager
|
||||
.update_position(3, 11)
|
||||
.await
|
||||
.expect("rebuilt checkpoint must remain writable");
|
||||
assert!(CheckpointManager::has_checkpoint(&disk, &task_id).await);
|
||||
temp_dir.close().expect("remove empty checkpoint test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deleted_checkpoint_is_not_recreated_by_an_old_manager() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create checkpoint manager");
|
||||
manager.cleanup().await.expect("delete checkpoint fixture");
|
||||
|
||||
let error = manager
|
||||
.update_position(1, 2)
|
||||
.await
|
||||
.expect_err("an old manager must not resurrect a deleted checkpoint");
|
||||
assert!(error.to_string().contains("removed after this manager saved it"));
|
||||
assert!(!CheckpointManager::has_checkpoint(&disk, &task_id).await);
|
||||
temp_dir.close().expect("remove deleted checkpoint test directory");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn checkpoint_cleanup_leaves_no_task_specific_lock_artifact() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create checkpoint manager");
|
||||
let lock_path = Path::new(BUCKET_META_PREFIX)
|
||||
.join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"))
|
||||
.with_extension("rustfs-cas.lock");
|
||||
let lock_path = temp_dir.path().join(RUSTFS_META_BUCKET).join(lock_path);
|
||||
|
||||
manager.cleanup().await.expect("delete checkpoint fixture");
|
||||
|
||||
assert!(
|
||||
!lock_path.exists(),
|
||||
"successful checkpoint cleanup must not leave a task-specific lock artifact"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_empty_blocked_marker_still_blocks_resume_selection() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create checkpoint manager");
|
||||
let blocked_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &blocked_path, EcstoreDiskBytes::new())
|
||||
.await
|
||||
.expect("write empty blocked marker fixture");
|
||||
|
||||
assert!(CheckpointManager::is_blocked(&disk, &task_id).await);
|
||||
assert!(CheckpointManager::is_resumable(&disk, &task_id).await.is_err());
|
||||
// Recovery requires replacing/cleaning the snapshot, then removing the
|
||||
// marker; ordinary selector retries are intentionally not an unlock path.
|
||||
manager.cleanup().await.expect("clean blocked checkpoint");
|
||||
assert!(!CheckpointManager::is_blocked(&disk, &task_id).await);
|
||||
temp_dir.close().expect("remove empty blocked marker test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resumable_selector_skips_healthy_tasks_with_blocked_markers() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let tasks = [
|
||||
(ResumeUtils::generate_task_id(), EcstoreDiskBytes::new()),
|
||||
(ResumeUtils::generate_task_id(), EcstoreDiskBytes::from_static(b"blocked")),
|
||||
];
|
||||
for (task_id, marker) in &tasks {
|
||||
ResumeManager::new(
|
||||
disk.clone(),
|
||||
task_id.clone(),
|
||||
"erasure_set".to_string(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec!["bucket".to_string()],
|
||||
)
|
||||
.await
|
||||
.expect("create healthy resume state");
|
||||
CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create healthy checkpoint");
|
||||
let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}");
|
||||
let checkpoint_bytes = disk
|
||||
.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
|
||||
.await
|
||||
.expect("read healthy checkpoint before blocking");
|
||||
let marker_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &marker_path, marker.clone())
|
||||
.await
|
||||
.expect("write blocked marker");
|
||||
|
||||
assert!(ResumeUtils::get_resumable_tasks(&disk).await.is_err());
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path)
|
||||
.await
|
||||
.expect("read healthy checkpoint after blocking"),
|
||||
checkpoint_bytes
|
||||
);
|
||||
}
|
||||
temp_dir.close().expect("remove blocked selector test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_checkpoint_manager_cannot_overwrite_newer_progress() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let first = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("create first checkpoint manager");
|
||||
let second = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("load second checkpoint manager");
|
||||
|
||||
second
|
||||
.update_position(4, 20)
|
||||
.await
|
||||
.expect("persist newer checkpoint progress");
|
||||
let error = first
|
||||
.update_position(1, 3)
|
||||
.await
|
||||
.expect_err("stale checkpoint manager must not overwrite newer progress");
|
||||
assert!(error.to_string().contains("newer progress"));
|
||||
|
||||
let persisted = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("load newer checkpoint progress")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!(persisted.current_bucket_index, 4);
|
||||
assert_eq!(persisted.current_object_index, 20);
|
||||
temp_dir.close().expect("remove stale manager test directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resumable_selector_isolates_future_and_corrupt_checkpoints() {
|
||||
let (temp_dir, disk) = schema_test_disk().await;
|
||||
let future_task = ResumeUtils::generate_task_id();
|
||||
let corrupt_task = ResumeUtils::generate_task_id();
|
||||
for task_id in [&future_task, &corrupt_task] {
|
||||
ResumeManager::new(
|
||||
disk.clone(),
|
||||
task_id.to_string(),
|
||||
"erasure_set".to_string(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec!["bucket".to_string()],
|
||||
)
|
||||
.await
|
||||
.expect("create resumable state fixture");
|
||||
}
|
||||
|
||||
let future_path = format!("{BUCKET_META_PREFIX}/{future_task}_{RESUME_CHECKPOINT_FILE}");
|
||||
let mut future = ResumeCheckpoint::new(future_task.clone());
|
||||
future.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1;
|
||||
let future_bytes = serde_json::to_vec(&future).expect("serialize future checkpoint fixture");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &future_path, future_bytes.clone().into())
|
||||
.await
|
||||
.expect("write future checkpoint fixture");
|
||||
let corrupt_path = format!("{BUCKET_META_PREFIX}/{corrupt_task}_{RESUME_CHECKPOINT_FILE}");
|
||||
let corrupt_bytes = b"{truncated";
|
||||
disk.write_all(RUSTFS_META_BUCKET, &corrupt_path, corrupt_bytes.as_slice().into())
|
||||
.await
|
||||
.expect("write corrupt checkpoint fixture");
|
||||
|
||||
assert!(CheckpointManager::is_resumable(&disk, &future_task).await.is_err());
|
||||
assert!(CheckpointManager::is_resumable(&disk, &corrupt_task).await.is_err());
|
||||
assert!(ResumeUtils::get_resumable_tasks(&disk).await.is_err());
|
||||
for (task_id, path, bytes) in [
|
||||
(&future_task, future_path, future_bytes),
|
||||
(&corrupt_task, corrupt_path, corrupt_bytes.to_vec()),
|
||||
] {
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, &path)
|
||||
.await
|
||||
.expect("read isolated checkpoint bytes"),
|
||||
bytes
|
||||
);
|
||||
let blocked_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}");
|
||||
assert!(
|
||||
!disk
|
||||
.read_all(RUSTFS_META_BUCKET, &blocked_path)
|
||||
.await
|
||||
.expect("read checkpoint blocked marker")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
temp_dir.close().expect("remove selector isolation test directory");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_persist_throttle_batches_until_threshold() {
|
||||
let mut throttle = PersistThrottle::new();
|
||||
|
||||
@@ -21,7 +21,7 @@ use uuid::Uuid;
|
||||
use super::super::{BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
|
||||
use super::replacement::{ReplacementPhase, ReplacementRecoveryRecord};
|
||||
use super::{
|
||||
EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE,
|
||||
CheckpointManager, EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE,
|
||||
REPLACEMENT_INTENT_FILE, RESUME_STATE_FILE, ResumeManager, ResumeStateFile, is_replacement_intent, path_to_str,
|
||||
replacement_recovery_corruption_for_state_load, replacement_recovery_dir, validate_resume_task_id,
|
||||
};
|
||||
@@ -67,6 +67,7 @@ impl ResumeUtils {
|
||||
// Extract task ID from filename: {task_id}_ahm_resume_state.json
|
||||
if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}"))
|
||||
&& validate_resume_task_id(task_id).is_ok()
|
||||
&& CheckpointManager::is_resumable(disk, task_id).await?
|
||||
{
|
||||
task_ids.push(task_id.to_string());
|
||||
}
|
||||
|
||||
@@ -35,7 +35,6 @@ use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, Pool
|
||||
|
||||
const META_BUCKET: &str = ".rustfs.sys";
|
||||
const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin";
|
||||
const SCOPED_JOURNAL_REL: &str = "buckets/.heal/mrf/journal-scoped.bin";
|
||||
|
||||
async fn heal_env() -> (Vec<std::path::PathBuf>, Arc<dyn HealStorageAPI>) {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
@@ -80,18 +79,14 @@ fn journal_record(kind: u8, bucket: &str, object: &str, version: Option<[u8; 16]
|
||||
body
|
||||
}
|
||||
|
||||
fn write_journal_path_to_disks(disk_paths: &[std::path::PathBuf], relative_path: &str, data: &[u8]) {
|
||||
fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
|
||||
for path in disk_paths {
|
||||
let journal = path.join(META_BUCKET).join(relative_path);
|
||||
let journal = path.join(META_BUCKET).join(JOURNAL_REL);
|
||||
std::fs::create_dir_all(journal.parent().expect("journal parent")).expect("create journal dir");
|
||||
std::fs::write(&journal, data).expect("write journal fixture");
|
||||
}
|
||||
}
|
||||
|
||||
fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
|
||||
write_journal_path_to_disks(disk_paths, JOURNAL_REL, data);
|
||||
}
|
||||
|
||||
async fn wait_until<F, Fut>(deadline: Duration, mut probe: F) -> bool
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
@@ -192,73 +187,8 @@ async fn journal_replay_arms_intents_and_deletes_the_file() {
|
||||
.all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()),
|
||||
"the journal file must be removed after a successful replay"
|
||||
);
|
||||
assert!(
|
||||
disk_paths
|
||||
.iter()
|
||||
.all(|path| !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()),
|
||||
"the authoritative journal file must also be removed after replay"
|
||||
);
|
||||
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
assert_eq!(snapshot.queued_by_priority.urgent, 1, "the decode-failure record must replay as Urgent");
|
||||
assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal");
|
||||
}
|
||||
|
||||
/// A canonical snapshot and its compatibility mirror may differ after a
|
||||
/// partial flush. Replay must choose the complete canonical epoch instead of
|
||||
/// combining records that never coexisted in memory.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
let mut endpoints: Vec<Endpoint> = disk_paths
|
||||
.iter()
|
||||
.map(|p| Endpoint::try_from(p.to_string_lossy().as_ref()).expect("endpoint from disk path"))
|
||||
.collect();
|
||||
for (i, endpoint) in endpoints.iter_mut().enumerate() {
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(i);
|
||||
}
|
||||
let pool = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: endpoints.len(),
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "mrf-authoritative-test".to_string(),
|
||||
platform: String::new(),
|
||||
};
|
||||
init_local_disks(EndpointServerPools::from(vec![pool]))
|
||||
.await
|
||||
.expect("local disks should register");
|
||||
|
||||
let authoritative = journal_record(1, "authoritative-bucket", "authoritative-object", None, 0);
|
||||
let legacy = journal_record(1, "legacy-bucket", "legacy-object", None, 0);
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &authoritative);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &legacy);
|
||||
|
||||
let manager = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&manager).await;
|
||||
assert_eq!(replayed, 1, "only the authoritative snapshot epoch may replay");
|
||||
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
assert_eq!(snapshot.queued_by_source.mrf, 1);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"replay cleanup must remove both journal paths"
|
||||
);
|
||||
|
||||
// A scoped-only snapshot is valid during a rollout where no legacy
|
||||
// compatibility mirror was written. Missing legacy files must not leave
|
||||
// the runtime in a permanent cleanup-retry state.
|
||||
let scoped_only = journal_record(1, "scoped-only-bucket", "scoped-only-object", None, 0);
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &scoped_only);
|
||||
assert_eq!(mrf_queue::replay_journal_once(&manager).await, 1);
|
||||
assert!(disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -1375,14 +1375,13 @@ impl FolderScanner {
|
||||
// Single-flight (backlog#1894 axis A) — the
|
||||
// recording mode and its guarantees are pinned by
|
||||
// corrupt_metadata_recording below.
|
||||
let mrf_result = rustfs_common::mrf_channel::try_send_mrf_intent_typed(
|
||||
let mrf_accepted = rustfs_common::mrf_channel::try_send_mrf_intent(
|
||||
rustfs_common::mrf_channel::MrfKind::MetadataCorruption,
|
||||
&item.bucket,
|
||||
&object,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
match corrupt_metadata_recording(mrf_result) {
|
||||
match corrupt_metadata_recording(mrf_accepted) {
|
||||
CorruptMetadataRecording::LedgerOnly => {
|
||||
// Recorded as Full (retry-later): admission
|
||||
// for this target happens in the MRF
|
||||
|
||||
@@ -50,12 +50,11 @@ pub(super) enum CorruptMetadataRecording {
|
||||
ImmediateAndLedger,
|
||||
}
|
||||
|
||||
pub(super) fn corrupt_metadata_recording(result: rustfs_common::mrf_channel::MrfIngressResult) -> CorruptMetadataRecording {
|
||||
match result {
|
||||
rustfs_common::mrf_channel::MrfIngressResult::Enqueued | rustfs_common::mrf_channel::MrfIngressResult::Coalesced => {
|
||||
CorruptMetadataRecording::LedgerOnly
|
||||
}
|
||||
rustfs_common::mrf_channel::MrfIngressResult::Dropped(_) => CorruptMetadataRecording::ImmediateAndLedger,
|
||||
pub(super) fn corrupt_metadata_recording(mrf_accepted: bool) -> CorruptMetadataRecording {
|
||||
if mrf_accepted {
|
||||
CorruptMetadataRecording::LedgerOnly
|
||||
} else {
|
||||
CorruptMetadataRecording::ImmediateAndLedger
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -48,20 +48,8 @@ fn scanner_alert_wire_names_match_canonical_event_names() {
|
||||
/// the backstop survives regardless of delivery.
|
||||
#[test]
|
||||
fn corrupt_metadata_recording_maps_delivery_to_backstop() {
|
||||
assert_eq!(
|
||||
corrupt_metadata_recording(rustfs_common::mrf_channel::MrfIngressResult::Enqueued),
|
||||
CorruptMetadataRecording::LedgerOnly
|
||||
);
|
||||
assert_eq!(
|
||||
corrupt_metadata_recording(rustfs_common::mrf_channel::MrfIngressResult::Coalesced),
|
||||
CorruptMetadataRecording::LedgerOnly
|
||||
);
|
||||
assert_eq!(
|
||||
corrupt_metadata_recording(rustfs_common::mrf_channel::MrfIngressResult::Dropped(
|
||||
rustfs_common::mrf_channel::MrfDropReason::Full
|
||||
)),
|
||||
CorruptMetadataRecording::ImmediateAndLedger
|
||||
);
|
||||
assert_eq!(corrupt_metadata_recording(true), CorruptMetadataRecording::LedgerOnly);
|
||||
assert_eq!(corrupt_metadata_recording(false), CorruptMetadataRecording::ImmediateAndLedger);
|
||||
}
|
||||
|
||||
fn cooldown_map_len() -> usize {
|
||||
|
||||
@@ -43,9 +43,6 @@ allow-git = [
|
||||
# RustFS fork carrying presigned expiry and constant-time authentication fixes.
|
||||
# owner: rustfs-maintainers review: 2026-10
|
||||
"https://github.com/rustfs/s3s.git",
|
||||
# MiMalloc fork pinned for hotpath allocation counting support.
|
||||
# owner: houseme review: 2026-10
|
||||
"https://github.com/xonatius/mimalloc_rust.git",
|
||||
]
|
||||
|
||||
[bans]
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
# CI gate matrix
|
||||
|
||||
This file is the source of truth for which validation runs on each event, its
|
||||
configured wall-clock budget, and whether it can block a merge. Test taxonomy,
|
||||
naming, and nextest serialization rules remain in [README.md](README.md); e2e
|
||||
membership and counts remain in
|
||||
[e2e-suite-inventory.md](e2e-suite-inventory.md).
|
||||
|
||||
The distinction between **required** and **report-only** is load-bearing:
|
||||
a failing job blocks a merge only when its exact check name is present in the
|
||||
live `main` ruleset. A workflow name, a `merge_group` trigger, or a red PR check
|
||||
does not make a job required by itself.
|
||||
|
||||
## Required merge checks
|
||||
|
||||
The live `main` ruleset (`6436880`) currently requires exactly these contexts:
|
||||
|
||||
| Required context | Producer | Validation |
|
||||
|---|---|---|
|
||||
| `CLA Check` | `.github/workflows/cla.yml` | Contributor agreement |
|
||||
| `Quick Checks` | `.github/workflows/ci.yml` | Formatting and repository guard scripts |
|
||||
| `Test and Lint` | `.github/workflows/ci.yml` | Clippy, workspace nextest excluding `e2e_test`, doctests, and migration proofs |
|
||||
|
||||
For pull requests limited to the paths excluded by the main CI workflow,
|
||||
`.github/workflows/ci-docs-only.yml` reports `Quick Checks` and
|
||||
`Test and Lint` under the same names. It runs the real quick checks and the
|
||||
planning-document guard; it does not claim that Rust compilation or runtime
|
||||
tests ran. Despite the workflow name, these paths also include selected deploy,
|
||||
workflow, and lock files.
|
||||
|
||||
Verify the live rule rather than trusting this snapshot before changing merge
|
||||
policy:
|
||||
|
||||
```bash
|
||||
gh api repos/rustfs/rustfs/rulesets/6436880 \
|
||||
--jq '.rules[] | select(.type == "required_status_checks") | .parameters'
|
||||
```
|
||||
|
||||
The ruleset currently has `strict_required_status_checks_policy=false`.
|
||||
`Continuous Integration` accepts `merge_group` events and runs `e2e-full` for
|
||||
them, but `End-to-End Tests (full merge gate)` is not currently a required
|
||||
context. Therefore the repository is prepared to test a merge-queue SHA, but
|
||||
the workflow alone does not prove that every merge passed that lane.
|
||||
|
||||
## Pull request and merge matrix
|
||||
|
||||
Budgets below are job `timeout-minutes`, not typical runtimes. “Report-only”
|
||||
means the result is visible and actionable but is not in the live required
|
||||
context list.
|
||||
|
||||
| Event | Validation | Budget | Merge status | Reproduction |
|
||||
|---|---|---:|---|---|
|
||||
| PR, non-doc change | `Quick Checks` | 10 min | Required | `make pre-commit` (broader local umbrella) |
|
||||
| PR, non-doc change | `Test and Lint` | 90 min | Required | `cargo nextest run --profile ci --all --exclude e2e_test` |
|
||||
| PR, non-doc change | `Typos` | 10 min | Report-only | `typos` |
|
||||
| PR, non-doc change | `ILM Integration (serial)` | 90 min | Report-only | Use the exact command in `.github/workflows/ci.yml` |
|
||||
| PR, non-doc change | rio-v2 / swift / sftp test-and-lint variants | 90 min each | Report-only | `cargo nextest run` with the workflow's feature set |
|
||||
| PR, non-doc change | `Build RustFS Debug Binary` | 30 min | Report-only; prerequisite for black-box lanes | `cargo build -p rustfs --bins` |
|
||||
| PR, non-doc change | `io_uring Integration (real)` | 30 min | Report-only | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
|
||||
| PR, non-doc change | `End-to-End Tests` (`e2e-smoke` plus `s3s-e2e`) | 30 min | Report-only | `cargo nextest run --profile e2e-smoke -p e2e_test`; then `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` |
|
||||
| PR, non-doc change | `S3 Implemented Tests` | 60 min | Report-only | Build `rustfs`, then run `scripts/s3-tests/run.sh` with `DEPLOY_MODE=binary`, `TEST_MODE=single`, and `MAXFAIL=0` |
|
||||
| PR, non-doc change | `S3 Lifecycle Behavior Tests` | 30 min | Report-only | Use the accelerated scanner environment in `.github/workflows/ci.yml` with `scripts/s3-tests/run.sh` |
|
||||
| PR touching dependency or workflow inputs | Cargo Deny / Workflow Pin Report / Dependency Review | 20 / 5 / 30 min | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
|
||||
| PR touching architecture rules or architecture docs | `Architecture Migration Rules` | 10 min | Report-only | `scripts/check_architecture_migration_rules.sh` |
|
||||
| PR touching Nix or workspace manifests | `Nix Build & Check` | 60 min | Report-only | `nix flake check` |
|
||||
| PR limited to main-CI-excluded paths | companion `Quick Checks` and `Test and Lint` | 10 min each | Required | `git diff --check`; `make doc-paths-check` when documentation paths changed |
|
||||
| `merge_group` | Standard CI plus `e2e-full` | 55 min for `e2e-full` | Standard required contexts only; `e2e-full` report-only | `cargo nextest run --profile e2e-full -p e2e_test` |
|
||||
| Push to `main` | Standard CI plus `e2e-full` | 55 min for `e2e-full` | Post-merge detection | Same as `merge_group` |
|
||||
| PR touching fuzz inputs or harness paths | Build plus five 60-second fuzz smoke targets | 60 min build; 30 min per target | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` |
|
||||
| PR touching selected ecstore disk/format paths | `Rename Safety` on Windows | 60 min | Report-only | Run the four `cargo test -p rustfs-ecstore --lib <filter>` commands in `windows-filesystem.yml` on Windows |
|
||||
|
||||
The authoritative e2e filters live in `.config/nextest.toml`; extend a profile
|
||||
instead of adding a second ad-hoc selector. Before a profile runs,
|
||||
`scripts/check_test_wiring.py` compares its exact membership to the committed
|
||||
digest so a silent test drop fails closed.
|
||||
|
||||
## Scheduled and manual validation
|
||||
|
||||
Scheduled lanes are independent fault domains. They do not block a pull
|
||||
request, but their workflow-local gate can fail the run and scheduled failures
|
||||
are routed to the shared failure-issue action. The scheduled-validation
|
||||
watchdog and freshness workflow separately detect incomplete runs and missing
|
||||
schedules.
|
||||
|
||||
| Cadence (UTC unless noted) | Workflow / validation | Budget | Verdict and artifacts | Reproduction |
|
||||
|---|---|---:|---|---|
|
||||
| Daily 02:17 | Fuzz: five nightly corpus targets | 60 min build; 60 min per target | Gate; corpus/crash artifacts, scheduled failure alert | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
|
||||
| Daily 03:17 | MinIO interop (EC + SSE read parity) | 40 min | Gate; scheduled failure alert | Dispatch `minio-interop.yml` or follow its pinned Docker fixture steps |
|
||||
| Daily 04:29 | Replication / cluster-fault / protocol e2e | 45 / 90 / 90 min | Three independent gates; JUnit, membership, and server logs | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` |
|
||||
| Daily 06:31 | Warp performance A/B | 180 min | Regression budget gate; A/B summaries and server logs | `bash scripts/run_hotpath_warp_abba.sh --help` |
|
||||
| Daily 00:07 Asia/Shanghai (16:07 UTC previous day) | Nightly GNU build and Vault lanes | 150 / 90 / 60 min | Build, live Vault, and HA failover gates | Use the commands and pinned Vault images in `nightly-gnu.yml` |
|
||||
| Daily 03:23 | Security Audit | 20 / 5 min, plus 30 min on PR dependency review | Cargo Deny and workflow-pin gates; scheduled failure alert | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
|
||||
| Daily 23:47 | Scheduled Validation Freshness | 10 min | Fails when a critical schedule was never created or is stale | Dispatch `scheduled-validation-freshness.yml` |
|
||||
| Sunday 00:11 | Full `Continuous Integration` matrix | Per-job budgets above | Weekly variant coverage, including dormant rio-v2 binary/e2e lanes | Dispatch `ci.yml` |
|
||||
| Sunday 01:13 | Seven-platform build matrix | 150 min per platform | Build/package integrity; scheduled failure alert | Dispatch `build.yml` with an exact platform set |
|
||||
| Sunday 02:19 | Ceph s3-tests full sweep: single and real four-node, four shards each | 180 min per shard | Compatibility gate; report, JUnit, exact node IDs, and server logs | `scripts/s3-tests/run.sh` against an existing single or distributed target |
|
||||
| Sunday 06:41 | Mint | 120 min | **Report-only by design**; per-suite PASS/FAIL/NA and raw `log.json` | Reproduce the pinned Docker sequence in `mint.yml` or dispatch it |
|
||||
| Sunday 07:43 | Workspace line coverage | 120 min | Report-only trend; lcov and JSON retained 90 days | `make coverage` |
|
||||
| Monthly, day 1 06:37 | Runner Hygiene | 15 min | Validates runner ephemerality; scheduled failure alert | Dispatch `runner-hygiene.yml` |
|
||||
|
||||
Manual `workflow_dispatch` exists for the scheduled workflows above. Manual
|
||||
runs are debugging evidence and intentionally do not open scheduled-failure
|
||||
issues. A manual performance run may explicitly allow a known regression; that
|
||||
override must not be treated as an ordinary passing baseline.
|
||||
|
||||
## Release validation
|
||||
|
||||
Release validation is post-merge and tag-driven; it does not substitute for a
|
||||
pull-request gate.
|
||||
|
||||
| Event | Validation | Budget | Result |
|
||||
|---|---|---:|---|
|
||||
| Push to `main` or weekly schedule | `Build and Release` platform matrix | 150 min per platform | Build artifacts for all selected targets; no release publication on a main push |
|
||||
| Valid release or preview tag | `Build and Release` plus asset checks | 150 min per platform | Draft release, checksummed assets, and publish step |
|
||||
| Successful non-preview release-tag build | Docker image build and image scan | 60 min build; 30 min scan | Multi-architecture images plus vulnerability report |
|
||||
| Successful release-tag build | DEB/RPM packaging | 30 min per architecture | Packages and checksum files uploaded to the release |
|
||||
| Successful non-preview release-tag build | Helm template test and package | 30 min build; 30 min publish | Versioned chart and repository index |
|
||||
|
||||
Use an exact preview tag for end-to-end release rehearsal. Manual dispatches
|
||||
are backfill/debug paths and do not prove the automatic `workflow_run` chain.
|
||||
|
||||
## Evidence requirements
|
||||
|
||||
A green check is useful only when it proves the intended behavior ran:
|
||||
|
||||
- Record the exact commit SHA and run URL.
|
||||
- Separate product failure from runner prerequisites, service readiness, and
|
||||
cancellation. Repair the precondition, then rerun the exact workload.
|
||||
- Preserve membership manifests, JUnit, raw compatibility logs, seeds, and
|
||||
server logs where the workflow provides them.
|
||||
- For a bug fix or a new fault checker, provide sensitivity evidence: the old
|
||||
behavior or an intentional mutation must fail the new oracle, and the fixed
|
||||
behavior must pass it.
|
||||
- Never promote a report-only lane to required from one green run. Require at
|
||||
least 14 days and 30 representative pull requests with at least 99% complete
|
||||
execution, then update the ruleset and this table together.
|
||||
|
||||
## Change checklist
|
||||
|
||||
Update this file in the same pull request when any of these change:
|
||||
|
||||
- workflow triggers, job names, timeouts, or nextest profile ownership;
|
||||
- required status contexts or strict/merge-queue policy;
|
||||
- scheduled cadence, alert routing, artifact contract, or local reproduction;
|
||||
- report-only versus gating semantics.
|
||||
|
||||
Do not copy per-module test counts here. Update
|
||||
[e2e-suite-inventory.md](e2e-suite-inventory.md) and its enforced membership
|
||||
digest instead.
|
||||
+2
-2
@@ -336,13 +336,13 @@ opentelemetry = { workspace = true }
|
||||
tracing-opentelemetry = { workspace = true }
|
||||
# Data structures
|
||||
hashbrown = { workspace = true, features = ["serde", "rayon"] }
|
||||
mimalloc = { workspace = true }
|
||||
rustfs-mimalloc = { workspace = true }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
libsystemd.workspace = true
|
||||
|
||||
[target.'cfg(not(target_os = "windows"))'.dependencies]
|
||||
libmimalloc-sys.workspace = true
|
||||
rustfs-mimalloc-sys.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
uuid = { workspace = true, features = ["v4", "v5", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
@@ -369,14 +369,8 @@ pub fn allocator_reclaim_controller_snapshot(ctx: &CancellationToken) -> Allocat
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
#[allow(unsafe_code)]
|
||||
fn collect_allocator_memory(force: bool) -> Result<(), String> {
|
||||
// SAFETY: `mi_collect` is provided by the active global allocator backend
|
||||
// on this target family. It is explicitly intended to reclaim retained
|
||||
// pages/segments and does not require additional invariants from the caller.
|
||||
unsafe {
|
||||
libmimalloc_sys::mi_collect(force);
|
||||
}
|
||||
rustfs_mimalloc::MiMalloc::collect(force);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
+10
-8
@@ -26,22 +26,22 @@ struct MiMallocAllocator;
|
||||
unsafe impl GlobalAlloc for MiMallocAllocator {
|
||||
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
|
||||
// SAFETY: the caller upholds GlobalAlloc's contract for layout.
|
||||
unsafe { mimalloc::MiMalloc.alloc(layout) }
|
||||
unsafe { rustfs_mimalloc::MiMalloc.alloc(layout) }
|
||||
}
|
||||
|
||||
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
|
||||
// SAFETY: the caller upholds GlobalAlloc's contract for layout.
|
||||
unsafe { mimalloc::MiMalloc.alloc_zeroed(layout) }
|
||||
unsafe { rustfs_mimalloc::MiMalloc.alloc_zeroed(layout) }
|
||||
}
|
||||
|
||||
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
|
||||
// SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
|
||||
unsafe { mimalloc::MiMalloc.dealloc(ptr, layout) }
|
||||
unsafe { rustfs_mimalloc::MiMalloc.dealloc(ptr, layout) }
|
||||
}
|
||||
|
||||
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
|
||||
// SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
|
||||
unsafe { mimalloc::MiMalloc.realloc(ptr, layout, new_size) }
|
||||
unsafe { rustfs_mimalloc::MiMalloc.realloc(ptr, layout, new_size) }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ static GLOBAL: hotpath::CountingAllocator<MiMallocAllocator> = hotpath::Counting
|
||||
|
||||
#[cfg(not(all(feature = "hotpath", feature = "hotpath-alloc")))]
|
||||
#[global_allocator]
|
||||
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||
static GLOBAL: rustfs_mimalloc::MiMalloc = rustfs_mimalloc::MiMalloc;
|
||||
|
||||
fn main() {
|
||||
let _hotpath_guard = hotpath::HotpathGuardBuilder::new("main").build();
|
||||
@@ -71,8 +71,9 @@ mod tests {
|
||||
allocation.extend_from_slice(&[7_u8; 64]);
|
||||
|
||||
assert_eq!(allocation.len(), 64);
|
||||
let heap = rustfs_mimalloc::heap::Heap::main();
|
||||
// SAFETY: the live Vec pointer is valid to inspect for heap ownership.
|
||||
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(allocation.as_ptr().cast()) });
|
||||
assert!(unsafe { heap.contains(allocation.as_ptr()) });
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -85,12 +86,13 @@ mod tests {
|
||||
let layout = Layout::from_size_align(32, 8).expect("valid test allocation layout");
|
||||
let grown_layout = Layout::from_size_align(64, 8).expect("valid grown test allocation layout");
|
||||
let allocator = super::MiMallocAllocator;
|
||||
let heap = rustfs_mimalloc::heap::Heap::main();
|
||||
|
||||
// SAFETY: The pointer is checked for null before use and later released
|
||||
// through the same allocator with the corresponding layout.
|
||||
let ptr = unsafe { allocator.alloc_zeroed(layout) };
|
||||
assert!(!ptr.is_null());
|
||||
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(ptr.cast()) });
|
||||
assert!(unsafe { heap.contains(ptr) });
|
||||
assert!(unsafe { std::slice::from_raw_parts(ptr, 32).iter().all(|byte| *byte == 0) });
|
||||
|
||||
// SAFETY: `ptr` was allocated by `allocator` with `layout`; on failure
|
||||
@@ -102,7 +104,7 @@ mod tests {
|
||||
panic!("mimalloc realloc failed in allocator smoke test");
|
||||
}
|
||||
|
||||
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(grown_ptr.cast()) });
|
||||
assert!(unsafe { heap.contains(grown_ptr) });
|
||||
// SAFETY: `grown_ptr` was reallocated by `allocator` and is released
|
||||
// with the matching grown layout.
|
||||
unsafe { allocator.dealloc(grown_ptr, grown_layout) };
|
||||
|
||||
@@ -17,10 +17,7 @@ use rustfs_io_metrics::{
|
||||
record_cpu_usage, record_memory_usage, record_process_memory_split,
|
||||
};
|
||||
use serde::Serialize;
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
use serde_json::Value;
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
use std::ffi::CStr;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::Duration;
|
||||
@@ -231,7 +228,18 @@ fn read_cgroup_memory_snapshot() -> Option<CgroupMemorySnapshot> {
|
||||
read_cgroup_v2().or_else(read_cgroup_v1)
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
|
||||
let json = rustfs_mimalloc::MiMalloc::stats_json();
|
||||
if json.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let observation = parse_mimalloc_stats_json(&json)?;
|
||||
Some(AllocatorMemorySnapshot {
|
||||
backend: crate::allocator_reclaim::allocator_backend(),
|
||||
observation,
|
||||
})
|
||||
}
|
||||
|
||||
fn numeric_json_value(value: &Value) -> Option<u64> {
|
||||
match value {
|
||||
Value::Number(number) => number
|
||||
@@ -242,7 +250,6 @@ fn numeric_json_value(value: &Value) -> Option<u64> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
|
||||
match value {
|
||||
Value::Object(fields) => fields
|
||||
@@ -254,7 +261,6 @@ fn numeric_json_field(value: &Value, field: &str) -> Option<u64> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option<u64> {
|
||||
match value {
|
||||
Value::Object(fields) => {
|
||||
@@ -271,12 +277,10 @@ fn mimalloc_stat_field(value: &Value, metric: &str, field: &str) -> Option<u64>
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn mimalloc_stat_current(value: &Value, metric: &str) -> Option<u64> {
|
||||
mimalloc_stat_field(value, metric, "current")
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn mimalloc_stat_sum(value: &Value, metrics: &[&str], field: &str) -> Option<u64> {
|
||||
metrics
|
||||
.iter()
|
||||
@@ -285,7 +289,6 @@ fn mimalloc_stat_sum(value: &Value, metrics: &[&str], field: &str) -> Option<u64
|
||||
.filter(|value| *value > 0)
|
||||
}
|
||||
|
||||
#[cfg(any(test, not(target_os = "windows")))]
|
||||
fn parse_mimalloc_stats_json(stats_json: &str) -> Option<AllocatorMemoryObservation> {
|
||||
let value = serde_json::from_str::<Value>(stats_json).ok()?;
|
||||
let malloc_metrics = ["malloc_normal", "malloc_huge"];
|
||||
@@ -312,33 +315,6 @@ fn parse_mimalloc_stats_json(stats_json: &str) -> Option<AllocatorMemoryObservat
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
#[allow(unsafe_code)]
|
||||
fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
|
||||
// SAFETY: `mi_stats_get_json` returns a null-terminated JSON buffer owned by
|
||||
// mimalloc when called with a null input buffer. The mimalloc API requires
|
||||
// freeing that buffer with `mi_free`; parsing finishes before the buffer is freed.
|
||||
let observation = unsafe {
|
||||
let stats_ptr = libmimalloc_sys::mi_stats_get_json(0, std::ptr::null_mut());
|
||||
if stats_ptr.is_null() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let observation = CStr::from_ptr(stats_ptr).to_str().ok().and_then(parse_mimalloc_stats_json);
|
||||
libmimalloc_sys::mi_free(stats_ptr.cast());
|
||||
observation?
|
||||
};
|
||||
Some(AllocatorMemorySnapshot {
|
||||
backend: crate::allocator_reclaim::allocator_backend(),
|
||||
observation,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn read_allocator_memory_snapshot() -> Option<AllocatorMemorySnapshot> {
|
||||
None
|
||||
}
|
||||
|
||||
fn configured_memory_observability_interval_secs() -> u64 {
|
||||
rustfs_utils::get_env_u64(ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS, DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS).max(1)
|
||||
}
|
||||
@@ -566,6 +542,13 @@ mod tests {
|
||||
assert_eq!(parse_mimalloc_stats_json(r#"{ "allocator": "unknown" }"#), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_allocator_memory_snapshot_uses_mimalloc_stats_json() {
|
||||
let snapshot = super::read_allocator_memory_snapshot();
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
assert!(snapshot.is_some(), "allocator snapshot should be available on non-Windows");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memory_observability_snapshot_reports_disabled_when_metrics_are_disabled() {
|
||||
let snapshot = build_memory_observability_status_snapshot(false, 15, false);
|
||||
|
||||
Reference in New Issue
Block a user