mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-05 19:55:37 +00:00
feat(heal): stage conservative legacy MRF migration evidence
Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
@@ -33,6 +33,9 @@ use std::collections::HashMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Explicit pending migration; never activates the production writer or GC.
|
||||
pub mod migration;
|
||||
|
||||
// Root-level control files avoid requiring a new directory before the first
|
||||
// atomic commit. They remain inside the storage owner's metadata volume.
|
||||
const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"];
|
||||
@@ -66,6 +69,23 @@ struct Manifest {
|
||||
}
|
||||
|
||||
impl Manifest {
|
||||
fn encode(owner: Uuid, sequence: u64, payload: &[u8]) -> Result<Vec<u8>, SnapshotError> {
|
||||
let mut bytes = Vec::with_capacity(MANIFEST_LEN);
|
||||
bytes.extend_from_slice(MAGIC);
|
||||
bytes.push(VERSION);
|
||||
bytes.extend_from_slice(owner.as_bytes());
|
||||
bytes.extend_from_slice(&sequence.to_le_bytes());
|
||||
bytes.extend_from_slice(
|
||||
&u64::try_from(payload.len())
|
||||
.map_err(|_| SnapshotError::TooLarge)?
|
||||
.to_le_bytes(),
|
||||
);
|
||||
bytes.extend_from_slice(&Sha256::digest(payload));
|
||||
bytes.extend_from_slice(&Sha256::digest(&bytes));
|
||||
Self::decode(&bytes, payload.len())?;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn decode(bytes: &[u8], limit: usize) -> Result<Self, SnapshotError> {
|
||||
if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC {
|
||||
return Err(SnapshotError::Corrupt);
|
||||
|
||||
@@ -0,0 +1,900 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Pending, owner-local legacy import. These paths are deliberately invisible
|
||||
//! to the active snapshot reader and legacy consumer. Source revalidation is
|
||||
//! not a writer freeze: no result here grants activation or reclamation rights.
|
||||
//! All source bytes and inherited responsibilities survive admission/replay.
|
||||
|
||||
use super::{MANIFEST_LEN, Manifest, SnapshotError, read_bounded};
|
||||
use crate::heal::RUSTFS_META_BUCKET;
|
||||
use crate::heal::mrf_queue::{MRF_JOURNAL_PATH, MRF_SCOPED_JOURNAL_PATH, decode_one};
|
||||
use crate::heal::storage_api::owner::{
|
||||
EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskStore,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeSet;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
const PAYLOADS: [&str; 2] = [".heal-mrf-import-pending.0.bin", ".heal-mrf-import-pending.1.bin"];
|
||||
const COMMITS: [&str; 2] = [".heal-mrf-import-commit.0.bin", ".heal-mrf-import-commit.1.bin"];
|
||||
const MAX_DISKS: usize = 64;
|
||||
const CLAIM: &str = ".heal-mrf-import-claim.bin";
|
||||
|
||||
/// Limits apply to the complete encoded candidate and all distinct raw records,
|
||||
/// including inherited sources. Exceeding either preserves previous anchors.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct MigrationLimits {
|
||||
pub max_bytes: usize,
|
||||
pub max_records: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum MigrationError {
|
||||
#[error(transparent)]
|
||||
Snapshot(#[from] SnapshotError),
|
||||
#[error("MRF migration requires every configured, formatted local disk")]
|
||||
CoverageGap,
|
||||
#[error("MRF migration source changed; all recovery anchors are retained")]
|
||||
SourceChanged,
|
||||
#[error("MRF migration candidate is invalid")]
|
||||
Invalid,
|
||||
#[error("MRF migration has no responsibility evidence")]
|
||||
Empty,
|
||||
#[error("MRF migration conditional publication conflicted")]
|
||||
Conflict,
|
||||
#[error("MRF migration staging is claimed; interrupted claims require separately fenced recovery")]
|
||||
Claimed,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
enum LegacyPath {
|
||||
Scoped,
|
||||
Mirror,
|
||||
}
|
||||
|
||||
impl LegacyPath {
|
||||
fn path(self) -> &'static str {
|
||||
match self {
|
||||
Self::Scoped => MRF_SCOPED_JOURNAL_PATH,
|
||||
Self::Mirror => MRF_JOURNAL_PATH,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Source {
|
||||
disk_id: Uuid,
|
||||
path: LegacyPath,
|
||||
digest: [u8; 32],
|
||||
// None proves an observed absent path, distinct from a present empty file.
|
||||
bytes: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PendingMigration {
|
||||
version: u8,
|
||||
sources: Vec<Source>,
|
||||
inherited: Vec<Source>,
|
||||
}
|
||||
|
||||
impl PendingMigration {
|
||||
/// Raw, complete records. No scope/version normalization, attempts pruning,
|
||||
/// incarnation inference or task-success interpretation is performed.
|
||||
pub fn replay_records(&self, limits: MigrationLimits) -> Result<Vec<Vec<u8>>, MigrationError> {
|
||||
self.validate_limits(limits)?;
|
||||
let mut records = BTreeSet::new();
|
||||
for source in self.sources.iter().chain(&self.inherited) {
|
||||
if source.disk_id.is_nil() {
|
||||
return Err(MigrationError::Invalid);
|
||||
}
|
||||
let bytes = source.bytes.as_deref().unwrap_or_default();
|
||||
if <[u8; 32]>::from(Sha256::digest(bytes)) != source.digest {
|
||||
return Err(MigrationError::Invalid);
|
||||
}
|
||||
let mut offset = 0;
|
||||
while offset < bytes.len() {
|
||||
let (_, consumed) = decode_one(&bytes[offset..]).ok_or(MigrationError::Invalid)?;
|
||||
let end = offset.checked_add(consumed).ok_or(MigrationError::Invalid)?;
|
||||
records.insert(bytes[offset..end].to_vec());
|
||||
if records.len() > limits.max_records {
|
||||
return Err(SnapshotError::TooLarge.into());
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
}
|
||||
if records.is_empty() {
|
||||
return Err(MigrationError::Empty);
|
||||
}
|
||||
Ok(records.into_iter().collect())
|
||||
}
|
||||
|
||||
fn validate_limits(&self, limits: MigrationLimits) -> Result<(), MigrationError> {
|
||||
if self.version != 1 {
|
||||
return Err(SnapshotError::Unsupported.into());
|
||||
}
|
||||
if self.sources.is_empty() || self.sources.len() > MAX_DISKS * 2 {
|
||||
return Err(MigrationError::Invalid);
|
||||
}
|
||||
// Bound the raw input before allocating the JSON representation.
|
||||
let total = self
|
||||
.sources
|
||||
.iter()
|
||||
.chain(&self.inherited)
|
||||
.try_fold(0usize, |total, source| {
|
||||
total
|
||||
.checked_add(source.bytes.as_ref().map_or(0, Vec::len))
|
||||
.and_then(|n| n.checked_add(128))
|
||||
})
|
||||
.ok_or(SnapshotError::TooLarge)?;
|
||||
if total > limits.max_bytes {
|
||||
return Err(SnapshotError::TooLarge.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn encode(&self, limits: MigrationLimits) -> Result<Vec<u8>, MigrationError> {
|
||||
self.replay_records(limits)?;
|
||||
let bytes = serde_json::to_vec(self).map_err(|_| MigrationError::Invalid)?;
|
||||
if bytes.len() > limits.max_bytes {
|
||||
return Err(SnapshotError::TooLarge.into());
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
async fn revalidate(&self, disks: &[EcstoreDiskStore], limits: MigrationLimits) -> Result<(), MigrationError> {
|
||||
let current = capture(disks, limits).await?;
|
||||
if current.sources != self.sources {
|
||||
return Err(MigrationError::SourceChanged);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn configured_disks(disks: &[Option<EcstoreDiskStore>]) -> Result<Vec<EcstoreDiskStore>, MigrationError> {
|
||||
if disks.is_empty() || disks.len() > MAX_DISKS {
|
||||
return Err(MigrationError::CoverageGap);
|
||||
}
|
||||
let mut ordered = std::collections::BTreeMap::new();
|
||||
for disk in disks {
|
||||
let disk = disk.as_ref().ok_or(MigrationError::CoverageGap)?;
|
||||
if !EcstoreDiskAPI::is_local(disk.as_ref()) {
|
||||
return Err(MigrationError::CoverageGap);
|
||||
}
|
||||
let id = EcstoreDiskAPI::get_disk_id(disk.as_ref())
|
||||
.await
|
||||
.map_err(SnapshotError::Disk)?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(MigrationError::CoverageGap)?;
|
||||
if ordered.insert(id, disk.clone()).is_some() {
|
||||
return Err(MigrationError::CoverageGap);
|
||||
}
|
||||
}
|
||||
Ok(ordered.into_values().collect())
|
||||
}
|
||||
|
||||
async fn capture(disks: &[EcstoreDiskStore], limits: MigrationLimits) -> Result<PendingMigration, MigrationError> {
|
||||
let mut sources = Vec::with_capacity(disks.len() * 2);
|
||||
let mut identities = BTreeSet::new();
|
||||
let mut remaining = limits.max_bytes;
|
||||
for disk in disks {
|
||||
let id = EcstoreDiskAPI::get_disk_id(disk.as_ref())
|
||||
.await
|
||||
.map_err(SnapshotError::Disk)?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(MigrationError::CoverageGap)?;
|
||||
if !identities.insert(id) {
|
||||
return Err(MigrationError::CoverageGap);
|
||||
}
|
||||
for path in [LegacyPath::Scoped, LegacyPath::Mirror] {
|
||||
// A missing metadata volume is a coverage gap, not an absent journal.
|
||||
let bytes = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path.path()).await {
|
||||
Ok(reader) => {
|
||||
let maximum = u64::try_from(remaining.checked_add(1).ok_or(SnapshotError::TooLarge)?)
|
||||
.map_err(|_| SnapshotError::TooLarge)?;
|
||||
let mut bytes = Vec::new();
|
||||
reader
|
||||
.take(maximum)
|
||||
.read_to_end(&mut bytes)
|
||||
.await
|
||||
.map_err(SnapshotError::Read)?;
|
||||
if bytes.len() > remaining {
|
||||
return Err(SnapshotError::TooLarge.into());
|
||||
}
|
||||
Some(bytes)
|
||||
}
|
||||
Err(EcstoreDiskError::FileNotFound) => None,
|
||||
Err(error) => return Err(SnapshotError::Disk(error).into()),
|
||||
};
|
||||
remaining = remaining
|
||||
.checked_sub(bytes.as_ref().map_or(0, Vec::len))
|
||||
.ok_or(SnapshotError::TooLarge)?;
|
||||
let digest = Sha256::digest(bytes.as_deref().unwrap_or_default()).into();
|
||||
sources.push(Source {
|
||||
disk_id: id,
|
||||
path,
|
||||
digest,
|
||||
bytes,
|
||||
});
|
||||
}
|
||||
}
|
||||
sources.sort_by_key(|source| (source.disk_id, matches!(source.path, LegacyPath::Mirror)));
|
||||
let candidate = PendingMigration {
|
||||
version: 1,
|
||||
sources,
|
||||
inherited: Vec::new(),
|
||||
};
|
||||
candidate.encode(limits)?;
|
||||
Ok(candidate)
|
||||
}
|
||||
|
||||
/// Inspect every configured source without merging v1 mirrors into a claimed
|
||||
/// latest snapshot. A complete subset/superset is retained as pending evidence.
|
||||
pub async fn capture_legacy_migration(
|
||||
disks: &[Option<EcstoreDiskStore>],
|
||||
limits: MigrationLimits,
|
||||
) -> Result<PendingMigration, MigrationError> {
|
||||
capture(&configured_disks(disks).await?, limits).await
|
||||
}
|
||||
|
||||
struct Staged {
|
||||
manifest: Manifest,
|
||||
candidate: PendingMigration,
|
||||
slot: usize,
|
||||
}
|
||||
|
||||
async fn read_staged(disks: &[EcstoreDiskStore], limits: MigrationLimits) -> Result<Option<Staged>, MigrationError> {
|
||||
let mut selected: Option<Staged> = None;
|
||||
let mut damage = None;
|
||||
let mut identities = std::collections::BTreeMap::new();
|
||||
for disk in disks {
|
||||
for slot in 0..2 {
|
||||
let result = async {
|
||||
let Some(bytes) = read_bounded(disk, COMMITS[slot], MANIFEST_LEN).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let manifest = Manifest::decode(&bytes, limits.max_bytes)?;
|
||||
let payload = read_bounded(disk, PAYLOADS[slot], manifest.payload_len)
|
||||
.await?
|
||||
.ok_or(MigrationError::Invalid)?;
|
||||
if payload.len() != manifest.payload_len || <[u8; 32]>::from(Sha256::digest(&payload)) != manifest.payload_digest
|
||||
{
|
||||
return Err(MigrationError::Invalid);
|
||||
}
|
||||
let candidate: PendingMigration = serde_json::from_slice(&payload).map_err(|_| MigrationError::Invalid)?;
|
||||
if candidate.encode(limits)? != payload {
|
||||
return Err(MigrationError::Invalid);
|
||||
}
|
||||
Ok(Some(Staged {
|
||||
manifest,
|
||||
candidate,
|
||||
slot,
|
||||
}))
|
||||
}
|
||||
.await;
|
||||
match result {
|
||||
Ok(Some(next)) => {
|
||||
let identity = (next.manifest.owner, next.manifest.payload_digest);
|
||||
if identities
|
||||
.insert(next.manifest.sequence, identity)
|
||||
.is_some_and(|old| old != identity)
|
||||
{
|
||||
return Err(MigrationError::Conflict);
|
||||
}
|
||||
if selected
|
||||
.as_ref()
|
||||
.is_none_or(|old| old.manifest.sequence < next.manifest.sequence)
|
||||
{
|
||||
selected = Some(next);
|
||||
}
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(MigrationError::Snapshot(SnapshotError::Unsupported)) => return Err(SnapshotError::Unsupported.into()),
|
||||
Err(error) => damage = Some(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
match (selected, damage) {
|
||||
(Some(staged), _) => Ok(Some(staged)),
|
||||
(None, Some(error)) => Err(error),
|
||||
(None, None) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
async fn install(disk: &EcstoreDiskStore, path: &str, bytes: &[u8], limit: usize) -> Result<(), MigrationError> {
|
||||
let expected = read_bounded(disk, path, limit).await?.map(EcstoreDiskBytes::from);
|
||||
let result = EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
expected,
|
||||
Some(EcstoreDiskBytes::copy_from_slice(bytes)),
|
||||
)
|
||||
.await
|
||||
.map_err(SnapshotError::Disk)?;
|
||||
if result != EcstoreConditionalFileUpdate::Updated {
|
||||
return Err(MigrationError::Conflict);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist an explicitly requested pending import using the storage owner's CAS
|
||||
/// (and its configured metadata durability). This does not freeze legacy ingress
|
||||
/// or grant a durable-acceptance/GC receipt. Activation requires W14/W21 evidence.
|
||||
pub async fn stage_legacy_migration(
|
||||
disks: &[Option<EcstoreDiskStore>],
|
||||
candidate: &PendingMigration,
|
||||
owner: Uuid,
|
||||
limits: MigrationLimits,
|
||||
) -> Result<u64, MigrationError> {
|
||||
let disks = configured_disks(disks).await?;
|
||||
// Claim every configured disk in identity order. A crash/cancellation leaves
|
||||
// claims intact; a new process cannot guess that the old writer is fenced.
|
||||
let claim = EcstoreDiskBytes::copy_from_slice(Uuid::new_v4().as_bytes());
|
||||
let mut claimed = Vec::new();
|
||||
let result = async {
|
||||
for disk in &disks {
|
||||
match EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, CLAIM, None, Some(claim.clone()))
|
||||
.await
|
||||
.map_err(SnapshotError::Disk)?
|
||||
{
|
||||
EcstoreConditionalFileUpdate::Updated => claimed.push(disk.clone()),
|
||||
_ => return Err(MigrationError::Claimed),
|
||||
}
|
||||
}
|
||||
stage_claimed(&disks, candidate, owner, limits).await
|
||||
}
|
||||
.await;
|
||||
for disk in claimed {
|
||||
let released =
|
||||
EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, CLAIM, Some(claim.clone()), None)
|
||||
.await
|
||||
.map_err(SnapshotError::Disk)?;
|
||||
if released != EcstoreConditionalFileUpdate::Updated {
|
||||
return Err(MigrationError::Claimed);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn stage_claimed(
|
||||
disks: &[EcstoreDiskStore],
|
||||
candidate: &PendingMigration,
|
||||
owner: Uuid,
|
||||
limits: MigrationLimits,
|
||||
) -> Result<u64, MigrationError> {
|
||||
candidate.revalidate(&disks, limits).await?;
|
||||
let previous = read_staged(&disks, limits).await?;
|
||||
if previous.as_ref().is_some_and(|old| old.manifest.owner != owner) {
|
||||
return Err(MigrationError::Conflict);
|
||||
}
|
||||
let mut candidate = candidate.clone();
|
||||
if let Some(old) = &previous {
|
||||
for source in old.candidate.sources.iter().chain(&old.candidate.inherited) {
|
||||
if !candidate.sources.contains(source) && !candidate.inherited.contains(source) {
|
||||
candidate.inherited.push(source.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
let payload = candidate.encode(limits)?;
|
||||
let digest: [u8; 32] = Sha256::digest(&payload).into();
|
||||
if let Some(old) = &previous
|
||||
&& old.manifest.payload_digest == digest
|
||||
{
|
||||
candidate.revalidate(&disks, limits).await?;
|
||||
return Ok(old.manifest.sequence);
|
||||
}
|
||||
let sequence = previous.as_ref().map_or(1, |old| old.manifest.sequence.saturating_add(1));
|
||||
let slot = previous.as_ref().map_or(0, |old| 1 - old.slot);
|
||||
let manifest = Manifest::encode(owner, sequence, &payload)?;
|
||||
for disk in disks {
|
||||
install(disk, PAYLOADS[slot], &payload, limits.max_bytes).await?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
tests::interrupt_at(owner, tests::Boundary::AfterPayload).await?;
|
||||
candidate.revalidate(&disks, limits).await?;
|
||||
#[cfg(test)]
|
||||
tests::interrupt_at(owner, tests::Boundary::BeforeManifest).await?;
|
||||
for disk in disks {
|
||||
install(disk, COMMITS[slot], &manifest, MANIFEST_LEN).await?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
tests::interrupt_at(owner, tests::Boundary::AfterManifest).await?;
|
||||
let recovered = read_staged(&disks, limits).await?.ok_or(MigrationError::Invalid)?;
|
||||
if recovered.manifest.sequence != sequence || recovered.manifest.payload_digest != digest {
|
||||
return Err(MigrationError::Conflict);
|
||||
}
|
||||
recovered.candidate.revalidate(&disks, limits).await?;
|
||||
#[cfg(test)]
|
||||
tests::interrupt_at(owner, tests::Boundary::AfterReadback).await?;
|
||||
Ok(sequence)
|
||||
}
|
||||
|
||||
/// Reload pending obligations after process restart. Manager admission never
|
||||
/// removes them. Missing/changed sources block migration, preserving all files.
|
||||
pub async fn recover_pending_migration(
|
||||
disks: &[Option<EcstoreDiskStore>],
|
||||
limits: MigrationLimits,
|
||||
) -> Result<Option<PendingMigration>, MigrationError> {
|
||||
let disks = configured_disks(disks).await?;
|
||||
let Some(staged) = read_staged(&disks, limits).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
staged.candidate.revalidate(&disks, limits).await?;
|
||||
Ok(Some(staged.candidate))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::heal::mrf_queue::encode_intent;
|
||||
use crate::heal::{DiskOption, Endpoint, new_disk};
|
||||
use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfScope};
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
|
||||
const LIMITS: MigrationLimits = MigrationLimits {
|
||||
max_bytes: 64 * 1024,
|
||||
max_records: 100,
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum Boundary {
|
||||
AfterPayload,
|
||||
BeforeManifest,
|
||||
AfterManifest,
|
||||
AfterReadback,
|
||||
}
|
||||
|
||||
static INTERRUPTIONS: std::sync::LazyLock<std::sync::Mutex<std::collections::BTreeMap<Uuid, Boundary>>> =
|
||||
std::sync::LazyLock::new(Default::default);
|
||||
|
||||
static SOURCE_CHANGES: std::sync::LazyLock<std::sync::Mutex<std::collections::BTreeMap<Uuid, (EcstoreDiskStore, Vec<u8>)>>> =
|
||||
std::sync::LazyLock::new(Default::default);
|
||||
|
||||
pub(super) async fn interrupt_at(owner: Uuid, boundary: Boundary) -> Result<(), MigrationError> {
|
||||
if boundary == Boundary::AfterPayload {
|
||||
let change = SOURCE_CHANGES.lock().expect("source fault map").remove(&owner);
|
||||
if let Some((disk, bytes)) = change {
|
||||
source(&disk, &bytes).await;
|
||||
}
|
||||
}
|
||||
let mut interruptions = INTERRUPTIONS.lock().expect("fault map");
|
||||
if interruptions.get(&owner) == Some(&boundary) {
|
||||
interruptions.remove(&owner);
|
||||
return Err(SnapshotError::Read(std::io::Error::new(
|
||||
std::io::ErrorKind::Interrupted,
|
||||
"injected migration boundary failure",
|
||||
))
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn record(object: &str, kind: MrfKind, scope: Option<MrfScope>) -> Vec<u8> {
|
||||
let mut bytes = Vec::new();
|
||||
assert!(encode_intent(
|
||||
&MrfIntent {
|
||||
bucket: Arc::from("bucket"),
|
||||
object: Arc::from(object),
|
||||
version_id: None,
|
||||
kind,
|
||||
scope,
|
||||
lease: None,
|
||||
enqueued_at_ms: 1,
|
||||
attempts: 255,
|
||||
},
|
||||
&mut bytes
|
||||
));
|
||||
bytes
|
||||
}
|
||||
|
||||
async fn disk(root: &TempDir, name: &str) -> EcstoreDiskStore {
|
||||
let path = root.path().join(name);
|
||||
std::fs::create_dir_all(&path).expect("create test disk");
|
||||
let mut endpoint = Endpoint::try_from(path.to_string_lossy().as_ref()).expect("disk endpoint");
|
||||
endpoint.set_idx = 0;
|
||||
endpoint.disk_idx = 0;
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("open disk");
|
||||
let created = EcstoreDiskAPI::make_volume(disk.as_ref(), RUSTFS_META_BUCKET).await;
|
||||
assert!(
|
||||
matches!(created, Ok(()) | Err(EcstoreDiskError::VolumeExists)),
|
||||
"metadata volume: {created:?}"
|
||||
);
|
||||
let id = Uuid::new_v4();
|
||||
let format = serde_json::json!({
|
||||
"version": "1", "format": "xl-single", "id": Uuid::new_v4(),
|
||||
"xl": { "version": "3", "this": id, "sets": [[id]], "distributionAlgo": "SIPMOD+PARITY" }
|
||||
});
|
||||
EcstoreDiskAPI::write_all(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
"format.json",
|
||||
serde_json::to_vec(&format).expect("format").into(),
|
||||
)
|
||||
.await
|
||||
.expect("format disk");
|
||||
assert_eq!(EcstoreDiskAPI::get_disk_id(disk.as_ref()).await.expect("formatted identity"), Some(id));
|
||||
disk
|
||||
}
|
||||
|
||||
async fn source(disk: &EcstoreDiskStore, bytes: &[u8]) {
|
||||
// Legacy source paths predate the root-level COW control files.
|
||||
EcstoreDiskAPI::write_all(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
MRF_SCOPED_JOURNAL_PATH,
|
||||
EcstoreDiskBytes::copy_from_slice(bytes),
|
||||
)
|
||||
.await
|
||||
.expect("write legacy source");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_subset_union_preserves_sources_across_reopen() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let first = disk(&root, "first").await;
|
||||
let second = disk(&root, "second").await;
|
||||
let a = record("a", MrfKind::PartialWrite, None);
|
||||
let b = record(
|
||||
"b",
|
||||
MrfKind::DecodeFailure,
|
||||
Some(MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 1,
|
||||
}),
|
||||
);
|
||||
source(&first, &a).await;
|
||||
source(&second, &[a.clone(), b.clone()].concat()).await;
|
||||
let disks = [Some(first.clone()), Some(second.clone())];
|
||||
let candidate = capture_legacy_migration(&disks, LIMITS)
|
||||
.await
|
||||
.expect("capture both complete sources");
|
||||
let reverse = capture_legacy_migration(&[Some(second.clone()), Some(first.clone())], LIMITS)
|
||||
.await
|
||||
.expect("reverse disk order");
|
||||
assert_eq!(
|
||||
candidate.encode(LIMITS).expect("candidate"),
|
||||
reverse.encode(LIMITS).expect("reversed candidate")
|
||||
);
|
||||
assert_eq!(candidate.replay_records(LIMITS).expect("raw records").len(), 2);
|
||||
let owner = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
stage_legacy_migration(&disks, &candidate, owner, LIMITS)
|
||||
.await
|
||||
.expect("stage candidate"),
|
||||
1
|
||||
);
|
||||
drop(candidate);
|
||||
let recovered = recover_pending_migration(&disks, LIMITS)
|
||||
.await
|
||||
.expect("restart read")
|
||||
.expect("pending import");
|
||||
// Reading or discarding a replay batch must not consume its stored anchor.
|
||||
let mut admitted = recovered.replay_records(LIMITS).expect("replay batch");
|
||||
admitted.pop();
|
||||
drop(admitted);
|
||||
assert_eq!(
|
||||
recover_pending_migration(&disks, LIMITS)
|
||||
.await
|
||||
.expect("second restart")
|
||||
.expect("anchor")
|
||||
.replay_records(LIMITS)
|
||||
.expect("records")
|
||||
.len(),
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
stage_legacy_migration(&disks, &recovered, owner, LIMITS)
|
||||
.await
|
||||
.expect("idempotent retry"),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
|
||||
.await
|
||||
.expect("first source"),
|
||||
a
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
|
||||
.await
|
||||
.expect("second source"),
|
||||
[a, b].concat()
|
||||
);
|
||||
assert!(
|
||||
super::super::read_committed(&[first, second], LIMITS.max_bytes)
|
||||
.await
|
||||
.expect("active reader")
|
||||
.is_none(),
|
||||
"pending import must not activate the production snapshot"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_source_change_and_capacity_failure_keep_old_commit() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let disk = disk(&root, "disk").await;
|
||||
let disks = [Some(disk.clone())];
|
||||
let owner = Uuid::new_v4();
|
||||
let a = record("a", MrfKind::PartialWrite, None);
|
||||
let b = record("b", MrfKind::PartialWrite, None);
|
||||
source(&disk, &a).await;
|
||||
let original = capture_legacy_migration(&disks, LIMITS).await.expect("initial capture");
|
||||
stage_legacy_migration(&disks, &original, owner, LIMITS)
|
||||
.await
|
||||
.expect("initial commit");
|
||||
let before = EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, COMMITS[0])
|
||||
.await
|
||||
.expect("old commit");
|
||||
source(&disk, &b).await;
|
||||
assert!(matches!(
|
||||
stage_legacy_migration(&disks, &original, owner, LIMITS).await,
|
||||
Err(MigrationError::SourceChanged)
|
||||
));
|
||||
let next = capture_legacy_migration(&disks, LIMITS).await.expect("new source capture");
|
||||
assert!(matches!(
|
||||
stage_legacy_migration(
|
||||
&disks,
|
||||
&next,
|
||||
owner,
|
||||
MigrationLimits {
|
||||
max_records: 1,
|
||||
..LIMITS
|
||||
}
|
||||
)
|
||||
.await,
|
||||
Err(MigrationError::Snapshot(SnapshotError::TooLarge))
|
||||
));
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, COMMITS[0])
|
||||
.await
|
||||
.expect("retained commit"),
|
||||
before
|
||||
);
|
||||
assert_eq!(
|
||||
stage_legacy_migration(&disks, &next, owner, LIMITS)
|
||||
.await
|
||||
.expect("COW successor"),
|
||||
2
|
||||
);
|
||||
let records = recover_pending_migration(&disks, LIMITS)
|
||||
.await
|
||||
.expect("successor restart")
|
||||
.expect("successor")
|
||||
.replay_records(LIMITS)
|
||||
.expect("responsibilities");
|
||||
assert!(
|
||||
records.contains(&a) && records.contains(&b),
|
||||
"successor must inherit old source responsibilities"
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, COMMITS[0])
|
||||
.await
|
||||
.expect("previous slot retained"),
|
||||
before
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_torn_inactive_payload_and_manifest_keep_previous_anchor() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let disk = disk(&root, "disk").await;
|
||||
let disks = [Some(disk.clone())];
|
||||
source(&disk, &record("a", MrfKind::PartialWrite, None)).await;
|
||||
let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture");
|
||||
stage_legacy_migration(&disks, &candidate, Uuid::new_v4(), LIMITS)
|
||||
.await
|
||||
.expect("initial commit");
|
||||
for (path, bytes) in [
|
||||
(PAYLOADS[1], b"torn payload".as_slice()),
|
||||
(COMMITS[1], b"torn manifest".as_slice()),
|
||||
] {
|
||||
install(&disk, path, bytes, LIMITS.max_bytes)
|
||||
.await
|
||||
.expect("interrupted inactive write");
|
||||
assert_eq!(
|
||||
recover_pending_migration(&disks, LIMITS)
|
||||
.await
|
||||
.expect("recover previous")
|
||||
.expect("anchor")
|
||||
.replay_records(LIMITS)
|
||||
.expect("retained responsibility")
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_missing_corrupt_and_empty_sources_fail_closed() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let disk = disk(&root, "disk").await;
|
||||
assert!(matches!(
|
||||
capture_legacy_migration(&[Some(disk.clone()), None], LIMITS).await,
|
||||
Err(MigrationError::CoverageGap)
|
||||
));
|
||||
assert!(matches!(
|
||||
capture_legacy_migration(&[Some(disk.clone())], LIMITS).await,
|
||||
Err(MigrationError::Empty)
|
||||
));
|
||||
source(&disk, b"corrupt").await;
|
||||
assert!(matches!(
|
||||
capture_legacy_migration(&[Some(disk)], LIMITS).await,
|
||||
Err(MigrationError::Invalid)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_commit_boundaries_and_lost_response_recover_idempotently() {
|
||||
for boundary in [
|
||||
Boundary::AfterPayload,
|
||||
Boundary::BeforeManifest,
|
||||
Boundary::AfterManifest,
|
||||
Boundary::AfterReadback,
|
||||
] {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let disk = disk(&root, "disk").await;
|
||||
let disks = [Some(disk.clone())];
|
||||
let bytes = record("a", MrfKind::PartialWrite, None);
|
||||
source(&disk, &bytes).await;
|
||||
let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture");
|
||||
let owner = Uuid::new_v4();
|
||||
INTERRUPTIONS.lock().expect("fault map").insert(owner, boundary);
|
||||
assert!(stage_legacy_migration(&disks, &candidate, owner, LIMITS).await.is_err());
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
|
||||
.await
|
||||
.expect("source survives interruption"),
|
||||
bytes
|
||||
);
|
||||
let recovery = recover_pending_migration(&disks, LIMITS).await.expect("restart inspection");
|
||||
assert_eq!(recovery.is_some(), matches!(boundary, Boundary::AfterManifest | Boundary::AfterReadback));
|
||||
assert_eq!(
|
||||
stage_legacy_migration(&disks, &candidate, owner, LIMITS)
|
||||
.await
|
||||
.expect("retry interrupted stage"),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
recover_pending_migration(&disks, LIMITS)
|
||||
.await
|
||||
.expect("restart after retry")
|
||||
.expect("anchor")
|
||||
.replay_records(LIMITS)
|
||||
.expect("records"),
|
||||
vec![bytes]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_interrupted_claim_does_not_authorize_takeover() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let disk = disk(&root, "disk").await;
|
||||
let disks = [Some(disk.clone())];
|
||||
source(&disk, &record("a", MrfKind::PartialWrite, None)).await;
|
||||
let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture");
|
||||
let owner = Uuid::new_v4();
|
||||
stage_legacy_migration(&disks, &candidate, owner, LIMITS)
|
||||
.await
|
||||
.expect("committed anchor");
|
||||
install(&disk, CLAIM, b"interrupted writer", 64)
|
||||
.await
|
||||
.expect("interrupted claim");
|
||||
assert!(matches!(
|
||||
stage_legacy_migration(&disks, &candidate, owner, LIMITS).await,
|
||||
Err(MigrationError::Claimed)
|
||||
));
|
||||
assert_eq!(
|
||||
recover_pending_migration(&disks, LIMITS)
|
||||
.await
|
||||
.expect("recovery remains read-only")
|
||||
.expect("anchor")
|
||||
.replay_records(LIMITS)
|
||||
.expect("record")
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, CLAIM)
|
||||
.await
|
||||
.expect("claim retained"),
|
||||
b"interrupted writer".as_slice()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_source_change_after_payload_prevents_manifest_publication() {
|
||||
let root = TempDir::new().expect("test directory");
|
||||
let disk = disk(&root, "disk").await;
|
||||
let disks = [Some(disk.clone())];
|
||||
source(&disk, &record("a", MrfKind::PartialWrite, None)).await;
|
||||
let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture");
|
||||
let owner = Uuid::new_v4();
|
||||
let changed = record("b", MrfKind::PartialWrite, None);
|
||||
SOURCE_CHANGES
|
||||
.lock()
|
||||
.expect("source fault map")
|
||||
.insert(owner, (disk.clone(), changed.clone()));
|
||||
assert!(matches!(
|
||||
stage_legacy_migration(&disks, &candidate, owner, LIMITS).await,
|
||||
Err(MigrationError::SourceChanged)
|
||||
));
|
||||
assert!(
|
||||
recover_pending_migration(&disks, LIMITS)
|
||||
.await
|
||||
.expect("no committed import")
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
|
||||
.await
|
||||
.expect("changed source survives"),
|
||||
changed
|
||||
);
|
||||
assert!(
|
||||
read_bounded(&disk, PAYLOADS[0], LIMITS.max_bytes)
|
||||
.await
|
||||
.expect("candidate retained")
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_raw_identity_preserves_kind_scope_and_nil_version() {
|
||||
let id = Uuid::new_v4();
|
||||
let mut variants = Vec::new();
|
||||
for kind in [MrfKind::PartialWrite, MrfKind::DecodeFailure] {
|
||||
for scope in [
|
||||
None,
|
||||
Some(MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
}),
|
||||
Some(MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 1,
|
||||
}),
|
||||
] {
|
||||
variants.push(record("same", kind, scope));
|
||||
}
|
||||
}
|
||||
let mut nil = record("same", MrfKind::PartialWrite, None);
|
||||
nil[12] = 1;
|
||||
nil.splice(13..13, [0; 16]);
|
||||
let end = nil.len() - 4;
|
||||
let mut crc = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
crc.update(&nil[..end]);
|
||||
nil[end..].copy_from_slice(&u32::try_from(crc.finalize()).expect("CRC").to_le_bytes());
|
||||
variants.push(nil);
|
||||
let bytes = variants.concat();
|
||||
let candidate = PendingMigration {
|
||||
version: 1,
|
||||
sources: vec![Source {
|
||||
disk_id: id,
|
||||
path: LegacyPath::Scoped,
|
||||
digest: Sha256::digest(&bytes).into(),
|
||||
bytes: Some(bytes),
|
||||
}],
|
||||
inherited: Vec::new(),
|
||||
};
|
||||
let records = candidate.replay_records(LIMITS).expect("raw identities");
|
||||
assert_eq!(records.len(), variants.len());
|
||||
for variant in variants {
|
||||
assert!(records.contains(&variant));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Pending MRF Migration
|
||||
|
||||
`heal::mrf_queue::snapshot::migration` exposes explicit capture, staging, and readback of pending legacy responsibility evidence. Nothing invokes it from the production MRF consumer. It does not enable the committed-snapshot writer, freeze legacy ingress, acknowledge durable admission, or authorize source garbage collection.
|
||||
|
||||
The caller supplies every configured local disk slot, including missing slots. Missing/unformatted/duplicate disks, unavailable metadata volumes, invalid records, empty evidence, and aggregate byte/record overflow fail closed. Both legacy paths retain their original bytes, disk identity, absent-versus-empty state, and SHA-256 digest. Complete subset/superset replicas become conservative pending evidence, never a claimed newest legacy snapshot. Raw record replay preserves kind, scope and nil/absent version encodings; unknown incarnation stays unknown.
|
||||
|
||||
Staging writes only `.heal-mrf-import-pending.{0,1}.bin`, `.heal-mrf-import-commit.{0,1}.bin`, and `.heal-mrf-import-claim.bin` under the metadata volume. It reuses the committed reader's manifest codec and the storage owner's conditional-file operation, including the configured metadata durability policy. A candidate is written before sources are revalidated, its manifest is then committed, and committed bytes plus source coverage are read back before success. Success is pending staging evidence, not a power-loss or cluster-quorum durability receipt.
|
||||
|
||||
A successor inherits prior source bytes even if a replay consumer has already read or admitted their records. Size limits include inherited evidence. There is no completion-based pruning. A changed source blocks recovery of that pending generation; an explicit new capture can stage a successor that retains both the previous and current responsibilities. The inactive slot is replaced while the preceding committed slot remains intact.
|
||||
|
||||
All participating disks are claimed in disk-identity order through CAS. Normal completion conditionally releases only the current invocation's claim. Cancellation, process death, or an ambiguous claim/release I/O failure may leave a claim behind. Read-only recovery remains available, but further staging is blocked until a separate storage-fenced recovery procedure is implemented. Process liveness and the legacy ingress lease do not authorize taking over or deleting a claim.
|
||||
|
||||
Run the focused fixtures with a nonzero test count:
|
||||
|
||||
```sh
|
||||
cargo test -p rustfs-heal --lib heal::mrf_queue::snapshot::migration::tests
|
||||
```
|
||||
|
||||
Fixtures use real local disks and the production CAS/readback path. They cover disk-order independence, retained raw identities, source change after candidate write, capacity rejection, interrupted commit boundaries, lost responses, torn inactive slots, and refusal to take over an interrupted claim. Boundary injection and same-process reopen are not process-kill, directory-fsync failure, disk-full, mixed-version, or power-loss tests. The actual manager Full/Accepted-to-crash pipeline remains outside this staged API.
|
||||
|
||||
Rollback leaves all pending and legacy artifacts untouched. Activation still requires legacy-writer coordination, bounded recoverable ingress, exact object-disposition/successor receipts, and the W14/W21 process-crash and compatibility gates. The production legacy replay deletion window remains unresolved by this staging-only phase.
|
||||
Reference in New Issue
Block a user