diff --git a/crates/heal/src/heal/resume.rs b/crates/heal/src/heal/resume.rs index d8553b4f8..291132c12 100644 --- a/crates/heal/src/heal/resume.rs +++ b/crates/heal/src/heal/resume.rs @@ -16,7 +16,6 @@ use crate::{Error, Result}; use serde::{Deserialize, Serialize}; #[cfg(test)] use std::collections::HashMap; -use std::collections::HashSet; use std::path::{Component, Path}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -25,14 +24,24 @@ use tracing::{debug, warn}; use uuid::Uuid; use super::{ - BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET, - storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskBytes}, + BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET, storage_api::owner::EcstoreDiskBytes, }; +mod checkpoint; +mod replacement; +mod utils; + +pub use checkpoint::{CheckpointManager, ResumeCheckpoint}; +pub(crate) use replacement::replacement_target_identities_match; +use replacement::replacement_targets_match_identities; +pub use replacement::{ + ReplacementPhase, ReplacementRecoveryRecord, ReplacementRecoveryState, ReplacementTargetIdentity, compose_key, +}; +pub use utils::ResumeUtils; + const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_RESUME: &str = "resume"; const EVENT_HEAL_RESUME_STATE: &str = "heal_resume_state"; -const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state"; /// resume state file constants const RESUME_STATE_FILE: &str = "ahm_resume_state.json"; @@ -48,244 +57,12 @@ const REPLACEMENT_INTENT_SEAL_FILE: &str = "ahm_replacement_intent_seal"; const LEGACY_REPLACEMENT_RECOVERY_MARKER_FILE: &str = "ahm_replacement_recovery.json"; const REPLACEMENT_RECOVERY_CONFLICT_PREFIX: &str = "replacement recovery conflict:"; const REPLACEMENT_RECOVERY_CORRUPTION_PREFIX: &str = "replacement recovery corruption:"; -const CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA: u32 = 1; /// Current on-disk schema version for `ResumeState`. Snapshots written by an /// older schema (which tracked latest-only object names and a positional /// cursor) are incompatible with the per-version resume cursor, so they are /// discarded on load and the scan restarts from the beginning. const CURRENT_RESUME_SCHEMA: u32 = 5; -/// Current on-disk schema version for `ResumeCheckpoint`. Same rationale as -/// `CURRENT_RESUME_SCHEMA`: pre-per-version dedup identities are not comparable -/// to the new `compose_key` identities, so a stale checkpoint is discarded. -const CURRENT_CHECKPOINT_SCHEMA: u32 = 5; - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ReplacementPhase { - #[default] - None, - Intent, - Rebuilding, - Verified, - CleanupPending, - Abandoned, -} - -/// Target-specific state for a durable automatic replacement generation. -/// -/// This is deliberately separate from the legacy background-heal status -/// contract. Consumers must treat [`Self::Unknown`] as non-definitive. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ReplacementRecoveryState { - WaitingForReplacement, - Running, - Incomplete, - Unrecoverable, - CleanupPending, - Completed, - Unknown, -} - -/// Read-only status derived from one durable replacement generation. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ReplacementRecoveryRecord { - pub task_id: String, - pub state: ReplacementRecoveryState, - pub generation: Option, - pub set_disk_id: Option, - pub target_slots: Vec, - pub reason: Option, - pub verified_at: Option, -} - -impl ReplacementRecoveryRecord { - fn from_state(state: ResumeState) -> Option { - if !is_replacement_intent(&state) { - return None; - } - - let invariant_holds = state.replacement_generation.as_deref() == Some(state.task_id.as_str()) - && replacement_targets_match_identities(&state.replacement_targets, &state.replacement_target_identities); - if !invariant_holds { - return Some(Self::unknown( - state.task_id, - "durable replacement state violates its generation or target identity binding", - )); - } - - let (state_kind, reason) = if !state.completed && state.retry_count >= state.max_retries { - ( - ReplacementRecoveryState::Unrecoverable, - Some("replacement retry budget exhausted".to_string()), - ) - } else if let Some(reason) = state.error_message.clone() { - (ReplacementRecoveryState::Incomplete, Some(reason)) - } else { - match state.replacement_phase { - ReplacementPhase::Intent => (ReplacementRecoveryState::WaitingForReplacement, None), - ReplacementPhase::Rebuilding => (ReplacementRecoveryState::Running, None), - ReplacementPhase::Verified | ReplacementPhase::CleanupPending => (ReplacementRecoveryState::CleanupPending, None), - ReplacementPhase::Abandoned => ( - ReplacementRecoveryState::Unrecoverable, - Some("replacement generation was abandoned".to_string()), - ), - ReplacementPhase::None => (ReplacementRecoveryState::Unknown, Some("replacement phase is missing".to_string())), - } - }; - - Some(Self { - task_id: state.task_id, - state: state_kind, - generation: state.replacement_generation, - set_disk_id: Some(state.set_disk_id), - target_slots: state.replacement_targets, - reason, - verified_at: None, - }) - } - - fn from_completion_proof(proof: &ReplacementCompletionProof) -> Self { - Self { - task_id: proof.task_id.clone(), - state: ReplacementRecoveryState::Completed, - generation: Some(proof.replacement_generation.clone()), - set_disk_id: Some(proof.set_disk_id.clone()), - target_slots: proof.replacement_targets.clone(), - reason: None, - verified_at: Some(proof.verified_at), - } - } - - fn unknown(task_id: String, reason: &str) -> Self { - Self { - task_id, - state: ReplacementRecoveryState::Unknown, - generation: None, - set_disk_id: None, - target_slots: Vec::new(), - reason: Some(reason.to_string()), - verified_at: None, - } - } -} - -fn replacement_targets_match_identities(targets: &[String], identities: &[ReplacementTargetIdentity]) -> bool { - !targets.is_empty() - && targets.len() == identities.len() - && targets.iter().collect::>().len() == targets.len() - && identities.iter().map(|identity| &identity.endpoint).eq(targets.iter()) -} - -/// Stable evidence for the mounted replacement instance that owns a repair -/// generation. Endpoint text alone is not sufficient because a later disk can -/// be mounted at the same configured path. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ReplacementTargetIdentity { - pub endpoint: String, - pub canonical_path: String, - pub physical_device_ids: Vec, - pub filesystem_identity: String, -} - -/// Durable terminal evidence for one automatic replacement generation. This -/// lives on the healthy non-target anchor rather than in the resumable state, -/// because resume cleanup must not erase proof that the generation completed. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct ReplacementCompletionProof { - pub schema_version: u32, - pub task_id: String, - pub replacement_generation: String, - pub set_disk_id: String, - pub replacement_targets: Vec, - pub replacement_target_identities: Vec, - pub verified_at: u64, -} - -impl ReplacementCompletionProof { - fn from_state(state: &ResumeState, verified_at: u64) -> Result { - let replacement_generation = state - .replacement_generation - .clone() - .ok_or_else(|| Error::TaskExecutionFailed { - message: format!("Replacement completion has no generation for task {}", state.task_id), - })?; - if replacement_generation != state.task_id - || state.replacement_targets.is_empty() - || state - .replacement_target_identities - .iter() - .map(|identity| &identity.endpoint) - .collect::>() - != state.replacement_targets.iter().collect::>() - { - return Err(Error::TaskExecutionFailed { - message: format!("Replacement completion identity does not match task {}", state.task_id), - }); - } - - Ok(Self { - schema_version: CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA, - task_id: state.task_id.clone(), - replacement_generation, - set_disk_id: state.set_disk_id.clone(), - replacement_targets: state.replacement_targets.clone(), - replacement_target_identities: state.replacement_target_identities.clone(), - verified_at, - }) - } - - fn matches_state(&self, state: &ResumeState) -> bool { - self.schema_version == CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA - && self.task_id == state.task_id - && state.replacement_generation.as_deref() == Some(self.replacement_generation.as_str()) - && self.set_disk_id == state.set_disk_id - && self.replacement_targets == state.replacement_targets - && self.replacement_target_identities == state.replacement_target_identities - } - - fn validate(&self, expected_task_id: &str) -> Result<()> { - if self.schema_version != CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA { - return Err(Error::TaskExecutionFailed { - message: format!("Replacement completion proof schema {} is unsupported", self.schema_version), - }); - } - validate_resume_task_id(expected_task_id)?; - if self.task_id != expected_task_id - || self.replacement_generation != self.task_id - || self.set_disk_id.is_empty() - || self.verified_at == 0 - || !replacement_targets_match_identities(&self.replacement_targets, &self.replacement_target_identities) - { - return Err(Error::TaskExecutionFailed { - message: format!("Replacement completion proof does not match task {expected_task_id}"), - }); - } - Ok(()) - } -} - -pub(crate) fn replacement_target_identities_match( - expected: &[ReplacementTargetIdentity], - actual: &[ReplacementTargetIdentity], -) -> bool { - let mut expected = expected.to_vec(); - let mut actual = actual.to_vec(); - expected.sort_by(|left, right| left.endpoint.cmp(&right.endpoint)); - actual.sort_by(|left, right| left.endpoint.cmp(&right.endpoint)); - expected == actual -} - -/// Build the canonical, provably-injective dedup identity for an object -/// version. Length-prefixing the object key makes the encoding injective: no -/// two distinct `(object, version_id)` pairs can collide, even for adversarial -/// keys containing `:` or embedded null bytes. This is the single source of -/// truth for per-version dedup across the heal loop and the checkpoint sets. -pub fn compose_key(object: &str, version_id: Option<&str>) -> String { - format!("{}:{}{}", object.len(), object, version_id.unwrap_or("")) -} /// Persistence throttle for per-object bookkeeping: flush after this many /// buffered mutations or once the interval elapses, whichever comes first. @@ -866,295 +643,6 @@ impl ResumeManager { Ok(manager) } - /// Seal a durably published intent before the caller may format a target. - /// A torn intent without this seal is known to have failed before its - /// creator returned and can be atomically recreated on retry. - async fn ensure_replacement_intent_seal(&self) -> Result<()> { - let task_id = self.state.read().await.task_id.clone(); - validate_resume_task_id(&task_id)?; - let path = replacement_intent_seal_path(&task_id); - let path = path_to_str(&path)?; - match self.disk.read_all(RUSTFS_META_BUCKET, path).await { - Ok(_) => return Ok(()), - Err(DiskError::FileNotFound) => {} - Err(error) => { - return Err(Error::TaskExecutionFailed { - message: format!("Failed to read replacement intent seal: {error}"), - }); - } - } - self.disk - .write_all(RUSTFS_META_BUCKET, path, b"sealed".as_slice().into()) - .await - .map_err(|error| Error::TaskExecutionFailed { - message: format!("Failed to save replacement intent seal: {error}"), - }) - } - - pub async fn mark_replacement_rebuilding( - &self, - mut replacement_target_identities: Vec, - ) -> Result<()> { - replacement_target_identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint)); - replacement_target_identities.dedup_by(|left, right| left.endpoint == right.endpoint); - let mut state = self.state.write().await; - if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) { - return Err(Error::TaskExecutionFailed { - message: format!("Replacement intent is not active for task {}", state.task_id), - }); - } - if replacement_target_identities - .iter() - .map(|identity| &identity.endpoint) - .collect::>() - != state.replacement_targets.iter().collect::>() - { - return Err(Error::TaskExecutionFailed { - message: format!("Replacement identities do not match targets for task {}", state.task_id), - }); - } - if !replacement_target_identities_match(&state.replacement_target_identities, &replacement_target_identities) { - return Err(Error::TaskExecutionFailed { - message: format!("Replacement target changed after format for task {}", state.task_id), - }); - } - state.replacement_phase = ReplacementPhase::Rebuilding; - state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); - drop(state); - self.save_state_strict().await - } - - /// Persist survivor-anchor completion proof before transitioning this - /// resumable state to `Verified`. If proof persistence fails, this state - /// stays rebuildable and the caller must retain the healing marker. - pub async fn mark_replacement_completed_and_verified(&self) -> Result<()> { - let state = self.state.read().await.clone(); - if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) { - return Err(Error::TaskExecutionFailed { - message: format!("Replacement verification is not active for task {}", state.task_id), - }); - } - let proof = self.write_replacement_completion_proof(&state, None).await?; - - let mut state = self.state.write().await; - if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) { - return Err(Error::TaskExecutionFailed { - message: format!("Replacement verification changed for task {}", state.task_id), - }); - } - state.mark_completed(); - state.replacement_phase = ReplacementPhase::Verified; - state.last_update = proof.verified_at; - drop(state); - self.save_state_strict().await - } - - /// Verify or backfill the terminal proof before marker removal or resume - /// cleanup. This supports restart recovery from a `Verified` state written - /// by a prior binary that did not yet have a separate proof record. - pub(crate) async fn ensure_replacement_completion_proof(&self) -> Result { - let state = self.state.read().await.clone(); - if !state.completed || !matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) { - return Err(Error::TaskExecutionFailed { - message: format!("Replacement completion is not verified for task {}", state.task_id), - }); - } - self.write_replacement_completion_proof(&state, Some(state.last_update)).await - } - - /// Record that the healing markers have been removed, so a later retry can - /// safely delete the remaining resume artifacts without touching markers. - pub async fn mark_replacement_cleanup_pending(&self) -> Result<()> { - let mut state = self.state.write().await; - if !state.completed || !matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) { - return Err(Error::TaskExecutionFailed { - message: format!("Replacement cleanup is not ready for task {}", state.task_id), - }); - } - state.replacement_phase = ReplacementPhase::CleanupPending; - state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); - drop(state); - self.save_state_strict().await - } - - /// Load the durable terminal proof from the healthy survivor anchor. - pub(crate) async fn load_replacement_completion_proof(disk: DiskStore, task_id: &str) -> Result { - Self::replacement_completion_proof_if_present(disk, task_id) - .await? - .ok_or_else(|| Error::TaskExecutionFailed { - message: format!("Failed to read replacement completion proof: proof is missing for task {task_id}"), - }) - } - - async fn replacement_completion_proof_if_present( - disk: DiskStore, - task_id: &str, - ) -> Result> { - validate_resume_task_id(task_id)?; - let mut proofs = Vec::new(); - for path in [ - replacement_completion_proof_path(task_id), - legacy_replacement_completion_proof_path(task_id), - ] { - let path_str = path_to_str(&path)?; - let bytes = match disk.read_all(RUSTFS_META_BUCKET, path_str).await { - Ok(bytes) => bytes, - Err(DiskError::FileNotFound) => continue, - Err(error) => { - return Err(Error::TaskExecutionFailed { - message: format!("Failed to read replacement completion proof: {error}"), - }); - } - }; - let proof: ReplacementCompletionProof = - serde_json::from_slice(&bytes).map_err(|error| Error::TaskExecutionFailed { - message: format!("Failed to deserialize replacement completion proof: {error}"), - })?; - proof.validate(task_id)?; - proofs.push(proof); - } - - match proofs.as_slice() { - [] => Ok(None), - [proof] => Ok(Some(proof.clone())), - [proof, legacy_proof] if proof == legacy_proof => Ok(Some(proof.clone())), - _ => Err(replacement_recovery_conflict(format!( - "Replacement completion proof conflicts with legacy proof for task {task_id}" - ))), - } - } - - /// Reconcile the proof-first publication order after a crash. A matching - /// proof is durable evidence that rebuilding finished, so it must win over - /// an older active state before a retry may format the target again. - async fn reconcile_replacement_completion_proof(&self) -> Result<()> { - let task_id = self.state.read().await.task_id.clone(); - let Some(proof) = Self::replacement_completion_proof_if_present(self.disk.clone(), &task_id).await? else { - return Ok(()); - }; - - let mut state = self.state.write().await; - if !proof.matches_state(&state) { - return Err(Error::TaskExecutionFailed { - message: format!("Replacement completion proof does not match active intent for task {}", state.task_id), - }); - } - if state.completed && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) { - return Ok(()); - } - if state.completed || !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) { - return Err(replacement_recovery_conflict(format!( - "Replacement completion proof conflicts with state for task {}", - state.task_id - ))); - } - - state.mark_completed(); - state.replacement_phase = ReplacementPhase::Verified; - state.last_update = proof.verified_at; - drop(state); - self.save_state_strict().await - } - - async fn migrate_legacy_replacement_completion_proof(disk: &DiskStore, task_id: &str) -> Result { - validate_resume_task_id(task_id)?; - let legacy_path = legacy_replacement_completion_proof_path(task_id); - let legacy_path_str = path_to_str(&legacy_path)?; - let legacy_bytes = match disk.read_all(RUSTFS_META_BUCKET, legacy_path_str).await { - Ok(bytes) => bytes, - Err(DiskError::FileNotFound) => return Ok(false), - Err(error) => { - return Err(Error::TaskExecutionFailed { - message: format!("Failed to read legacy replacement completion proof: {error}"), - }); - } - }; - let legacy_proof: ReplacementCompletionProof = serde_json::from_slice(&legacy_bytes).map_err(|error| { - replacement_recovery_corruption(format!("Failed to deserialize legacy replacement completion proof: {error}")) - })?; - legacy_proof - .validate(task_id) - .map_err(|error| replacement_recovery_corruption(format!("Invalid legacy replacement completion proof: {error}")))?; - - ensure_replacement_recovery_dir(disk) - .await - .map_err(|error| Error::TaskExecutionFailed { - message: format!("Failed to create replacement recovery directory: {error}"), - })?; - let path = replacement_completion_proof_path(task_id); - let path_str = path_to_str(&path)?; - for _ in 0..2 { - match disk.read_all(RUSTFS_META_BUCKET, path_str).await { - Ok(bytes) => { - let proof: ReplacementCompletionProof = - serde_json::from_slice(&bytes).map_err(|error| Error::TaskExecutionFailed { - message: format!("Failed to deserialize replacement completion proof: {error}"), - })?; - proof.validate(task_id).map_err(|error| { - replacement_recovery_corruption(format!("Invalid replacement completion proof: {error}")) - })?; - if proof != legacy_proof { - return Err(replacement_recovery_conflict(format!( - "Replacement completion proof conflicts with legacy proof for task {task_id}" - ))); - } - delete_resume_file(disk, &legacy_path).await?; - return Ok(true); - } - Err(DiskError::FileNotFound) => {} - Err(error) => { - return Err(Error::TaskExecutionFailed { - message: format!("Failed to read replacement completion proof: {error}"), - }); - } - } - - match super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file( - disk.as_ref(), - RUSTFS_META_BUCKET, - path_str, - None, - Some(legacy_bytes.clone()), - ) - .await - { - Ok(EcstoreConditionalFileUpdate::Updated) => { - delete_resume_file(disk, &legacy_path).await?; - return Ok(true); - } - Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue, - Err(error) => { - return Err(Error::TaskExecutionFailed { - message: format!("Failed to migrate replacement completion proof: {error}"), - }); - } - } - } - - Err(Error::TaskExecutionFailed { - message: format!("Replacement completion proof changed while migrating task {task_id}"), - }) - } - - pub async fn abandon_replacement_intent(&self) -> Result<()> { - let mut state = self.state.write().await; - if matches!(state.replacement_phase, ReplacementPhase::Abandoned) { - return Ok(()); - } - state.replacement_phase = ReplacementPhase::Abandoned; - state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); - drop(state); - self.save_state_strict().await - } - - pub async fn set_replacement_targets(&self, replacement_targets: Vec) -> Result<()> { - { - let mut state = self.state.write().await; - state.replacement_targets = replacement_targets; - } - self.save_state().await - } - /// Load an ordinary resume state from disk. Replacement intents have a /// separate namespace so they cannot be mistaken for ordinary work by an /// older binary. @@ -1474,116 +962,6 @@ impl ResumeManager { self.save_state_with_unformatted_policy(false).await } - async fn publish_new_replacement_intent(&self, expected: Option) -> Result<()> { - let state = self.state.read().await.clone(); - validate_resume_task_id(&state.task_id)?; - let state_data = EcstoreDiskBytes::from(serde_json::to_vec(&state).map_err(|error| Error::TaskExecutionFailed { - message: format!("Failed to serialize resume state: {error}"), - })?); - let path = self.state_file.path(&state.task_id); - let path = path_to_str(&path)?; - - ensure_replacement_recovery_dir(&self.disk) - .await - .map_err(|error| Error::TaskExecutionFailed { - message: format!("Failed to create replacement recovery directory: {error}"), - })?; - match super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file( - self.disk.as_ref(), - RUSTFS_META_BUCKET, - path, - expected, - Some(state_data), - ) - .await - { - Ok(EcstoreConditionalFileUpdate::Updated) => Ok(()), - Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => { - Err(Error::TaskExecutionFailed { - message: format!("Replacement intent changed before publication for task {}", state.task_id), - }) - } - Err(error) => Err(Error::TaskExecutionFailed { - message: format!("Failed to save resume state: {error}"), - }), - } - } - - async fn write_replacement_completion_proof( - &self, - state: &ResumeState, - verified_at: Option, - ) -> Result { - ensure_replacement_recovery_dir(&self.disk) - .await - .map_err(|error| Error::TaskExecutionFailed { - message: format!("Failed to create replacement recovery directory: {error}"), - })?; - let path = replacement_completion_proof_path(&state.task_id); - let path_str = path_to_str(&path)?; - let proof = ReplacementCompletionProof::from_state( - state, - verified_at.unwrap_or_else(|| SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs()), - )?; - let proof_data = EcstoreDiskBytes::from(serde_json::to_vec(&proof).map_err(|e| Error::TaskExecutionFailed { - message: format!("Failed to serialize replacement completion proof: {e}"), - })?); - if let Some(error) = injected_replacement_proof_write_error(path_str) { - return Err(Error::TaskExecutionFailed { - message: format!("Failed to save replacement completion proof: {error}"), - }); - } - - // Publish through the disk CAS primitive: `write_all` can expose a - // partially written proof to a crash/restart reader. If a prior - // version left torn bytes behind, replace exactly the observed bytes; - // a concurrently published valid proof is never overwritten. - for _ in 0..2 { - let expected = match self.disk.read_all(RUSTFS_META_BUCKET, path_str).await { - Ok(existing) => match serde_json::from_slice::(&existing) { - Ok(existing_proof) => { - existing_proof.validate(&state.task_id)?; - if existing_proof.matches_state(state) { - return Ok(existing_proof); - } - return Err(Error::TaskExecutionFailed { - message: format!("Replacement completion proof does not match task {}", state.task_id), - }); - } - Err(_) => Some(existing), - }, - Err(DiskError::FileNotFound) => None, - Err(error) => { - return Err(Error::TaskExecutionFailed { - message: format!("Failed to read replacement completion proof: {error}"), - }); - } - }; - - match super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file( - self.disk.as_ref(), - RUSTFS_META_BUCKET, - path_str, - expected, - Some(proof_data.clone()), - ) - .await - { - Ok(EcstoreConditionalFileUpdate::Updated) => return Ok(proof), - Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue, - Err(error) => { - return Err(Error::TaskExecutionFailed { - message: format!("Failed to save replacement completion proof: {error}"), - }); - } - } - } - - Err(Error::TaskExecutionFailed { - message: format!("Replacement completion proof changed while publishing task {}", state.task_id), - }) - } - async fn save_state_with_unformatted_policy(&self, allow_unformatted: bool) -> Result<()> { let state = self.state.read().await.clone(); validate_resume_task_id(&state.task_id)?; @@ -1630,35 +1008,6 @@ impl ResumeManager { Ok(()) } - async fn write_replacement_intent_state( - &self, - path: &str, - state_data: EcstoreDiskBytes, - ) -> std::result::Result<(), DiskError> { - ensure_replacement_recovery_dir(&self.disk).await?; - for _ in 0..2 { - let expected = match self.disk.read_all(RUSTFS_META_BUCKET, path).await { - Ok(existing) => Some(existing), - Err(DiskError::FileNotFound) => None, - Err(error) => return Err(error), - }; - match super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file( - self.disk.as_ref(), - RUSTFS_META_BUCKET, - path, - expected, - Some(state_data.clone()), - ) - .await - { - Ok(EcstoreConditionalFileUpdate::Updated) => return Ok(()), - Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue, - Err(error) => return Err(error), - } - } - Err(DiskError::other("replacement intent changed while publishing")) - } - /// read state file from disk async fn read_state_file(disk: &DiskStore, task_id: &str, state_file: ResumeStateFile) -> Result> { validate_resume_task_id(task_id)?; @@ -1672,2571 +1021,5 @@ impl ResumeManager { } } -/// resume checkpoint -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ResumeCheckpoint { - /// on-disk schema version; absent in legacy snapshots (defaults to 0) - #[serde(default)] - pub schema_version: u32, - /// task id - pub task_id: String, - /// checkpoint time - pub checkpoint_time: u64, - /// current bucket index - pub current_bucket_index: usize, - /// current object index - pub current_object_index: usize, - /// Objects healed since the last completed page. HashSet: with the - /// previous Vec the per-object `contains` was O(n) and made large-bucket - /// heals O(N²). Only spans the in-flight page — completed pages are - /// covered by `current_object_index`, so `complete_page` prunes the sets. - pub processed_objects: HashSet, - /// failed objects - pub failed_objects: HashSet, - /// skipped objects - pub skipped_objects: HashSet, -} - -impl ResumeCheckpoint { - pub fn new(task_id: String) -> Self { - Self { - schema_version: CURRENT_CHECKPOINT_SCHEMA, - task_id, - checkpoint_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(), - current_bucket_index: 0, - current_object_index: 0, - processed_objects: HashSet::new(), - failed_objects: HashSet::new(), - skipped_objects: HashSet::new(), - } - } - - pub fn update_position(&mut self, bucket_index: usize, object_index: usize) { - self.current_bucket_index = bucket_index; - self.current_object_index = object_index; - self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); - } - - pub fn add_processed_object(&mut self, object: String) { - self.processed_objects.insert(object); - } - - pub fn add_failed_object(&mut self, object: String) { - self.failed_objects.insert(object); - } - - pub fn add_skipped_object(&mut self, object: String) { - self.skipped_objects.insert(object); - } - - /// Advance past a fully-processed page: objects below `object_index` are - /// skipped by position on resume, so the per-object sets no longer need - /// their entries and would otherwise grow with the whole bucket. - pub fn complete_page(&mut self, bucket_index: usize, object_index: usize) { - self.update_position(bucket_index, object_index); - self.processed_objects.clear(); - self.skipped_objects.clear(); - self.failed_objects.clear(); - } - - /// Reset the scan to the start and clear the per-object sets so a retry - /// re-scans the whole set. - pub fn reset_for_retry(&mut self) { - self.update_position(0, 0); - self.processed_objects.clear(); - self.skipped_objects.clear(); - self.failed_objects.clear(); - } -} - -/// resume checkpoint manager -pub struct CheckpointManager { - disk: DiskStore, - checkpoint: Arc>, - throttle: Mutex, -} - -impl CheckpointManager { - /// create new checkpoint manager - pub async fn new(disk: DiskStore, task_id: String) -> Result { - validate_resume_task_id(&task_id)?; - let checkpoint = ResumeCheckpoint::new(task_id); - let manager = Self { - disk, - checkpoint: Arc::new(RwLock::new(checkpoint)), - throttle: Mutex::new(PersistThrottle::new()), - }; - - // save initial checkpoint - if let Err(e) = manager.save_checkpoint().await { - warn!( - target: "rustfs::heal::resume", - event = EVENT_HEAL_CHECKPOINT_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_RESUME, - state = "initial_save_failed", - error = %e, - "Heal checkpoint persistence failed" - ); - } - Ok(manager) - } - - /// load checkpoint from disk - pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result { - validate_resume_task_id(task_id)?; - let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?; - let mut checkpoint: ResumeCheckpoint = - serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed { - message: format!("Failed to deserialize checkpoint: {e}"), - })?; - if checkpoint.task_id != task_id { - return Err(Error::TaskExecutionFailed { - message: "Resume checkpoint task id does not match filename".to_string(), - }); - } - - // A checkpoint from an older schema stored latest-only dedup identities - // that are not comparable to the new per-version `compose_key` - // identities. Discard the stale sets and position, then stamp the - // current schema so the scan restarts cleanly. - if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA { - return Err(Error::TaskExecutionFailed { - message: format!( - "Checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}", - checkpoint.schema_version - ), - }); - } - if checkpoint.schema_version < CURRENT_CHECKPOINT_SCHEMA { - warn!( - target: "rustfs::heal::resume", - event = EVENT_HEAL_CHECKPOINT_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_RESUME, - task_id, - found_schema = checkpoint.schema_version, - current_schema = CURRENT_CHECKPOINT_SCHEMA, - state = "schema_discarded", - "Heal checkpoint schema is stale; discarding dedup sets and position" - ); - checkpoint.processed_objects.clear(); - checkpoint.failed_objects.clear(); - checkpoint.skipped_objects.clear(); - checkpoint.current_bucket_index = 0; - checkpoint.current_object_index = 0; - checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA; - } - - Ok(Self { - disk, - checkpoint: Arc::new(RwLock::new(checkpoint)), - throttle: Mutex::new(PersistThrottle::new()), - }) - } - - /// check if checkpoint exists - pub async fn has_checkpoint(disk: &DiskStore, task_id: &str) -> bool { - if validate_resume_task_id(task_id).is_err() { - return false; - } - let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); - match path_to_str(&file_path) { - Ok(path_str) => match disk.read_all(RUSTFS_META_BUCKET, path_str).await { - Ok(data) => !data.is_empty(), - Err(_) => false, - }, - Err(_) => false, - } - } - - /// get current checkpoint - pub async fn get_checkpoint(&self) -> ResumeCheckpoint { - self.checkpoint.read().await.clone() - } - - /// update position - pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> { - let mut checkpoint = self.checkpoint.write().await; - checkpoint.update_position(bucket_index, object_index); - drop(checkpoint); - self.save_checkpoint_throttled().await - } - - /// Advance past a completed page and prune the per-object sets, then persist. - pub async fn complete_page(&self, bucket_index: usize, object_index: usize) -> Result<()> { - let mut checkpoint = self.checkpoint.write().await; - checkpoint.complete_page(bucket_index, object_index); - drop(checkpoint); - self.save_checkpoint_throttled().await - } - - /// Reset the checkpoint to the start of the scan for a retry, then persist. - pub async fn reset_for_retry(&self) -> Result<()> { - let mut checkpoint = self.checkpoint.write().await; - checkpoint.reset_for_retry(); - drop(checkpoint); - self.save_checkpoint_throttled().await - } - - /// Add a processed object. Called once per healed object, so persistence - /// is batched (`PERSIST_EVERY_MUTATIONS` / `PERSIST_INTERVAL`); positions - /// and page boundaries still persist unconditionally. - pub async fn add_processed_object(&self, object: String) -> Result<()> { - let mut checkpoint = self.checkpoint.write().await; - checkpoint.add_processed_object(object); - drop(checkpoint); - self.save_checkpoint_if_due().await - } - - /// add failed object (batched, see `add_processed_object`) - pub async fn add_failed_object(&self, object: String) -> Result<()> { - let mut checkpoint = self.checkpoint.write().await; - checkpoint.add_failed_object(object); - drop(checkpoint); - self.save_checkpoint_if_due().await - } - - /// add skipped object (batched, see `add_processed_object`) - pub async fn add_skipped_object(&self, object: String) -> Result<()> { - let mut checkpoint = self.checkpoint.write().await; - checkpoint.add_skipped_object(object); - drop(checkpoint); - self.save_checkpoint_if_due().await - } - - async fn save_checkpoint_if_due(&self) -> Result<()> { - let should_save = self.throttle.lock().map(|mut throttle| throttle.record()).unwrap_or(true); - if !should_save { - return Ok(()); - } - self.save_checkpoint_throttled().await - } - - async fn save_checkpoint_throttled(&self) -> Result<()> { - let result = self.save_checkpoint().await; - if result.is_ok() - && let Ok(mut throttle) = self.throttle.lock() - { - throttle.mark_saved(); - } - result - } - - /// cleanup checkpoint - pub async fn cleanup(&self) -> Result<()> { - let task_id = self.checkpoint.read().await.task_id.clone(); - validate_resume_task_id(&task_id)?; - - let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); - delete_resume_file(&self.disk, &checkpoint_file).await?; - - debug!( - target: "rustfs::heal::resume", - event = EVENT_HEAL_CHECKPOINT_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_RESUME, - task_id, - state = "cleaned", - "Heal checkpoint cleaned" - ); - Ok(()) - } - - /// save checkpoint to disk - async fn save_checkpoint(&self) -> Result<()> { - let checkpoint = self.checkpoint.read().await; - validate_resume_task_id(&checkpoint.task_id)?; - let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed { - message: format!("Failed to serialize checkpoint: {e}"), - })?; - - let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", checkpoint.task_id, RESUME_CHECKPOINT_FILE)); - - let path_str = path_to_str(&file_path)?; - self.disk - .write_all(RUSTFS_META_BUCKET, path_str, checkpoint_data.into()) - .await - .map_err(|e| Error::TaskExecutionFailed { - message: format!("Failed to save checkpoint: {e}"), - })?; - - debug!( - target: "rustfs::heal::resume", - event = EVENT_HEAL_CHECKPOINT_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_RESUME, - task_id = %checkpoint.task_id, - state = "saved", - "Heal checkpoint persisted" - ); - Ok(()) - } - - /// read checkpoint file from disk - async fn read_checkpoint_file(disk: &DiskStore, task_id: &str) -> Result> { - validate_resume_task_id(task_id)?; - let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); - - let path_str = path_to_str(&file_path)?; - disk.read_all(RUSTFS_META_BUCKET, path_str) - .await - .map(|bytes| bytes.to_vec()) - .map_err(|e| Error::TaskExecutionFailed { - message: format!("Failed to read checkpoint file: {e}"), - }) - } -} - -/// resume utils -pub struct ResumeUtils; - -impl ResumeUtils { - /// generate unique task id - pub fn generate_task_id() -> String { - Uuid::new_v4().to_string() - } - - /// check if task can be resumed - pub async fn can_resume_task(disk: &DiskStore, task_id: &str) -> bool { - ResumeManager::has_resume_state(disk, task_id).await - } - - /// get all resumable task ids - pub async fn get_resumable_tasks(disk: &DiskStore) -> Result> { - // List all files in the buckets metadata directory - let entries = match disk.list_dir("", RUSTFS_META_BUCKET, BUCKET_META_PREFIX, -1).await { - Ok(entries) => entries, - Err(e) => { - debug!( - target: "rustfs::heal::resume", - event = EVENT_HEAL_RESUME_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_RESUME, - state = "list_failed", - error = %e, - "Heal resume state listing failed" - ); - return Ok(Vec::new()); - } - }; - - let mut task_ids = Vec::new(); - - // Filter files that end with ahm_resume_state.json and extract task IDs - for entry in entries { - if entry.ends_with(&format!("_{RESUME_STATE_FILE}")) { - // Extract task ID from filename: {task_id}_ahm_resume_state.json - if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}")) - && validate_resume_task_id(task_id).is_ok() - { - task_ids.push(task_id.to_string()); - } - } - } - - debug!( - target: "rustfs::heal::resume", - event = EVENT_HEAL_RESUME_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_RESUME, - task_count = task_ids.len(), - state = "listed", - "Heal resume states listed" - ); - Ok(task_ids) - } - - /// Return replacement intent task IDs from the dedicated recovery - /// directory. Periodic recovery must never enumerate the ordinary resume - /// directory, whose cardinality is unrelated to replacement work. - pub async fn get_replacement_intent_tasks(disk: &DiskStore) -> Result> { - let entries = Self::replacement_recovery_entries(disk).await?; - let suffix = format!("_{REPLACEMENT_INTENT_FILE}"); - let mut task_ids = HashSet::new(); - - for entry in entries { - if let Some(task_id) = entry.strip_suffix(&suffix) - && validate_resume_task_id(task_id).is_ok() - { - task_ids.insert(task_id.to_string()); - continue; - } - } - - let mut task_ids = task_ids.into_iter().collect::>(); - task_ids.sort_unstable(); - Ok(task_ids) - } - - async fn replacement_recovery_entries(disk: &DiskStore) -> Result> { - let recovery_dir = replacement_recovery_dir(); - let recovery_dir = path_to_str(&recovery_dir)?; - match disk.list_dir("", RUSTFS_META_BUCKET, recovery_dir, -1).await { - Ok(entries) => Ok(entries), - Err(DiskError::FileNotFound) => Ok(Vec::new()), - Err(error @ DiskError::UnformattedDisk) => Err(error.into()), - Err(error) => Err(Error::TaskExecutionFailed { - message: format!("Failed to list replacement recovery records: {error}"), - }), - } - } - - /// Migrate flat replacement artifacts from earlier builds exactly once at - /// manager startup. The normal scanner only uses the dedicated directory; - /// ordinary resume JSON is never read on its periodic path. - pub async fn migrate_legacy_replacement_records(disk: &DiskStore) -> Result<()> { - let entries = disk - .list_dir("", RUSTFS_META_BUCKET, BUCKET_META_PREFIX, -1) - .await - .map_err(|error| Error::TaskExecutionFailed { - message: format!("Failed to list legacy replacement records: {error}"), - })?; - let ordinary_suffix = format!("_{RESUME_STATE_FILE}"); - let intent_suffix = format!("_{REPLACEMENT_INTENT_FILE}"); - let proof_suffix = format!("_{REPLACEMENT_COMPLETION_PROOF_FILE}"); - let mut ordinary_task_ids = HashSet::new(); - let mut intent_task_ids = HashSet::new(); - let mut proof_task_ids = HashSet::new(); - - for entry in entries { - if let Some(task_id) = entry.strip_suffix(&intent_suffix) - && validate_resume_task_id(task_id).is_ok() - { - intent_task_ids.insert(task_id.to_string()); - continue; - } - if let Some(task_id) = entry.strip_suffix(&ordinary_suffix) - && validate_resume_task_id(task_id).is_ok() - { - ordinary_task_ids.insert(task_id.to_string()); - continue; - } - if let Some(task_id) = entry.strip_suffix(&proof_suffix) - && validate_resume_task_id(task_id).is_ok() - { - proof_task_ids.insert(task_id.to_string()); - } - } - - let mut state_task_ids = intent_task_ids.into_iter().collect::>(); - state_task_ids.extend(ordinary_task_ids); - state_task_ids.sort_unstable(); - state_task_ids.dedup(); - for task_id in state_task_ids { - let has_flat_intent = ResumeManager::has_state_file(disk, &task_id, ResumeStateFile::LegacyReplacementIntent).await; - if !has_flat_intent { - let manager = ResumeManager::load_from_disk(disk.clone(), &task_id).await.map_err(|error| { - replacement_recovery_corruption_for_state_load( - format!("Failed to load legacy replacement recovery candidate {task_id}"), - error, - ) - })?; - if !is_replacement_intent(&manager.get_state().await) { - continue; - } - } - ResumeManager::load_replacement_intent(disk.clone(), &task_id).await?; - } - - for task_id in proof_task_ids { - ResumeManager::migrate_legacy_replacement_completion_proof(disk, &task_id).await?; - } - Ok(()) - } - - /// Return all durable replacement states and completion proofs stored on - /// one survivor disk. Unlike the legacy resumable-task helper, listing - /// failures are returned to the caller so an observability surface cannot - /// silently turn an unreadable durable record into a green result. - pub async fn get_replacement_recovery_records(disk: &DiskStore) -> Result> { - let entries = Self::replacement_recovery_entries(disk).await?; - let proof_suffix = format!("_{REPLACEMENT_COMPLETION_PROOF_FILE}"); - let mut records = Vec::new(); - let mut intent_task_ids = HashSet::new(); - - for task_id in Self::get_replacement_intent_tasks(disk).await? { - let state = ResumeManager::load_replacement_intent(disk.clone(), &task_id) - .await? - .get_state() - .await; - intent_task_ids.insert(task_id.clone()); - records.push(ReplacementRecoveryRecord::from_state(state).unwrap_or_else(|| { - ReplacementRecoveryRecord::unknown( - task_id, - "isolated replacement intent violates its generation or target identity binding", - ) - })); - } - - for entry in entries { - let Some(task_id) = entry.strip_suffix(&proof_suffix) else { - continue; - }; - if validate_resume_task_id(task_id).is_err() { - continue; - } - if intent_task_ids.contains(task_id) { - continue; - } - let proof = ResumeManager::load_replacement_completion_proof(disk.clone(), task_id).await?; - records.push(ReplacementRecoveryRecord::from_completion_proof(&proof)); - } - - records.sort_by(|left, right| left.task_id.cmp(&right.task_id).then(left.state.cmp(&right.state))); - Ok(records) - } - - /// cleanup expired resume states - pub async fn cleanup_expired_states(disk: &DiskStore, max_age_hours: u64) -> Result<()> { - let task_ids = Self::get_resumable_tasks(disk).await?; - let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); - - for task_id in task_ids { - if let Ok(resume_manager) = ResumeManager::load_from_disk(disk.clone(), &task_id).await { - let state = resume_manager.get_state().await; - let age_hours = current_time.saturating_sub(state.last_update) / 3600; - - if !state.completed && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) - { - continue; - } - if state.completed - && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) - { - continue; - } - - if age_hours > max_age_hours { - debug!( - target: "rustfs::heal::resume", - event = EVENT_HEAL_RESUME_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_RESUME, - task_id, - age_hours, - state = "expired_cleanup_started", - "Heal resume cleanup started" - ); - if let Err(e) = resume_manager.cleanup().await { - warn!( - target: "rustfs::heal::resume", - event = EVENT_HEAL_RESUME_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_RESUME, - task_id, - age_hours, - state = "expired_cleanup_failed", - error = %e, - "Heal resume state cleanup failed" - ); - } - } - } - } - - for task_id in Self::get_replacement_intent_tasks(disk).await? { - if let Ok(resume_manager) = ResumeManager::load_replacement_intent(disk.clone(), &task_id).await { - let state = resume_manager.get_state().await; - let age_hours = current_time.saturating_sub(state.last_update) / 3600; - - if !state.completed && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) - { - continue; - } - if state.completed - && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) - { - continue; - } - - if age_hours > max_age_hours - && let Err(e) = resume_manager.cleanup().await - { - warn!( - target: "rustfs::heal::resume", - event = EVENT_HEAL_RESUME_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_RESUME, - task_id, - age_hours, - state = "expired_cleanup_failed", - error = %e, - "Replacement intent cleanup failed" - ); - } - } - } - - Ok(()) - } -} - #[cfg(test)] -mod tests { - use super::*; - - async fn schema_test_disk() -> (tempfile::TempDir, DiskStore) { - use super::super::{DiskOption, Endpoint, new_disk}; - - let temp_dir = tempfile::TempDir::new().expect("create schema test directory"); - let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create schema test disk endpoint"); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .expect("create schema test disk"); - match disk.make_volume(RUSTFS_META_BUCKET).await { - Ok(()) | Err(DiskError::VolumeExists) => {} - Err(error) => panic!("create metadata volume: {error}"), - } - match disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await { - Ok(()) | Err(DiskError::VolumeExists) => {} - Err(error) => panic!("create resume metadata volume: {error}"), - } - - (temp_dir, disk) - } - - #[tokio::test] - async fn test_resume_state_creation() { - let task_id = ResumeUtils::generate_task_id(); - let buckets = vec!["bucket1".to_string(), "bucket2".to_string()]; - let state = ResumeState::new(task_id.clone(), "erasure_set".to_string(), "pool_0_set_0".to_string(), buckets); - - assert_eq!(state.task_id, task_id); - assert_eq!(state.task_type, "erasure_set"); - assert!(!state.completed); - assert_eq!(state.processed_objects, 0); - assert_eq!(state.pending_buckets.len(), 2); - } - - #[test] - fn replacement_intent_binds_a_generation_before_format() { - let state = ResumeState::replacement_intent( - "generation-a".to_string(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket-a".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ); - - assert_eq!(state.replacement_generation.as_deref(), Some("generation-a")); - assert_eq!(state.replacement_phase, ReplacementPhase::Intent); - assert_eq!(state.replacement_targets, ["replacement-a"]); - assert!(state.resume_cursor.is_none(), "a new replacement must start from the beginning"); - - let mut state = state; - state.complete_bucket("bucket-a"); - assert_eq!( - state.replacement_buckets, - ["bucket-a"], - "recovery must retain the original positional bucket plan" - ); - } - - #[tokio::test] - async fn replacement_terminal_phases_are_durable() { - use super::super::{DiskOption, Endpoint, new_disk}; - use tempfile::TempDir; - - let temp_dir = TempDir::new().expect("create replacement phase test directory"); - let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create test disk endpoint"); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .expect("create test disk"); - match disk.make_volume(RUSTFS_META_BUCKET).await { - Ok(()) | Err(DiskError::VolumeExists) => {} - Err(err) => panic!("create metadata volume for replacement phase test: {err}"), - } - - let task_id = ResumeUtils::generate_task_id(); - let manager = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - .expect("replacement intent should persist"); - manager - .mark_replacement_completed_and_verified() - .await - .expect("completion and verified phase must persist together"); - - let verified = ResumeManager::load_replacement_intent(disk.clone(), &task_id) - .await - .expect("verified phase must survive a restart") - .get_state() - .await; - assert!(verified.completed); - assert_eq!(verified.replacement_phase, ReplacementPhase::Verified); - - let resumed = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["ignored-after-restart".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - .expect("verified replacement must be reopenable for terminal cleanup"); - assert_eq!( - resumed.get_state().await.replacement_buckets, - ["bucket"], - "terminal recovery must preserve the original generation bucket plan" - ); - - manager - .mark_replacement_cleanup_pending() - .await - .expect("cleanup-pending phase must persist after marker removal"); - let cleanup_pending = ResumeManager::load_replacement_intent(disk, &task_id) - .await - .expect("cleanup-pending phase must survive a restart") - .get_state() - .await; - assert!(cleanup_pending.completed); - assert_eq!(cleanup_pending.replacement_phase, ReplacementPhase::CleanupPending); - } - - #[tokio::test] - async fn replacement_proof_before_verified_state_is_reconciled_after_restart() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let manager = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - .expect("replacement intent should persist"); - - let proof = ReplacementCompletionProof::from_state(&manager.get_state().await, 42) - .expect("active replacement state should build a completion proof"); - let proof_path = replacement_completion_proof_path(&task_id); - disk.write_all( - RUSTFS_META_BUCKET, - proof_path.to_str().expect("completion proof path must be UTF-8"), - serde_json::to_vec(&proof) - .expect("completion proof fixture should serialize") - .into(), - ) - .await - .expect("proof-first crash fixture should persist"); - - let recovered = ResumeManager::load_replacement_intent(disk.clone(), &task_id) - .await - .expect("matching completion proof must prevent another rebuild") - .get_state() - .await; - assert!(recovered.completed); - assert_eq!(recovered.replacement_phase, ReplacementPhase::Verified); - assert_eq!(recovered.last_update, proof.verified_at); - - let records = ResumeUtils::get_replacement_recovery_records(&disk) - .await - .expect("reconciled replacement state should be observable"); - assert_eq!(records.len(), 1); - assert_eq!(records[0].state, ReplacementRecoveryState::CleanupPending); - } - - #[tokio::test] - async fn replacement_proof_conflicting_with_active_state_fails_closed_after_restart() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let manager = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - .expect("replacement intent should persist"); - - let mut proof = ReplacementCompletionProof::from_state(&manager.get_state().await, 42) - .expect("active replacement state should build a completion proof"); - proof.set_disk_id = "pool_0_set_1".to_string(); - let proof_path = replacement_completion_proof_path(&task_id); - disk.write_all( - RUSTFS_META_BUCKET, - proof_path.to_str().expect("completion proof path must be UTF-8"), - serde_json::to_vec(&proof) - .expect("completion proof fixture should serialize") - .into(), - ) - .await - .expect("conflicting proof fixture should persist"); - - let error = match ResumeManager::load_replacement_intent(disk, &task_id).await { - Ok(_) => panic!("a mismatched proof must not permit another rebuild"), - Err(error) => error, - }; - assert!(error.to_string().contains("does not match active intent")); - } - - #[tokio::test] - async fn replacement_intent_is_not_an_ordinary_resumable_task() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let manager = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - .expect("replacement intent should persist in its isolated namespace"); - - assert!( - !ResumeManager::has_resume_state(&disk, &task_id).await, - "an old ordinary-resume lookup must not discover a replacement intent" - ); - assert!(ResumeManager::has_replacement_intent(&disk, &task_id).await); - assert!( - !ResumeUtils::get_resumable_tasks(&disk) - .await - .expect("ordinary resume listing should succeed") - .contains(&task_id), - "the old filename enumeration must not return replacement work" - ); - assert_eq!( - ResumeUtils::get_replacement_intent_tasks(&disk) - .await - .expect("replacement intent listing should succeed"), - vec![task_id.clone()] - ); - assert_eq!( - ResumeManager::load_replacement_intent(disk, &task_id) - .await - .expect("new replacement reader should load the isolated state") - .get_state() - .await, - manager.get_state().await - ); - } - - #[tokio::test] - async fn replacement_intent_recovers_from_torn_publication_before_formatting() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let intent_path = ResumeStateFile::ReplacementIntent.path(&task_id); - disk.write_all( - RUSTFS_META_BUCKET, - intent_path.to_str().expect("replacement intent path must be UTF-8"), - b"{torn replacement intent".as_slice().into(), - ) - .await - .expect("torn replacement intent fixture should persist"); - - let manager = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - .expect("a retry must atomically replace only a torn pre-format intent"); - - let recovered = ResumeManager::load_replacement_intent(disk, &task_id) - .await - .expect("recovered intent should be readable after restart") - .get_state() - .await; - assert_eq!(recovered, manager.get_state().await); - assert_eq!(recovered.replacement_phase, ReplacementPhase::Intent); - } - - #[tokio::test] - async fn torn_intent_recovery_cas_preserves_a_concurrent_valid_binding() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let intent_path = ResumeStateFile::ReplacementIntent.path(&task_id); - let intent_path = intent_path.to_str().expect("replacement intent path must be UTF-8"); - let torn = EcstoreDiskBytes::from_static(b"{torn replacement intent"); - disk.write_all(RUSTFS_META_BUCKET, intent_path, torn) - .await - .expect("torn intent fixture should persist"); - - let expected = ResumeManager::torn_replacement_intent_bytes(&disk, &task_id) - .await - .expect("torn intent should be recoverable before a seal exists") - .expect("torn intent bytes should be retained as the CAS precondition"); - let winner = ResumeState::replacement_intent( - task_id.clone(), - "erasure_set".to_string(), - "pool_0_set_winner".to_string(), - vec!["winner-bucket".to_string()], - vec!["replacement-winner".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-winner".to_string(), - canonical_path: "/mnt/replacement-winner".to_string(), - physical_device_ids: vec!["device-winner".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ); - let winner_bytes = EcstoreDiskBytes::from(serde_json::to_vec(&winner).expect("winner intent should serialize")); - disk.write_all(RUSTFS_META_BUCKET, intent_path, winner_bytes.clone()) - .await - .expect("concurrent valid intent fixture should persist"); - - let loser = ResumeManager { - disk: disk.clone(), - state: Arc::new(RwLock::new(ResumeState::replacement_intent( - task_id.clone(), - "erasure_set".to_string(), - "pool_0_set_loser".to_string(), - vec!["loser-bucket".to_string()], - vec!["replacement-loser".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-loser".to_string(), - canonical_path: "/mnt/replacement-loser".to_string(), - physical_device_ids: vec!["device-loser".to_string()], - filesystem_identity: "4:5:6".to_string(), - }], - ))), - throttle: Mutex::new(PersistThrottle::new()), - state_file: ResumeStateFile::ReplacementIntent, - }; - let error = match loser.publish_new_replacement_intent(Some(expected)).await { - Ok(()) => panic!("a stale torn-intent recovery must not overwrite a concurrent valid binding"), - Err(error) => error, - }; - assert!(error.to_string().contains("changed before publication")); - assert_eq!( - disk.read_all(RUSTFS_META_BUCKET, intent_path) - .await - .expect("concurrent valid intent must remain durable"), - winner_bytes - ); - } - - #[tokio::test] - async fn replacement_intent_does_not_recreate_torn_state_after_seal_publication() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let intent_path = ResumeStateFile::ReplacementIntent.path(&task_id); - let intent_path = intent_path.to_str().expect("replacement intent path must be UTF-8"); - let torn = b"{torn replacement intent"; - disk.write_all(RUSTFS_META_BUCKET, intent_path, torn.as_slice().into()) - .await - .expect("torn replacement intent fixture should persist"); - let marker_path = replacement_intent_seal_path(&task_id); - disk.write_all( - RUSTFS_META_BUCKET, - marker_path.to_str().expect("replacement seal path must be UTF-8"), - b"sealed".as_slice().into(), - ) - .await - .expect("replacement seal fixture should persist"); - - let error = match ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - { - Ok(_) => panic!("a seal means a torn state may have crossed the format boundary"), - Err(error) => error, - }; - assert!(error.to_string().contains("Failed to deserialize resume state")); - assert_eq!( - disk.read_all(RUSTFS_META_BUCKET, intent_path) - .await - .expect("torn intent must remain for operator recovery"), - torn.as_slice() - ); - } - - #[tokio::test] - async fn replacement_intent_migrates_from_legacy_resume_filename() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let legacy = ResumeState::replacement_intent( - task_id.clone(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ); - let legacy_path = ResumeStateFile::Ordinary.path(&task_id); - disk.write_all( - RUSTFS_META_BUCKET, - legacy_path.to_str().expect("legacy resume path must be UTF-8"), - serde_json::to_vec(&legacy) - .expect("serialize legacy replacement state") - .into(), - ) - .await - .expect("write legacy replacement state"); - - let migrated = ResumeManager::load_replacement_intent(disk.clone(), &task_id) - .await - .expect("new binary should migrate a legacy replacement state"); - assert_eq!(migrated.get_state().await, legacy); - assert!( - !ResumeManager::has_resume_state(&disk, &task_id).await, - "migration must remove the old-binary-visible state only after the new state is durable" - ); - assert!(ResumeManager::has_replacement_intent(&disk, &task_id).await); - } - - #[tokio::test] - async fn ordinary_targeted_resume_is_not_migrated_as_a_replacement_intent() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let manager = ResumeManager::new( - disk.clone(), - task_id.clone(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - ) - .await - .expect("ordinary targeted resume should persist"); - manager - .set_replacement_targets(vec!["manual-target".to_string()]) - .await - .expect("ordinary targeted resume should retain its target filter"); - - assert!(!ResumeManager::has_replacement_intent(&disk, &task_id).await); - assert!(ResumeManager::load_replacement_intent(disk.clone(), &task_id).await.is_err()); - assert!(ResumeManager::has_resume_state(&disk, &task_id).await); - } - - #[tokio::test] - async fn malformed_isolated_replacement_intent_is_reported_as_unknown() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let malformed = ResumeState::new( - task_id.clone(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - ); - let path = ResumeStateFile::ReplacementIntent.path(&task_id); - disk.write_all( - RUSTFS_META_BUCKET, - path.to_str().expect("isolated replacement path must be UTF-8"), - serde_json::to_vec(&malformed) - .expect("malformed replacement fixture should serialize") - .into(), - ) - .await - .expect("malformed isolated replacement state should persist"); - - let records = ResumeUtils::get_replacement_recovery_records(&disk) - .await - .expect("isolated replacement listing should succeed"); - assert_eq!(records.len(), 1); - assert_eq!(records[0].task_id, task_id); - assert_eq!(records[0].state, ReplacementRecoveryState::Unknown); - } - - #[tokio::test] - async fn startup_migration_moves_flat_replacement_artifacts_to_dedicated_directory() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let mut state = ResumeState::replacement_intent( - task_id.clone(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ); - state.mark_completed(); - state.replacement_phase = ReplacementPhase::Verified; - let legacy_intent = ResumeStateFile::LegacyReplacementIntent.path(&task_id); - disk.write_all( - RUSTFS_META_BUCKET, - legacy_intent.to_str().expect("legacy intent path must be UTF-8"), - serde_json::to_vec(&state) - .expect("serialize legacy replacement intent") - .into(), - ) - .await - .expect("write legacy replacement intent"); - let proof = ReplacementCompletionProof::from_state(&state, state.last_update).expect("build legacy completion proof"); - let legacy_proof = legacy_replacement_completion_proof_path(&task_id); - disk.write_all( - RUSTFS_META_BUCKET, - legacy_proof.to_str().expect("legacy proof path must be UTF-8"), - serde_json::to_vec(&proof).expect("serialize legacy completion proof").into(), - ) - .await - .expect("write legacy completion proof"); - - ResumeUtils::migrate_legacy_replacement_records(&disk) - .await - .expect("startup migration should move flat replacement artifacts"); - - assert_eq!( - ResumeUtils::get_replacement_intent_tasks(&disk) - .await - .expect("dedicated intent listing should succeed"), - vec![task_id.clone()] - ); - assert_eq!( - ResumeManager::load_replacement_completion_proof(disk.clone(), &task_id) - .await - .expect("dedicated completion proof should be readable"), - proof - ); - assert!(matches!( - disk.read_all(RUSTFS_META_BUCKET, legacy_intent.to_str().expect("legacy intent path must be UTF-8")) - .await, - Err(DiskError::FileNotFound) - )); - assert!(matches!( - disk.read_all(RUSTFS_META_BUCKET, legacy_proof.to_str().expect("legacy proof path must be UTF-8")) - .await, - Err(DiskError::FileNotFound) - )); - } - - #[tokio::test] - async fn startup_migration_moves_ordinary_replacement_resume_to_dedicated_directory() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let state = ResumeState::replacement_intent( - task_id.clone(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ); - let legacy_resume = ResumeStateFile::Ordinary.path(&task_id); - disk.write_all( - RUSTFS_META_BUCKET, - legacy_resume.to_str().expect("legacy resume path must be UTF-8"), - serde_json::to_vec(&state) - .expect("serialize legacy ordinary replacement state") - .into(), - ) - .await - .expect("write legacy ordinary replacement state"); - - ResumeUtils::migrate_legacy_replacement_records(&disk) - .await - .expect("startup migration should move ordinary replacement state"); - - assert_eq!( - ResumeUtils::get_replacement_intent_tasks(&disk) - .await - .expect("dedicated replacement listing should succeed"), - vec![task_id.clone()] - ); - assert_eq!( - ResumeManager::load_replacement_intent(disk.clone(), &task_id) - .await - .expect("migrated replacement intent should be readable") - .get_state() - .await, - state - ); - assert!(matches!( - disk.read_all(RUSTFS_META_BUCKET, legacy_resume.to_str().expect("legacy resume path must be UTF-8")) - .await, - Err(DiskError::FileNotFound) - )); - } - - #[tokio::test] - async fn startup_migration_reports_corrupt_ordinary_replacement_candidate() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let legacy_resume = ResumeStateFile::Ordinary.path(&task_id); - let legacy_resume = legacy_resume.to_str().expect("legacy resume path must be UTF-8"); - disk.write_all(RUSTFS_META_BUCKET, legacy_resume, b"{corrupt replacement resume".as_slice().into()) - .await - .expect("corrupt legacy replacement candidate should persist"); - - let error = ResumeUtils::migrate_legacy_replacement_records(&disk) - .await - .expect_err("a corrupt UUID-named legacy candidate must fail closed"); - - assert!(replacement_recovery_error_requires_block(&error)); - assert!(error.to_string().contains("replacement recovery corruption")); - assert_eq!( - disk.read_all(RUSTFS_META_BUCKET, legacy_resume) - .await - .expect("corrupt legacy state must remain for operator recovery"), - b"{corrupt replacement resume".as_slice() - ); - } - - #[tokio::test] - async fn startup_migration_reports_corrupt_flat_replacement_intent() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let legacy_intent = ResumeStateFile::LegacyReplacementIntent.path(&task_id); - let legacy_intent = legacy_intent.to_str().expect("legacy intent path must be UTF-8"); - disk.write_all(RUSTFS_META_BUCKET, legacy_intent, b"{corrupt replacement intent".as_slice().into()) - .await - .expect("corrupt legacy replacement intent should persist"); - - let error = ResumeUtils::migrate_legacy_replacement_records(&disk) - .await - .expect_err("a corrupt flat legacy intent must fail closed"); - - assert!(replacement_recovery_error_requires_block(&error)); - assert!(error.to_string().contains("replacement recovery corruption")); - assert_eq!( - disk.read_all(RUSTFS_META_BUCKET, legacy_intent) - .await - .expect("corrupt legacy intent must remain for operator recovery"), - b"{corrupt replacement intent".as_slice() - ); - } - - #[tokio::test] - async fn startup_migration_preserves_conflicting_dedicated_and_legacy_state() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - .expect("dedicated replacement intent should persist"); - let dedicated_path = ResumeStateFile::ReplacementIntent.path(&task_id); - let dedicated_bytes = disk - .read_all(RUSTFS_META_BUCKET, dedicated_path.to_str().expect("dedicated intent path must be UTF-8")) - .await - .expect("dedicated replacement intent should be readable"); - - let legacy_state = ResumeState::replacement_intent( - task_id.clone(), - "erasure_set".to_string(), - "pool_0_set_1".to_string(), - vec!["other-bucket".to_string()], - vec!["replacement-b".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-b".to_string(), - canonical_path: "/mnt/replacement-b".to_string(), - physical_device_ids: vec!["device-b".to_string()], - filesystem_identity: "4:5:6".to_string(), - }], - ); - let legacy_path = ResumeStateFile::LegacyReplacementIntent.path(&task_id); - let legacy_bytes = serde_json::to_vec(&legacy_state).expect("serialize conflicting legacy replacement state"); - disk.write_all( - RUSTFS_META_BUCKET, - legacy_path.to_str().expect("legacy intent path must be UTF-8"), - legacy_bytes.clone().into(), - ) - .await - .expect("conflicting legacy replacement intent should persist"); - - let error = ResumeUtils::migrate_legacy_replacement_records(&disk) - .await - .expect_err("conflicting replacement states must fail closed"); - assert!(error.to_string().contains("conflicting legacy state")); - assert_eq!( - disk.read_all(RUSTFS_META_BUCKET, dedicated_path.to_str().expect("dedicated intent path must be UTF-8"),) - .await - .expect("dedicated state must remain after conflict"), - dedicated_bytes - ); - assert_eq!( - disk.read_all(RUSTFS_META_BUCKET, legacy_path.to_str().expect("legacy intent path must be UTF-8"),) - .await - .expect("legacy state must remain after conflict"), - legacy_bytes - ); - } - - #[tokio::test] - async fn startup_migration_preserves_conflicting_dedicated_and_legacy_proof() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let manager = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - .expect("dedicated replacement intent should persist"); - manager - .mark_replacement_completed_and_verified() - .await - .expect("dedicated completion proof should persist"); - let dedicated_path = replacement_completion_proof_path(&task_id); - let dedicated_bytes = disk - .read_all(RUSTFS_META_BUCKET, dedicated_path.to_str().expect("dedicated proof path must be UTF-8")) - .await - .expect("dedicated completion proof should be readable"); - - let mut legacy_proof = ReplacementCompletionProof::from_state(&manager.get_state().await, 42) - .expect("legacy completion proof fixture should build"); - legacy_proof.set_disk_id = "pool_0_set_1".to_string(); - let legacy_path = legacy_replacement_completion_proof_path(&task_id); - let legacy_bytes = serde_json::to_vec(&legacy_proof).expect("serialize conflicting legacy completion proof"); - disk.write_all( - RUSTFS_META_BUCKET, - legacy_path.to_str().expect("legacy proof path must be UTF-8"), - legacy_bytes.clone().into(), - ) - .await - .expect("conflicting legacy completion proof should persist"); - - let error = ResumeUtils::migrate_legacy_replacement_records(&disk) - .await - .expect_err("conflicting completion proofs must fail closed"); - assert!(error.to_string().contains("conflicts with legacy proof")); - assert_eq!( - disk.read_all(RUSTFS_META_BUCKET, dedicated_path.to_str().expect("dedicated proof path must be UTF-8"),) - .await - .expect("dedicated proof must remain after conflict"), - dedicated_bytes - ); - assert_eq!( - disk.read_all(RUSTFS_META_BUCKET, legacy_path.to_str().expect("legacy proof path must be UTF-8"),) - .await - .expect("legacy proof must remain after conflict"), - legacy_bytes - ); - let error = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await { - Ok(_) => panic!("a conflicting legacy proof must not be ignored during recovery"), - Err(error) => error, - }; - assert!(error.to_string().contains("conflicts with legacy proof")); - } - - #[tokio::test] - async fn replacement_discovery_does_not_read_ordinary_resume_directory() { - let (_temp_dir, disk) = schema_test_disk().await; - for _ in 0..3 { - ResumeManager::new( - disk.clone(), - ResumeUtils::generate_task_id(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - ) - .await - .expect("ordinary resume state should persist"); - } - let corrupt_task_id = ResumeUtils::generate_task_id(); - let corrupt_path = ResumeStateFile::Ordinary.path(&corrupt_task_id); - disk.write_all( - RUSTFS_META_BUCKET, - corrupt_path.to_str().expect("ordinary resume path must be UTF-8"), - b"not-json".to_vec().into(), - ) - .await - .expect("corrupt ordinary resume state should persist"); - - let replacement_task_id = ResumeUtils::generate_task_id(); - ResumeManager::new_replacement_intent( - disk.clone(), - replacement_task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - .expect("replacement intent should persist in the dedicated directory"); - - assert_eq!( - ResumeUtils::get_replacement_intent_tasks(&disk) - .await - .expect("dedicated replacement listing should not parse ordinary JSON"), - vec![replacement_task_id] - ); - } - - #[tokio::test] - async fn empty_replacement_recovery_directory_is_not_an_error() { - let (_temp_dir, disk) = schema_test_disk().await; - assert!( - ResumeUtils::get_replacement_intent_tasks(&disk) - .await - .expect("missing recovery directory should be empty") - .is_empty() - ); - assert!( - ResumeUtils::get_replacement_recovery_records(&disk) - .await - .expect("missing recovery directory should have no records") - .is_empty() - ); - ResumeUtils::cleanup_expired_states(&disk, 0) - .await - .expect("missing recovery directory should not block expiry cleanup"); - } - - #[tokio::test] - async fn replacement_completion_proof_survives_resume_cleanup() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let identity = ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }; - let manager = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![identity.clone()], - ) - .await - .expect("replacement intent should persist"); - manager - .mark_replacement_completed_and_verified() - .await - .expect("verified replacement must persist proof before completion"); - - let proof = ResumeManager::load_replacement_completion_proof(disk.clone(), &task_id) - .await - .expect("completion proof must be readable from the survivor anchor"); - assert_eq!(proof.task_id, task_id); - assert_eq!(proof.replacement_generation, proof.task_id); - assert_eq!(proof.set_disk_id, "pool_0_set_0"); - assert_eq!(proof.replacement_targets, ["replacement-a"]); - assert_eq!(proof.replacement_target_identities, vec![identity]); - assert!(proof.verified_at > 0); - - manager.cleanup().await.expect("resume cleanup should succeed"); - assert!( - !ResumeManager::has_replacement_intent(&disk, &proof.task_id).await, - "completion cleanup must remove the resumable state" - ); - assert_eq!( - ResumeManager::load_replacement_completion_proof(disk, &proof.task_id) - .await - .expect("survivor proof must outlive resume cleanup"), - proof - ); - } - - #[tokio::test] - async fn replacement_recovery_records_distinguish_active_and_proven_completion() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let manager = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - .expect("replacement intent should persist"); - - let active = ResumeUtils::get_replacement_recovery_records(&disk) - .await - .expect("active replacement record should be readable"); - assert_eq!(active.len(), 1); - assert_eq!(active[0].task_id, task_id); - assert_eq!(active[0].state, ReplacementRecoveryState::WaitingForReplacement); - assert_eq!(active[0].generation.as_deref(), Some(task_id.as_str())); - - manager - .mark_replacement_completed_and_verified() - .await - .expect("completion proof should persist"); - - let cleanup_pending = ResumeUtils::get_replacement_recovery_records(&disk) - .await - .expect("cleanup-pending replacement record should be readable"); - assert_eq!(cleanup_pending.len(), 1); - assert_eq!(cleanup_pending[0].state, ReplacementRecoveryState::CleanupPending); - - manager.cleanup().await.expect("resume state cleanup should succeed"); - - let completed = ResumeUtils::get_replacement_recovery_records(&disk) - .await - .expect("completion proof should remain readable"); - assert_eq!(completed.len(), 1); - assert_eq!(completed[0].state, ReplacementRecoveryState::Completed); - assert!(completed[0].verified_at.is_some()); - } - - #[tokio::test] - async fn replacement_completion_write_failure_cannot_mark_completed() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let manager = ResumeManager::new_replacement_intent( - disk, - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - .expect("replacement intent should persist"); - let proof_path = replacement_completion_proof_path(&task_id) - .to_str() - .expect("completion proof path must be UTF-8") - .to_string(); - let _failure = ReplacementProofWriteFailure::install(proof_path, DiskError::DiskAccessDenied); - - let error = manager - .mark_replacement_completed_and_verified() - .await - .expect_err("completion must fail closed when durable proof cannot be written"); - assert!(error.to_string().contains("Failed to save replacement completion proof")); - let state = manager.get_state().await; - assert!(!state.completed, "a failed proof write must not produce a completed state"); - assert_eq!(state.replacement_phase, ReplacementPhase::Intent); - } - - #[tokio::test] - async fn replacement_completion_repairs_torn_proof_without_wedging_rebuild() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let identity = ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }; - let manager = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![identity.clone()], - ) - .await - .expect("replacement intent should persist"); - manager - .mark_replacement_rebuilding(vec![identity]) - .await - .expect("replacement fixture should enter rebuilding before proof publication"); - let proof_path = replacement_completion_proof_path(&task_id); - let proof_path = proof_path.to_str().expect("completion proof path must be UTF-8"); - disk.write_all(RUSTFS_META_BUCKET, proof_path, b"{torn completion proof".as_slice().into()) - .await - .expect("torn proof fixture should persist"); - - manager - .mark_replacement_completed_and_verified() - .await - .expect("a rebuilding generation must replace only its torn completion proof"); - - let proof = ResumeManager::load_replacement_completion_proof(disk, &task_id) - .await - .expect("repaired completion proof should be durable and readable"); - assert_eq!(proof.task_id, task_id); - assert_eq!(proof.replacement_generation, proof.task_id); - } - - #[tokio::test] - async fn replacement_completion_does_not_replace_a_valid_mismatched_proof() { - let (_temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let manager = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }], - ) - .await - .expect("replacement intent should persist"); - let mut conflicting = ReplacementCompletionProof::from_state(&manager.get_state().await, 1) - .expect("replacement state should build a proof fixture"); - conflicting.set_disk_id = "pool_0_set_1".to_string(); - let proof_path = replacement_completion_proof_path(&task_id); - let proof_path = proof_path.to_str().expect("completion proof path must be UTF-8"); - disk.write_all( - RUSTFS_META_BUCKET, - proof_path, - serde_json::to_vec(&conflicting) - .expect("proof fixture should serialize") - .into(), - ) - .await - .expect("conflicting proof fixture should persist"); - - let error = manager - .mark_replacement_completed_and_verified() - .await - .expect_err("a distinct durable generation binding must not be overwritten"); - assert!(error.to_string().contains("does not match task")); - assert_eq!( - ResumeManager::load_replacement_completion_proof(disk, &task_id) - .await - .expect("valid conflicting proof should remain intact"), - conflicting - ); - } - - #[tokio::test] - async fn cleanup_expired_states_keeps_all_durable_replacement_phases() { - use super::super::{DiskOption, Endpoint, new_disk}; - use tempfile::TempDir; - - let temp_dir = TempDir::new().expect("create replacement expiry test directory"); - let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create test disk endpoint"); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .expect("create test disk"); - match disk.make_volume(RUSTFS_META_BUCKET).await { - Ok(()) | Err(DiskError::VolumeExists) => {} - Err(err) => panic!("create metadata volume for replacement expiry test: {err}"), - } - - let target = ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }; - let intent_task_id = ResumeUtils::generate_task_id(); - let rebuilding_task_id = ResumeUtils::generate_task_id(); - let verified_task_id = ResumeUtils::generate_task_id(); - let cleanup_pending_task_id = ResumeUtils::generate_task_id(); - let abandoned_task_id = ResumeUtils::generate_task_id(); - let ordinary_task_id = ResumeUtils::generate_task_id(); - let intent = ResumeManager::new_replacement_intent( - disk.clone(), - intent_task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![target.clone()], - ) - .await - .expect("replacement intent should persist"); - let rebuilding = ResumeManager::new_replacement_intent( - disk.clone(), - rebuilding_task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![target.clone()], - ) - .await - .expect("replacement rebuilding state should persist"); - rebuilding - .mark_replacement_rebuilding(vec![target.clone()]) - .await - .expect("replacement rebuilding phase should persist"); - let verified = ResumeManager::new_replacement_intent( - disk.clone(), - verified_task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![target.clone()], - ) - .await - .expect("replacement verified state should persist"); - verified - .mark_replacement_completed_and_verified() - .await - .expect("replacement verified phase should persist"); - let cleanup_pending = ResumeManager::new_replacement_intent( - disk.clone(), - cleanup_pending_task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![target.clone()], - ) - .await - .expect("replacement cleanup-pending state should persist"); - cleanup_pending - .mark_replacement_completed_and_verified() - .await - .expect("replacement completion should persist"); - cleanup_pending - .mark_replacement_cleanup_pending() - .await - .expect("replacement cleanup-pending phase should persist"); - let abandoned = ResumeManager::new_replacement_intent( - disk.clone(), - abandoned_task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - vec!["replacement-a".to_string()], - vec![target], - ) - .await - .expect("replacement abandoned state should persist"); - abandoned - .abandon_replacement_intent() - .await - .expect("replacement abandoned phase should persist"); - let ordinary = ResumeManager::new( - disk.clone(), - ordinary_task_id.clone(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - ) - .await - .expect("ordinary resume state should persist"); - - for manager in [&intent, &rebuilding, &verified, &cleanup_pending, &abandoned, &ordinary] { - manager.state.write().await.last_update = 0; - manager.save_state_strict().await.expect("persist expired resume state"); - } - - ResumeUtils::cleanup_expired_states(&disk, 0) - .await - .expect("replacement expiry cleanup should complete"); - - for (task_id, expected_phase) in [ - (intent_task_id.as_str(), ReplacementPhase::Intent), - (rebuilding_task_id.as_str(), ReplacementPhase::Rebuilding), - (verified_task_id.as_str(), ReplacementPhase::Verified), - (cleanup_pending_task_id.as_str(), ReplacementPhase::CleanupPending), - ] { - let state = ResumeManager::load_replacement_intent(disk.clone(), task_id) - .await - .expect("durable replacement state must survive expiry cleanup") - .get_state() - .await; - assert_eq!(state.replacement_phase, expected_phase); - } - assert!( - !ResumeManager::has_replacement_intent(&disk, &abandoned_task_id).await, - "an abandoned replacement must expire" - ); - assert!( - !ResumeManager::has_resume_state(&disk, &ordinary_task_id).await, - "an ordinary expired resume must expire" - ); - } - - #[tokio::test] - async fn test_resume_state_progress() { - let task_id = ResumeUtils::generate_task_id(); - let buckets = vec!["bucket1".to_string()]; - let mut state = ResumeState::new(task_id, "erasure_set".to_string(), "pool_0_set_0".to_string(), buckets); - - state.update_progress(10, 8, 1, 1); - assert_eq!(state.processed_objects, 10); - assert_eq!(state.successful_objects, 8); - assert_eq!(state.failed_objects, 1); - assert_eq!(state.skipped_objects, 1); - - let progress = state.get_progress_percentage(); - assert_eq!(progress, 0.0); // total_objects is 0 - - state.total_objects = 100; - let progress = state.get_progress_percentage(); - assert_eq!(progress, 10.0); - } - - #[tokio::test] - async fn replacement_intent_rejects_a_new_mount_at_the_same_endpoint() { - use super::super::{DiskOption, Endpoint, new_disk}; - use tempfile::TempDir; - - let temp_dir = TempDir::new().unwrap(); - let disk_path = temp_dir.path().join("resume_disk"); - std::fs::create_dir_all(&disk_path).unwrap(); - let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).unwrap(); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .unwrap(); - let _ = disk.make_volume(RUSTFS_META_BUCKET).await; - let _ = disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await; - - let task_id = ResumeUtils::generate_task_id(); - let targets = vec!["replacement-a".to_string()]; - let first = ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: vec!["device-a".to_string()], - filesystem_identity: "1:2:3".to_string(), - }; - ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket-a".to_string()], - targets.clone(), - vec![first.clone()], - ) - .await - .unwrap(); - - let reused = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket-b".to_string()], - targets.clone(), - vec![first.clone()], - ) - .await - .unwrap(); - assert_eq!( - reused.get_state().await.replacement_buckets, - ["bucket-a"], - "retries must keep the first generation's bucket plan" - ); - - let mut second = ReplacementTargetIdentity { - endpoint: "replacement-a".to_string(), - canonical_path: "/mnt/replacement-a".to_string(), - physical_device_ids: first.physical_device_ids.clone(), - filesystem_identity: first.filesystem_identity.clone(), - }; - for changed_identity in [ - { - second.physical_device_ids = vec!["device-b".to_string()]; - second.clone() - }, - { - second.physical_device_ids = first.physical_device_ids.clone(); - second.filesystem_identity = "4:5:6".to_string(); - second.clone() - }, - { - second.filesystem_identity = first.filesystem_identity.clone(); - second.canonical_path = "/mnt/replacement-b".to_string(); - second.clone() - }, - ] { - let result = ResumeManager::new_replacement_intent( - disk.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket-a".to_string()], - targets.clone(), - vec![changed_identity], - ) - .await; - assert!(result.is_err(), "a new mounted instance must not reuse the old replacement cursor"); - } - temp_dir.close().unwrap(); - } - - #[test] - fn reset_for_retry_clears_progress_but_keeps_retry_budget() { - // backlog#855 / #799 B6: a retry must re-scan from the start without - // spending the retry budget's identity. - let buckets = vec!["bucket1".to_string(), "bucket2".to_string()]; - let mut state = ResumeState::new("t".to_string(), "erasure_set".to_string(), "pool_0_set_0".to_string(), buckets); - state.update_progress(10, 8, 2, 0); - state.complete_bucket("bucket1"); - state.increment_retry(); - state.mark_completed(); - - state.reset_for_retry(); - - assert!(!state.completed, "retry must un-complete the task"); - assert_eq!(state.completed_buckets.len(), 0, "all buckets must be re-scanned"); - assert_eq!(state.processed_objects, 0); - assert_eq!(state.successful_objects, 0); - assert_eq!(state.failed_objects, 0); - assert_eq!(state.skipped_objects, 0); - assert_eq!(state.retry_count, 1, "retry budget must be preserved"); - } - - #[test] - fn can_retry_is_bounded_by_max_retries() { - let mut state = ResumeState::new("t".to_string(), "erasure_set".to_string(), "pool_0_set_0".to_string(), vec![]); - assert!(state.can_retry()); - for _ in 0..state.max_retries { - assert!(state.can_retry()); - state.increment_retry(); - } - assert!(!state.can_retry(), "retries must stop after max_retries"); - } - - #[test] - fn checkpoint_reset_for_retry_rewinds_position_and_clears_sets() { - let mut checkpoint = ResumeCheckpoint::new("task".to_string()); - checkpoint.update_position(3, 42); - checkpoint.add_processed_object("bucket/a".to_string()); - checkpoint.add_failed_object("bucket/b".to_string()); - checkpoint.add_skipped_object("bucket/c".to_string()); - - checkpoint.reset_for_retry(); - - assert_eq!(checkpoint.current_bucket_index, 0); - assert_eq!(checkpoint.current_object_index, 0); - assert!(checkpoint.processed_objects.is_empty()); - assert!(checkpoint.failed_objects.is_empty()); - assert!(checkpoint.skipped_objects.is_empty()); - } - - #[tokio::test] - async fn test_resume_state_bucket_completion() { - let task_id = ResumeUtils::generate_task_id(); - let buckets = vec!["bucket1".to_string(), "bucket2".to_string()]; - let mut state = ResumeState::new(task_id, "erasure_set".to_string(), "pool_0_set_0".to_string(), buckets); - - assert_eq!(state.pending_buckets.len(), 2); - assert_eq!(state.completed_buckets.len(), 0); - - state.complete_bucket("bucket1"); - assert_eq!(state.pending_buckets.len(), 1); - assert_eq!(state.completed_buckets.len(), 1); - assert!(state.completed_buckets.contains(&"bucket1".to_string())); - } - - #[test] - fn test_checkpoint_object_sets_dedupe_and_prune() { - let mut checkpoint = ResumeCheckpoint::new("task".to_string()); - checkpoint.add_processed_object("bucket/a".to_string()); - checkpoint.add_processed_object("bucket/a".to_string()); - checkpoint.add_skipped_object("bucket/b".to_string()); - checkpoint.add_failed_object("bucket/c".to_string()); - assert_eq!(checkpoint.processed_objects.len(), 1); - assert!(checkpoint.processed_objects.contains("bucket/a")); - - checkpoint.complete_page(2, 2000); - assert_eq!(checkpoint.current_bucket_index, 2); - assert_eq!(checkpoint.current_object_index, 2000); - assert!(checkpoint.processed_objects.is_empty()); - assert!(checkpoint.skipped_objects.is_empty()); - assert!(checkpoint.failed_objects.is_empty()); - } - - #[test] - fn test_checkpoint_loads_legacy_vec_format() { - // Checkpoints written before the HashSet migration stored the object - // lists as JSON arrays (possibly with duplicates); they must still load. - let legacy = r#"{ - "task_id": "t1", - "checkpoint_time": 1700000000, - "current_bucket_index": 1, - "current_object_index": 42, - "processed_objects": ["a", "b", "a"], - "failed_objects": [], - "skipped_objects": ["c"] - }"#; - let checkpoint: ResumeCheckpoint = serde_json::from_str(legacy).unwrap(); - assert_eq!(checkpoint.current_object_index, 42); - assert_eq!(checkpoint.processed_objects.len(), 2); - assert!(checkpoint.processed_objects.contains("a")); - assert!(checkpoint.skipped_objects.contains("c")); - } - - #[test] - fn test_compose_key_injective_with_adversarial_keys() { - // Length-prefixing must keep the encoding injective even when keys - // contain the delimiter, embedded nulls, or look like a composed key. - assert_ne!(compose_key("a\0b", None), compose_key("a", Some("b"))); - assert_ne!(compose_key("3:xy", None), compose_key("x", Some("y"))); - assert_ne!(compose_key("a:b", None), compose_key("a", Some("b"))); - assert_ne!(compose_key("", Some("x")), compose_key("x", None)); - // Identical inputs must produce identical keys (stable identity). - assert_eq!(compose_key("obj", Some("v1")), compose_key("obj", Some("v1"))); - } - - #[test] - fn test_composite_key_dedup_distinguishes_versions() { - // Two versions of the same object must be distinct dedup identities, and - // the delete-marker/nil (None) version must not collide with a real one. - let mut checkpoint = ResumeCheckpoint::new("task".to_string()); - checkpoint.add_processed_object(compose_key("obj", Some("v1"))); - checkpoint.add_processed_object(compose_key("obj", Some("v2"))); - checkpoint.add_processed_object(compose_key("obj", None)); - assert_eq!(checkpoint.processed_objects.len(), 3); - assert!(checkpoint.processed_objects.contains(&compose_key("obj", Some("v1")))); - assert!(checkpoint.processed_objects.contains(&compose_key("obj", Some("v2")))); - assert!(checkpoint.processed_objects.contains(&compose_key("obj", None))); - // A different object with the same version id is still distinct. - assert!(!checkpoint.processed_objects.contains(&compose_key("other", Some("v1")))); - } - - #[tokio::test] - async fn test_resumestate_schema_v0_discarded_on_load() { - let (temp_dir, disk) = schema_test_disk().await; - - // Legacy snapshot: no schema_version, a stale positional cursor and progress. - let legacy = r#"{ - "task_id": "old-task", - "task_type": "erasure_set", - "set_disk_id": "pool_0_set_0", - "start_time": 1700000000, - "last_update": 1700000000, - "completed": true, - "total_objects": 100, - "processed_objects": 50, - "successful_objects": 40, - "failed_objects": 10, - "skipped_objects": 0, - "current_bucket": null, - "current_object": null, - "completed_buckets": ["b1"], - "pending_buckets": [], - "error_message": null, - "retry_count": 1, - "max_retries": 3, - "resume_cursor": "v1:stale-token" - }"#; - let task_id = "00000000-0000-4000-8000-000000000001"; - let legacy = legacy.replace("old-task", task_id); - let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); - disk.write_all(RUSTFS_META_BUCKET, &file_path, legacy.as_bytes().to_vec().into()) - .await - .expect("write legacy resume state"); - - let manager = ResumeManager::load_from_disk(disk.clone(), task_id).await.unwrap(); - let state = manager.get_state().await; - assert_eq!(state.schema_version, CURRENT_RESUME_SCHEMA, "schema must be stamped current"); - assert_eq!(state.resume_cursor, None, "stale cursor must be cleared"); - assert_eq!(state.processed_objects, 0); - assert_eq!(state.successful_objects, 0); - assert_eq!(state.failed_objects, 0); - assert!(!state.completed); - temp_dir.close().expect("remove schema test directory"); - } - - #[tokio::test] - async fn test_checkpoint_schema_v4_discarded_on_load() { - let (temp_dir, disk) = schema_test_disk().await; - - // The previous checkpoint schema is unsafe once its paired resume - // state is discarded: retaining either position would skip work. - let task_id = "00000000-0000-4000-8000-000000000002"; - let legacy = r#"{ - "schema_version": 4, - "task_id": "00000000-0000-4000-8000-000000000002", - "checkpoint_time": 1700000000, - "current_bucket_index": 2, - "current_object_index": 500, - "processed_objects": ["a", "b"], - "failed_objects": ["c"], - "skipped_objects": ["d"] - }"#; - let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); - disk.write_all(RUSTFS_META_BUCKET, &file_path, legacy.as_bytes().to_vec().into()) - .await - .expect("write legacy checkpoint"); - - let manager = CheckpointManager::load_from_disk(disk.clone(), task_id).await.unwrap(); - let checkpoint = manager.get_checkpoint().await; - assert_eq!(checkpoint.schema_version, CURRENT_CHECKPOINT_SCHEMA, "schema must be stamped current"); - assert_eq!(checkpoint.current_bucket_index, 0, "stale bucket position must be reset"); - assert_eq!(checkpoint.current_object_index, 0, "stale position must be reset"); - assert!(checkpoint.processed_objects.is_empty()); - assert!(checkpoint.failed_objects.is_empty()); - assert!(checkpoint.skipped_objects.is_empty()); - temp_dir.close().expect("remove schema test directory"); - } - - #[tokio::test] - async fn current_normal_resume_schema_preserves_progress() { - let (temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let mut state = ResumeState::new( - task_id.clone(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket-b".to_string()], - ); - state.resume_cursor = Some("opaque-marker".to_string()); - state.processed_objects = 7; - state.successful_objects = 6; - state.failed_objects = 1; - state.completed_buckets = vec!["bucket-a".to_string()]; - let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); - let state_data = serde_json::to_vec(&state).expect("serialize current normal resume state"); - disk.write_all(RUSTFS_META_BUCKET, &file_path, state_data.into()) - .await - .expect("write current normal resume state"); - - let restored = ResumeManager::load_from_disk(disk.clone(), &task_id) - .await - .expect("load current normal resume state") - .get_state() - .await; - - assert_eq!(restored.schema_version, CURRENT_RESUME_SCHEMA); - assert_eq!(restored.resume_cursor.as_deref(), Some("opaque-marker")); - assert_eq!(restored.processed_objects, 7); - assert_eq!(restored.successful_objects, 6); - assert_eq!(restored.failed_objects, 1); - assert_eq!(restored.completed_buckets, ["bucket-a"]); - assert!(restored.replacement_targets.is_empty()); - assert_eq!(restored.replacement_generation, None); - assert_eq!(restored.replacement_phase, ReplacementPhase::None); - temp_dir.close().expect("remove schema test directory"); - } - - #[tokio::test] - async fn future_resume_and_checkpoint_schemas_are_rejected() { - let (temp_dir, disk) = schema_test_disk().await; - let task_id = ResumeUtils::generate_task_id(); - let mut state = ResumeState::new(task_id.clone(), "erasure_set".to_string(), "pool_0_set_0".to_string(), Vec::new()); - state.schema_version = CURRENT_RESUME_SCHEMA + 1; - let state_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); - let state_data = serde_json::to_vec(&state).expect("serialize future resume state"); - disk.write_all(RUSTFS_META_BUCKET, &state_path, state_data.into()) - .await - .expect("write future resume state"); - - let resume_error = match ResumeManager::load_from_disk(disk.clone(), &task_id).await { - Ok(_) => panic!("future resume schema must not load"), - Err(error) => error, - }; - assert!(matches!(resume_error, Error::TaskExecutionFailed { .. })); - assert!(resume_error.to_string().contains("newer than supported schema")); - - let mut checkpoint = ResumeCheckpoint::new(task_id.clone()); - checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1; - let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); - let checkpoint_data = serde_json::to_vec(&checkpoint).expect("serialize future checkpoint"); - disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, checkpoint_data.into()) - .await - .expect("write future checkpoint"); - - let checkpoint_error = match CheckpointManager::load_from_disk(disk.clone(), &task_id).await { - Ok(_) => panic!("future checkpoint schema must not load"), - Err(error) => error, - }; - assert!(matches!(checkpoint_error, Error::TaskExecutionFailed { .. })); - assert!(checkpoint_error.to_string().contains("newer than supported schema")); - temp_dir.close().expect("remove schema test directory"); - } - - #[test] - fn test_persist_throttle_batches_until_threshold() { - let mut throttle = PersistThrottle::new(); - for _ in 0..PERSIST_EVERY_MUTATIONS - 1 { - assert!(!throttle.record(), "must not flush below the mutation threshold"); - } - assert!(throttle.record(), "must flush at the mutation threshold"); - throttle.mark_saved(); - assert!(!throttle.record(), "counter must reset after a save"); - } - - #[tokio::test] - async fn completion_persists_immediately_and_cleanup_propagates_delete_errors() { - use super::super::{DiskOption, Endpoint, new_disk}; - use tempfile::TempDir; - - let temp_dir = TempDir::new().expect("create resume persistence test directory"); - let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create test disk endpoint"); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .expect("create resume persistence test disk"); - match disk.make_volume(RUSTFS_META_BUCKET).await { - Ok(()) | Err(DiskError::VolumeExists) => {} - Err(err) => panic!("create metadata volume for resume persistence test: {err}"), - } - - let task_id = ResumeUtils::generate_task_id(); - let manager = ResumeManager::new( - disk.clone(), - task_id.clone(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - ) - .await - .expect("create resume manager"); - manager - .update_progress(1, 1, 0, 0) - .await - .expect("buffer progress below the persistence threshold"); - manager.mark_completed().await.expect("persist completed resume state"); - - let persisted = ResumeManager::load_from_disk(disk.clone(), &task_id) - .await - .expect("reload completed resume state") - .get_state() - .await; - assert!(persisted.completed, "completion must be persisted without waiting for the throttle"); - assert_eq!(persisted.processed_objects, 1, "the completion write must include buffered progress"); - - let state_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); - let failure = ResumeDeleteFailure::install(state_path, DiskError::DiskAccessDenied); - let error = manager - .cleanup() - .await - .expect_err("resume cleanup must propagate a real delete failure"); - assert!(matches!(error, Error::Disk(DiskError::DiskAccessDenied))); - drop(failure); - manager.cleanup().await.expect("resume cleanup must be retryable"); - manager - .cleanup() - .await - .expect("missing resume files must be idempotent success"); - - let checkpoint = CheckpointManager::new(disk.clone(), task_id.clone()) - .await - .expect("create checkpoint manager"); - let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); - let failure = ResumeDeleteFailure::install(checkpoint_path, DiskError::DiskAccessDenied); - let error = checkpoint - .cleanup() - .await - .expect_err("checkpoint cleanup must propagate a real delete failure"); - assert!(matches!(error, Error::Disk(DiskError::DiskAccessDenied))); - drop(failure); - checkpoint.cleanup().await.expect("checkpoint cleanup must be retryable"); - checkpoint - .cleanup() - .await - .expect("missing checkpoint must be idempotent success"); - } - - #[tokio::test] - async fn checkpoint_rejects_a_task_id_mismatched_to_its_file_name() { - use super::super::{DiskOption, Endpoint, new_disk}; - use tempfile::TempDir; - - let temp_dir = TempDir::new().expect("create checkpoint binding test directory"); - let endpoint = - Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create checkpoint binding test endpoint"); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .expect("create checkpoint binding test disk"); - match disk.make_volume(RUSTFS_META_BUCKET).await { - Ok(()) | Err(DiskError::VolumeExists) => {} - Err(err) => panic!("create checkpoint binding metadata volume: {err}"), - } - - let requested_task_id = ResumeUtils::generate_task_id(); - let checkpoint = ResumeCheckpoint::new(ResumeUtils::generate_task_id()); - let checkpoint_path = format!("{BUCKET_META_PREFIX}/{requested_task_id}_{RESUME_CHECKPOINT_FILE}"); - disk.write_all( - RUSTFS_META_BUCKET, - &checkpoint_path, - serde_json::to_vec(&checkpoint) - .expect("serialize mismatched checkpoint") - .into(), - ) - .await - .expect("persist mismatched checkpoint"); - - let error = match CheckpointManager::load_from_disk(disk, &requested_task_id).await { - Ok(_) => panic!("checkpoint task id must be bound to its file name"), - Err(error) => error, - }; - - assert!(error.to_string().contains("does not match")); - temp_dir.close().expect("remove checkpoint binding test directory"); - } - - #[tokio::test] - async fn test_resume_utils() { - let task_id1 = ResumeUtils::generate_task_id(); - let task_id2 = ResumeUtils::generate_task_id(); - - assert_ne!(task_id1, task_id2); - assert_eq!(task_id1.len(), 36); // UUID length - assert_eq!(task_id2.len(), 36); - assert!(validate_resume_task_id(&task_id1).is_ok()); - assert!(validate_resume_task_id(&format!("pool_0_set_0_{task_id1}")).is_err()); - assert!(validate_resume_task_id(&task_id1.to_uppercase()).is_err()); - } - - #[tokio::test] - async fn test_get_resumable_tasks_integration() { - use super::super::{DiskOption, Endpoint, new_disk}; - use tempfile::TempDir; - - // Create a temporary directory for testing - let temp_dir = TempDir::new().unwrap(); - let disk_path = temp_dir.path().join("test_disk"); - std::fs::create_dir_all(&disk_path).unwrap(); - - // Create a local disk for testing - let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).unwrap(); - let disk_option = DiskOption { - cleanup: false, - health_check: false, - }; - let disk = new_disk(&endpoint, &disk_option).await.unwrap(); - - // Create necessary directories first (ignore if already exist) - let _ = disk.make_volume(RUSTFS_META_BUCKET).await; - let _ = disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await; - - // Create some test resume state files - let task_ids = vec![ - ResumeUtils::generate_task_id(), - ResumeUtils::generate_task_id(), - ResumeUtils::generate_task_id(), - ]; - - // Save resume state files for each task - for task_id in &task_ids { - let state = ResumeState::new( - task_id.clone(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket1".to_string(), "bucket2".to_string()], - ); - - let state_data = serde_json::to_vec(&state).unwrap(); - let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); - - disk.write_all(RUSTFS_META_BUCKET, &file_path, state_data.into()) - .await - .unwrap(); - } - - // Also create some non-resume state files to test filtering - let non_resume_files = vec![ - "other_file.txt", - "task4_ahm_checkpoint.json", - "task5_ahm_progress.json", - "_ahm_resume_state.json", // Invalid: empty task ID - "not-a-uuid_ahm_resume_state.json", - "00000000-0000-4000-8000-000000000001_extra_ahm_resume_state.json", - ]; - - for file_name in non_resume_files { - let file_path = format!("{BUCKET_META_PREFIX}/{file_name}"); - disk.write_all(RUSTFS_META_BUCKET, &file_path, b"test data".to_vec().into()) - .await - .unwrap(); - } - - // Now call get_resumable_tasks to see if it finds the correct files - let found_task_ids = ResumeUtils::get_resumable_tasks(&disk).await.unwrap(); - - // Verify that only the valid resume state files are found - assert_eq!(found_task_ids.len(), 3); - for task_id in &task_ids { - assert!(found_task_ids.contains(task_id), "Task ID {task_id} not found"); - } - - // Verify that invalid files are not included - assert!(!found_task_ids.contains(&"".to_string())); - assert!(!found_task_ids.contains(&"task4".to_string())); - assert!(!found_task_ids.contains(&"task5".to_string())); - assert!(!found_task_ids.contains(&"not-a-uuid".to_string())); - - let error = match ResumeManager::load_from_disk(disk.clone(), "../not-a-uuid").await { - Ok(_) => panic!("a traversal-like task id must be rejected before reading metadata"), - Err(error) => error, - }; - assert!(matches!( - error, - Error::TaskExecutionFailed { message } if message == "Invalid resume task id" - )); - - // Clean up - temp_dir.close().unwrap(); - } - - #[tokio::test] - async fn resume_state_rejects_filename_and_json_task_id_mismatch() { - use super::super::{DiskOption, Endpoint, new_disk}; - use tempfile::TempDir; - - let temp_dir = TempDir::new().expect("create resume mismatch test directory"); - let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create test disk endpoint"); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .expect("create resume mismatch test disk"); - match disk.make_volume(RUSTFS_META_BUCKET).await { - Ok(()) | Err(DiskError::VolumeExists) => {} - Err(error) => panic!("create metadata volume for resume mismatch test: {error}"), - } - - let filename_task_id = ResumeUtils::generate_task_id(); - let state = ResumeState::new( - ResumeUtils::generate_task_id(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - ); - let path = format!("{BUCKET_META_PREFIX}/{filename_task_id}_{RESUME_STATE_FILE}"); - disk.write_all( - RUSTFS_META_BUCKET, - &path, - serde_json::to_vec(&state).expect("serialize mismatched resume state").into(), - ) - .await - .expect("write mismatched resume state"); - - let error = match ResumeManager::load_from_disk(disk.clone(), &filename_task_id).await { - Ok(_) => panic!("resume state task id must match its filename"), - Err(error) => error, - }; - assert!(matches!( - error, - Error::TaskExecutionFailed { message } if message == "Resume state task id does not match filename" - )); - - let checkpoint_filename_task_id = ResumeUtils::generate_task_id(); - let checkpoint = ResumeCheckpoint::new(ResumeUtils::generate_task_id()); - let checkpoint_path = format!("{BUCKET_META_PREFIX}/{checkpoint_filename_task_id}_{RESUME_CHECKPOINT_FILE}"); - disk.write_all( - RUSTFS_META_BUCKET, - &checkpoint_path, - serde_json::to_vec(&checkpoint) - .expect("serialize mismatched resume checkpoint") - .into(), - ) - .await - .expect("write mismatched resume checkpoint"); - - let error = match CheckpointManager::load_from_disk(disk, &checkpoint_filename_task_id).await { - Ok(_) => panic!("resume checkpoint task id must match its filename"), - Err(error) => error, - }; - assert!(matches!( - error, - Error::TaskExecutionFailed { message } if message == "Resume checkpoint task id does not match filename" - )); - } -} +mod tests; diff --git a/crates/heal/src/heal/resume/checkpoint.rs b/crates/heal/src/heal/resume/checkpoint.rs new file mode 100644 index 000000000..1b4b7ece3 --- /dev/null +++ b/crates/heal/src/heal/resume/checkpoint.rs @@ -0,0 +1,351 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{Error, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::sync::RwLock; +use tracing::{debug, warn}; + +use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET}; +use super::{ + LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_FILE, delete_resume_file, path_to_str, + validate_resume_task_id, +}; + +const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state"; + +/// Current on-disk schema version for `ResumeCheckpoint`. Same rationale as +/// `CURRENT_RESUME_SCHEMA`: pre-per-version dedup identities are not comparable +/// to the new `compose_key` identities, so a stale checkpoint is discarded. +pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 5; + +/// resume checkpoint +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResumeCheckpoint { + /// on-disk schema version; absent in legacy snapshots (defaults to 0) + #[serde(default)] + pub schema_version: u32, + /// task id + pub task_id: String, + /// checkpoint time + pub checkpoint_time: u64, + /// current bucket index + pub current_bucket_index: usize, + /// current object index + pub current_object_index: usize, + /// Objects healed since the last completed page. HashSet: with the + /// previous Vec the per-object `contains` was O(n) and made large-bucket + /// heals O(N²). Only spans the in-flight page — completed pages are + /// covered by `current_object_index`, so `complete_page` prunes the sets. + pub processed_objects: HashSet, + /// failed objects + pub failed_objects: HashSet, + /// skipped objects + pub skipped_objects: HashSet, +} + +impl ResumeCheckpoint { + pub fn new(task_id: String) -> Self { + Self { + schema_version: CURRENT_CHECKPOINT_SCHEMA, + task_id, + checkpoint_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(), + current_bucket_index: 0, + current_object_index: 0, + processed_objects: HashSet::new(), + failed_objects: HashSet::new(), + skipped_objects: HashSet::new(), + } + } + + pub fn update_position(&mut self, bucket_index: usize, object_index: usize) { + self.current_bucket_index = bucket_index; + self.current_object_index = object_index; + self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + } + + pub fn add_processed_object(&mut self, object: String) { + self.processed_objects.insert(object); + } + + pub fn add_failed_object(&mut self, object: String) { + self.failed_objects.insert(object); + } + + pub fn add_skipped_object(&mut self, object: String) { + self.skipped_objects.insert(object); + } + + /// Advance past a fully-processed page: objects below `object_index` are + /// skipped by position on resume, so the per-object sets no longer need + /// their entries and would otherwise grow with the whole bucket. + pub fn complete_page(&mut self, bucket_index: usize, object_index: usize) { + self.update_position(bucket_index, object_index); + self.processed_objects.clear(); + self.skipped_objects.clear(); + self.failed_objects.clear(); + } + + /// Reset the scan to the start and clear the per-object sets so a retry + /// re-scans the whole set. + pub fn reset_for_retry(&mut self) { + self.update_position(0, 0); + self.processed_objects.clear(); + self.skipped_objects.clear(); + self.failed_objects.clear(); + } +} + +/// resume checkpoint manager +pub struct CheckpointManager { + disk: DiskStore, + checkpoint: Arc>, + throttle: Mutex, +} + +impl CheckpointManager { + /// create new checkpoint manager + pub async fn new(disk: DiskStore, task_id: String) -> Result { + validate_resume_task_id(&task_id)?; + let checkpoint = ResumeCheckpoint::new(task_id); + let manager = Self { + disk, + checkpoint: Arc::new(RwLock::new(checkpoint)), + throttle: Mutex::new(PersistThrottle::new()), + }; + + // save initial checkpoint + if let Err(e) = manager.save_checkpoint().await { + warn!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_CHECKPOINT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + state = "initial_save_failed", + error = %e, + "Heal checkpoint persistence failed" + ); + } + Ok(manager) + } + + /// load checkpoint from disk + pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result { + validate_resume_task_id(task_id)?; + let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?; + let mut checkpoint: ResumeCheckpoint = + serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to deserialize checkpoint: {e}"), + })?; + if checkpoint.task_id != task_id { + return Err(Error::TaskExecutionFailed { + message: "Resume checkpoint task id does not match filename".to_string(), + }); + } + + // A checkpoint from an older schema stored latest-only dedup identities + // that are not comparable to the new per-version `compose_key` + // identities. Discard the stale sets and position, then stamp the + // current schema so the scan restarts cleanly. + if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA { + return Err(Error::TaskExecutionFailed { + message: format!( + "Checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}", + checkpoint.schema_version + ), + }); + } + if checkpoint.schema_version < CURRENT_CHECKPOINT_SCHEMA { + warn!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_CHECKPOINT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + task_id, + found_schema = checkpoint.schema_version, + current_schema = CURRENT_CHECKPOINT_SCHEMA, + state = "schema_discarded", + "Heal checkpoint schema is stale; discarding dedup sets and position" + ); + checkpoint.processed_objects.clear(); + checkpoint.failed_objects.clear(); + checkpoint.skipped_objects.clear(); + checkpoint.current_bucket_index = 0; + checkpoint.current_object_index = 0; + checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA; + } + + Ok(Self { + disk, + checkpoint: Arc::new(RwLock::new(checkpoint)), + throttle: Mutex::new(PersistThrottle::new()), + }) + } + + /// check if checkpoint exists + pub async fn has_checkpoint(disk: &DiskStore, task_id: &str) -> bool { + if validate_resume_task_id(task_id).is_err() { + return false; + } + let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); + match path_to_str(&file_path) { + Ok(path_str) => match disk.read_all(RUSTFS_META_BUCKET, path_str).await { + Ok(data) => !data.is_empty(), + Err(_) => false, + }, + Err(_) => false, + } + } + + /// get current checkpoint + pub async fn get_checkpoint(&self) -> ResumeCheckpoint { + self.checkpoint.read().await.clone() + } + + /// update position + pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.update_position(bucket_index, object_index); + drop(checkpoint); + self.save_checkpoint_throttled().await + } + + /// Advance past a completed page and prune the per-object sets, then persist. + pub async fn complete_page(&self, bucket_index: usize, object_index: usize) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.complete_page(bucket_index, object_index); + drop(checkpoint); + self.save_checkpoint_throttled().await + } + + /// Reset the checkpoint to the start of the scan for a retry, then persist. + pub async fn reset_for_retry(&self) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.reset_for_retry(); + drop(checkpoint); + self.save_checkpoint_throttled().await + } + + /// Add a processed object. Called once per healed object, so persistence + /// is batched (`PERSIST_EVERY_MUTATIONS` / `PERSIST_INTERVAL`); positions + /// and page boundaries still persist unconditionally. + pub async fn add_processed_object(&self, object: String) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.add_processed_object(object); + drop(checkpoint); + self.save_checkpoint_if_due().await + } + + /// add failed object (batched, see `add_processed_object`) + pub async fn add_failed_object(&self, object: String) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.add_failed_object(object); + drop(checkpoint); + self.save_checkpoint_if_due().await + } + + /// add skipped object (batched, see `add_processed_object`) + pub async fn add_skipped_object(&self, object: String) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.add_skipped_object(object); + drop(checkpoint); + self.save_checkpoint_if_due().await + } + + async fn save_checkpoint_if_due(&self) -> Result<()> { + let should_save = self.throttle.lock().map(|mut throttle| throttle.record()).unwrap_or(true); + if !should_save { + return Ok(()); + } + self.save_checkpoint_throttled().await + } + + async fn save_checkpoint_throttled(&self) -> Result<()> { + let result = self.save_checkpoint().await; + if result.is_ok() + && let Ok(mut throttle) = self.throttle.lock() + { + throttle.mark_saved(); + } + result + } + + /// cleanup checkpoint + pub async fn cleanup(&self) -> Result<()> { + let task_id = self.checkpoint.read().await.task_id.clone(); + validate_resume_task_id(&task_id)?; + + let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); + delete_resume_file(&self.disk, &checkpoint_file).await?; + + debug!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_CHECKPOINT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + task_id, + state = "cleaned", + "Heal checkpoint cleaned" + ); + Ok(()) + } + + /// save checkpoint to disk + async fn save_checkpoint(&self) -> Result<()> { + let checkpoint = self.checkpoint.read().await; + validate_resume_task_id(&checkpoint.task_id)?; + let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to serialize checkpoint: {e}"), + })?; + + let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", checkpoint.task_id, RESUME_CHECKPOINT_FILE)); + + let path_str = path_to_str(&file_path)?; + self.disk + .write_all(RUSTFS_META_BUCKET, path_str, checkpoint_data.into()) + .await + .map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to save checkpoint: {e}"), + })?; + + debug!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_CHECKPOINT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + task_id = %checkpoint.task_id, + state = "saved", + "Heal checkpoint persisted" + ); + Ok(()) + } + + /// read checkpoint file from disk + async fn read_checkpoint_file(disk: &DiskStore, task_id: &str) -> Result> { + validate_resume_task_id(task_id)?; + let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); + + let path_str = path_to_str(&file_path)?; + disk.read_all(RUSTFS_META_BUCKET, path_str) + .await + .map(|bytes| bytes.to_vec()) + .map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to read checkpoint file: {e}"), + }) + } +} diff --git a/crates/heal/src/heal/resume/replacement.rs b/crates/heal/src/heal/resume/replacement.rs new file mode 100644 index 000000000..c9d1ebc5e --- /dev/null +++ b/crates/heal/src/heal/resume/replacement.rs @@ -0,0 +1,688 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{Error, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::time::{SystemTime, UNIX_EPOCH}; + +use super::super::HealDiskExt as _; + +use super::super::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskBytes}; +use super::{ + DiskError, DiskStore, RUSTFS_META_BUCKET, ResumeManager, ResumeState, delete_resume_file, ensure_replacement_recovery_dir, + injected_replacement_proof_write_error, is_replacement_intent, legacy_replacement_completion_proof_path, path_to_str, + replacement_completion_proof_path, replacement_intent_seal_path, replacement_recovery_conflict, + replacement_recovery_corruption, validate_resume_task_id, +}; + +/// Durable-proof schema version. +const CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA: u32 = 1; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReplacementPhase { + #[default] + None, + Intent, + Rebuilding, + Verified, + CleanupPending, + Abandoned, +} + +/// Target-specific state for a durable automatic replacement generation. +/// +/// This is deliberately separate from the legacy background-heal status +/// contract. Consumers must treat [`Self::Unknown`] as non-definitive. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReplacementRecoveryState { + WaitingForReplacement, + Running, + Incomplete, + Unrecoverable, + CleanupPending, + Completed, + Unknown, +} + +/// Read-only status derived from one durable replacement generation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReplacementRecoveryRecord { + pub task_id: String, + pub state: ReplacementRecoveryState, + pub generation: Option, + pub set_disk_id: Option, + pub target_slots: Vec, + pub reason: Option, + pub verified_at: Option, +} + +impl ReplacementRecoveryRecord { + pub(super) fn from_state(state: ResumeState) -> Option { + if !is_replacement_intent(&state) { + return None; + } + + let invariant_holds = state.replacement_generation.as_deref() == Some(state.task_id.as_str()) + && replacement_targets_match_identities(&state.replacement_targets, &state.replacement_target_identities); + if !invariant_holds { + return Some(Self::unknown( + state.task_id, + "durable replacement state violates its generation or target identity binding", + )); + } + + let (state_kind, reason) = if !state.completed && state.retry_count >= state.max_retries { + ( + ReplacementRecoveryState::Unrecoverable, + Some("replacement retry budget exhausted".to_string()), + ) + } else if let Some(reason) = state.error_message.clone() { + (ReplacementRecoveryState::Incomplete, Some(reason)) + } else { + match state.replacement_phase { + ReplacementPhase::Intent => (ReplacementRecoveryState::WaitingForReplacement, None), + ReplacementPhase::Rebuilding => (ReplacementRecoveryState::Running, None), + ReplacementPhase::Verified | ReplacementPhase::CleanupPending => (ReplacementRecoveryState::CleanupPending, None), + ReplacementPhase::Abandoned => ( + ReplacementRecoveryState::Unrecoverable, + Some("replacement generation was abandoned".to_string()), + ), + ReplacementPhase::None => (ReplacementRecoveryState::Unknown, Some("replacement phase is missing".to_string())), + } + }; + + Some(Self { + task_id: state.task_id, + state: state_kind, + generation: state.replacement_generation, + set_disk_id: Some(state.set_disk_id), + target_slots: state.replacement_targets, + reason, + verified_at: None, + }) + } + + pub(super) fn from_completion_proof(proof: &ReplacementCompletionProof) -> Self { + Self { + task_id: proof.task_id.clone(), + state: ReplacementRecoveryState::Completed, + generation: Some(proof.replacement_generation.clone()), + set_disk_id: Some(proof.set_disk_id.clone()), + target_slots: proof.replacement_targets.clone(), + reason: None, + verified_at: Some(proof.verified_at), + } + } + + pub(super) fn unknown(task_id: String, reason: &str) -> Self { + Self { + task_id, + state: ReplacementRecoveryState::Unknown, + generation: None, + set_disk_id: None, + target_slots: Vec::new(), + reason: Some(reason.to_string()), + verified_at: None, + } + } +} + +pub(super) fn replacement_targets_match_identities(targets: &[String], identities: &[ReplacementTargetIdentity]) -> bool { + !targets.is_empty() + && targets.len() == identities.len() + && targets.iter().collect::>().len() == targets.len() + && identities.iter().map(|identity| &identity.endpoint).eq(targets.iter()) +} + +/// Stable evidence for the mounted replacement instance that owns a repair +/// generation. Endpoint text alone is not sufficient because a later disk can +/// be mounted at the same configured path. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ReplacementTargetIdentity { + pub endpoint: String, + pub canonical_path: String, + pub physical_device_ids: Vec, + pub filesystem_identity: String, +} + +/// Durable terminal evidence for one automatic replacement generation. This +/// lives on the healthy non-target anchor rather than in the resumable state, +/// because resume cleanup must not erase proof that the generation completed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct ReplacementCompletionProof { + pub schema_version: u32, + pub task_id: String, + pub replacement_generation: String, + pub set_disk_id: String, + pub replacement_targets: Vec, + pub replacement_target_identities: Vec, + pub verified_at: u64, +} + +impl ReplacementCompletionProof { + pub(super) fn from_state(state: &ResumeState, verified_at: u64) -> Result { + let replacement_generation = state + .replacement_generation + .clone() + .ok_or_else(|| Error::TaskExecutionFailed { + message: format!("Replacement completion has no generation for task {}", state.task_id), + })?; + if replacement_generation != state.task_id + || state.replacement_targets.is_empty() + || state + .replacement_target_identities + .iter() + .map(|identity| &identity.endpoint) + .collect::>() + != state.replacement_targets.iter().collect::>() + { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement completion identity does not match task {}", state.task_id), + }); + } + + Ok(Self { + schema_version: CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA, + task_id: state.task_id.clone(), + replacement_generation, + set_disk_id: state.set_disk_id.clone(), + replacement_targets: state.replacement_targets.clone(), + replacement_target_identities: state.replacement_target_identities.clone(), + verified_at, + }) + } + + fn matches_state(&self, state: &ResumeState) -> bool { + self.schema_version == CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA + && self.task_id == state.task_id + && state.replacement_generation.as_deref() == Some(self.replacement_generation.as_str()) + && self.set_disk_id == state.set_disk_id + && self.replacement_targets == state.replacement_targets + && self.replacement_target_identities == state.replacement_target_identities + } + + fn validate(&self, expected_task_id: &str) -> Result<()> { + if self.schema_version != CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement completion proof schema {} is unsupported", self.schema_version), + }); + } + validate_resume_task_id(expected_task_id)?; + if self.task_id != expected_task_id + || self.replacement_generation != self.task_id + || self.set_disk_id.is_empty() + || self.verified_at == 0 + || !replacement_targets_match_identities(&self.replacement_targets, &self.replacement_target_identities) + { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement completion proof does not match task {expected_task_id}"), + }); + } + Ok(()) + } +} + +pub(crate) fn replacement_target_identities_match( + expected: &[ReplacementTargetIdentity], + actual: &[ReplacementTargetIdentity], +) -> bool { + let mut expected = expected.to_vec(); + let mut actual = actual.to_vec(); + expected.sort_by(|left, right| left.endpoint.cmp(&right.endpoint)); + actual.sort_by(|left, right| left.endpoint.cmp(&right.endpoint)); + expected == actual +} + +/// Build the canonical, provably-injective dedup identity for an object +/// version. Length-prefixing the object key makes the encoding injective: no +/// two distinct `(object, version_id)` pairs can collide, even for adversarial +/// keys containing `:` or embedded null bytes. This is the single source of +/// truth for per-version dedup across the heal loop and the checkpoint sets. +pub fn compose_key(object: &str, version_id: Option<&str>) -> String { + format!("{}:{}{}", object.len(), object, version_id.unwrap_or("")) +} + +impl ResumeManager { + /// Seal a durably published intent before the caller may format a target. + /// A torn intent without this seal is known to have failed before its + /// creator returned and can be atomically recreated on retry. + pub(super) async fn ensure_replacement_intent_seal(&self) -> Result<()> { + let task_id = self.state.read().await.task_id.clone(); + validate_resume_task_id(&task_id)?; + let path = replacement_intent_seal_path(&task_id); + let path = path_to_str(&path)?; + match self.disk.read_all(RUSTFS_META_BUCKET, path).await { + Ok(_) => return Ok(()), + Err(DiskError::FileNotFound) => {} + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to read replacement intent seal: {error}"), + }); + } + } + self.disk + .write_all(RUSTFS_META_BUCKET, path, b"sealed".as_slice().into()) + .await + .map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to save replacement intent seal: {error}"), + }) + } + + pub async fn mark_replacement_rebuilding( + &self, + mut replacement_target_identities: Vec, + ) -> Result<()> { + replacement_target_identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint)); + replacement_target_identities.dedup_by(|left, right| left.endpoint == right.endpoint); + let mut state = self.state.write().await; + if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement intent is not active for task {}", state.task_id), + }); + } + if replacement_target_identities + .iter() + .map(|identity| &identity.endpoint) + .collect::>() + != state.replacement_targets.iter().collect::>() + { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement identities do not match targets for task {}", state.task_id), + }); + } + if !replacement_target_identities_match(&state.replacement_target_identities, &replacement_target_identities) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement target changed after format for task {}", state.task_id), + }); + } + state.replacement_phase = ReplacementPhase::Rebuilding; + state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + drop(state); + self.save_state_strict().await + } + + /// Persist survivor-anchor completion proof before transitioning this + /// resumable state to `Verified`. If proof persistence fails, this state + /// stays rebuildable and the caller must retain the healing marker. + pub async fn mark_replacement_completed_and_verified(&self) -> Result<()> { + let state = self.state.read().await.clone(); + if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement verification is not active for task {}", state.task_id), + }); + } + let proof = self.write_replacement_completion_proof(&state, None).await?; + + let mut state = self.state.write().await; + if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement verification changed for task {}", state.task_id), + }); + } + state.mark_completed(); + state.replacement_phase = ReplacementPhase::Verified; + state.last_update = proof.verified_at; + drop(state); + self.save_state_strict().await + } + + /// Verify or backfill the terminal proof before marker removal or resume + /// cleanup. This supports restart recovery from a `Verified` state written + /// by a prior binary that did not yet have a separate proof record. + pub(crate) async fn ensure_replacement_completion_proof(&self) -> Result { + let state = self.state.read().await.clone(); + if !state.completed || !matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement completion is not verified for task {}", state.task_id), + }); + } + self.write_replacement_completion_proof(&state, Some(state.last_update)).await + } + + /// Record that the healing markers have been removed, so a later retry can + /// safely delete the remaining resume artifacts without touching markers. + pub async fn mark_replacement_cleanup_pending(&self) -> Result<()> { + let mut state = self.state.write().await; + if !state.completed || !matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement cleanup is not ready for task {}", state.task_id), + }); + } + state.replacement_phase = ReplacementPhase::CleanupPending; + state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + drop(state); + self.save_state_strict().await + } + + /// Load the durable terminal proof from the healthy survivor anchor. + pub(crate) async fn load_replacement_completion_proof(disk: DiskStore, task_id: &str) -> Result { + Self::replacement_completion_proof_if_present(disk, task_id) + .await? + .ok_or_else(|| Error::TaskExecutionFailed { + message: format!("Failed to read replacement completion proof: proof is missing for task {task_id}"), + }) + } + + async fn replacement_completion_proof_if_present( + disk: DiskStore, + task_id: &str, + ) -> Result> { + validate_resume_task_id(task_id)?; + let mut proofs = Vec::new(); + for path in [ + replacement_completion_proof_path(task_id), + legacy_replacement_completion_proof_path(task_id), + ] { + let path_str = path_to_str(&path)?; + let bytes = match disk.read_all(RUSTFS_META_BUCKET, path_str).await { + Ok(bytes) => bytes, + Err(DiskError::FileNotFound) => continue, + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to read replacement completion proof: {error}"), + }); + } + }; + let proof: ReplacementCompletionProof = + serde_json::from_slice(&bytes).map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to deserialize replacement completion proof: {error}"), + })?; + proof.validate(task_id)?; + proofs.push(proof); + } + + match proofs.as_slice() { + [] => Ok(None), + [proof] => Ok(Some(proof.clone())), + [proof, legacy_proof] if proof == legacy_proof => Ok(Some(proof.clone())), + _ => Err(replacement_recovery_conflict(format!( + "Replacement completion proof conflicts with legacy proof for task {task_id}" + ))), + } + } + + /// Reconcile the proof-first publication order after a crash. A matching + /// proof is durable evidence that rebuilding finished, so it must win over + /// an older active state before a retry may format the target again. + pub(super) async fn reconcile_replacement_completion_proof(&self) -> Result<()> { + let task_id = self.state.read().await.task_id.clone(); + let Some(proof) = Self::replacement_completion_proof_if_present(self.disk.clone(), &task_id).await? else { + return Ok(()); + }; + + let mut state = self.state.write().await; + if !proof.matches_state(&state) { + return Err(Error::TaskExecutionFailed { + message: format!("Replacement completion proof does not match active intent for task {}", state.task_id), + }); + } + if state.completed && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) { + return Ok(()); + } + if state.completed || !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) { + return Err(replacement_recovery_conflict(format!( + "Replacement completion proof conflicts with state for task {}", + state.task_id + ))); + } + + state.mark_completed(); + state.replacement_phase = ReplacementPhase::Verified; + state.last_update = proof.verified_at; + drop(state); + self.save_state_strict().await + } + + pub(super) async fn migrate_legacy_replacement_completion_proof(disk: &DiskStore, task_id: &str) -> Result { + validate_resume_task_id(task_id)?; + let legacy_path = legacy_replacement_completion_proof_path(task_id); + let legacy_path_str = path_to_str(&legacy_path)?; + let legacy_bytes = match disk.read_all(RUSTFS_META_BUCKET, legacy_path_str).await { + Ok(bytes) => bytes, + Err(DiskError::FileNotFound) => return Ok(false), + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to read legacy replacement completion proof: {error}"), + }); + } + }; + let legacy_proof: ReplacementCompletionProof = serde_json::from_slice(&legacy_bytes).map_err(|error| { + replacement_recovery_corruption(format!("Failed to deserialize legacy replacement completion proof: {error}")) + })?; + legacy_proof + .validate(task_id) + .map_err(|error| replacement_recovery_corruption(format!("Invalid legacy replacement completion proof: {error}")))?; + + ensure_replacement_recovery_dir(disk) + .await + .map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to create replacement recovery directory: {error}"), + })?; + let path = replacement_completion_proof_path(task_id); + let path_str = path_to_str(&path)?; + for _ in 0..2 { + match disk.read_all(RUSTFS_META_BUCKET, path_str).await { + Ok(bytes) => { + let proof: ReplacementCompletionProof = + serde_json::from_slice(&bytes).map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to deserialize replacement completion proof: {error}"), + })?; + proof.validate(task_id).map_err(|error| { + replacement_recovery_corruption(format!("Invalid replacement completion proof: {error}")) + })?; + if proof != legacy_proof { + return Err(replacement_recovery_conflict(format!( + "Replacement completion proof conflicts with legacy proof for task {task_id}" + ))); + } + delete_resume_file(disk, &legacy_path).await?; + return Ok(true); + } + Err(DiskError::FileNotFound) => {} + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to read replacement completion proof: {error}"), + }); + } + } + + match super::super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + path_str, + None, + Some(legacy_bytes.clone()), + ) + .await + { + Ok(EcstoreConditionalFileUpdate::Updated) => { + delete_resume_file(disk, &legacy_path).await?; + return Ok(true); + } + Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue, + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to migrate replacement completion proof: {error}"), + }); + } + } + } + + Err(Error::TaskExecutionFailed { + message: format!("Replacement completion proof changed while migrating task {task_id}"), + }) + } + + pub async fn abandon_replacement_intent(&self) -> Result<()> { + let mut state = self.state.write().await; + if matches!(state.replacement_phase, ReplacementPhase::Abandoned) { + return Ok(()); + } + state.replacement_phase = ReplacementPhase::Abandoned; + state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + drop(state); + self.save_state_strict().await + } + + pub async fn set_replacement_targets(&self, replacement_targets: Vec) -> Result<()> { + { + let mut state = self.state.write().await; + state.replacement_targets = replacement_targets; + } + self.save_state().await + } + + pub(super) async fn publish_new_replacement_intent(&self, expected: Option) -> Result<()> { + let state = self.state.read().await.clone(); + validate_resume_task_id(&state.task_id)?; + let state_data = EcstoreDiskBytes::from(serde_json::to_vec(&state).map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to serialize resume state: {error}"), + })?); + let path = self.state_file.path(&state.task_id); + let path = path_to_str(&path)?; + + ensure_replacement_recovery_dir(&self.disk) + .await + .map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to create replacement recovery directory: {error}"), + })?; + match super::super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file( + self.disk.as_ref(), + RUSTFS_META_BUCKET, + path, + expected, + Some(state_data), + ) + .await + { + Ok(EcstoreConditionalFileUpdate::Updated) => Ok(()), + Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => { + Err(Error::TaskExecutionFailed { + message: format!("Replacement intent changed before publication for task {}", state.task_id), + }) + } + Err(error) => Err(Error::TaskExecutionFailed { + message: format!("Failed to save resume state: {error}"), + }), + } + } + + async fn write_replacement_completion_proof( + &self, + state: &ResumeState, + verified_at: Option, + ) -> Result { + ensure_replacement_recovery_dir(&self.disk) + .await + .map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to create replacement recovery directory: {error}"), + })?; + let path = replacement_completion_proof_path(&state.task_id); + let path_str = path_to_str(&path)?; + let proof = ReplacementCompletionProof::from_state( + state, + verified_at.unwrap_or_else(|| SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs()), + )?; + let proof_data = EcstoreDiskBytes::from(serde_json::to_vec(&proof).map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to serialize replacement completion proof: {e}"), + })?); + if let Some(error) = injected_replacement_proof_write_error(path_str) { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to save replacement completion proof: {error}"), + }); + } + + // Publish through the disk CAS primitive: `write_all` can expose a + // partially written proof to a crash/restart reader. If a prior + // version left torn bytes behind, replace exactly the observed bytes; + // a concurrently published valid proof is never overwritten. + for _ in 0..2 { + let expected = match self.disk.read_all(RUSTFS_META_BUCKET, path_str).await { + Ok(existing) => match serde_json::from_slice::(&existing) { + Ok(existing_proof) => { + existing_proof.validate(&state.task_id)?; + if existing_proof.matches_state(state) { + return Ok(existing_proof); + } + return Err(Error::TaskExecutionFailed { + message: format!("Replacement completion proof does not match task {}", state.task_id), + }); + } + Err(_) => Some(existing), + }, + Err(DiskError::FileNotFound) => None, + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to read replacement completion proof: {error}"), + }); + } + }; + + match super::super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file( + self.disk.as_ref(), + RUSTFS_META_BUCKET, + path_str, + expected, + Some(proof_data.clone()), + ) + .await + { + Ok(EcstoreConditionalFileUpdate::Updated) => return Ok(proof), + Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue, + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to save replacement completion proof: {error}"), + }); + } + } + } + + Err(Error::TaskExecutionFailed { + message: format!("Replacement completion proof changed while publishing task {}", state.task_id), + }) + } + + pub(super) async fn write_replacement_intent_state( + &self, + path: &str, + state_data: EcstoreDiskBytes, + ) -> std::result::Result<(), DiskError> { + ensure_replacement_recovery_dir(&self.disk).await?; + for _ in 0..2 { + let expected = match self.disk.read_all(RUSTFS_META_BUCKET, path).await { + Ok(existing) => Some(existing), + Err(DiskError::FileNotFound) => None, + Err(error) => return Err(error), + }; + match super::super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file( + self.disk.as_ref(), + RUSTFS_META_BUCKET, + path, + expected, + Some(state_data.clone()), + ) + .await + { + Ok(EcstoreConditionalFileUpdate::Updated) => return Ok(()), + Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue, + Err(error) => return Err(error), + } + } + Err(DiskError::other("replacement intent changed while publishing")) + } +} diff --git a/crates/heal/src/heal/resume/tests.rs b/crates/heal/src/heal/resume/tests.rs new file mode 100644 index 000000000..8d8e35cfa --- /dev/null +++ b/crates/heal/src/heal/resume/tests.rs @@ -0,0 +1,1981 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::checkpoint::CURRENT_CHECKPOINT_SCHEMA; +use super::replacement::ReplacementCompletionProof; +use super::*; + +async fn schema_test_disk() -> (tempfile::TempDir, DiskStore) { + use super::super::{DiskOption, Endpoint, new_disk}; + + let temp_dir = tempfile::TempDir::new().expect("create schema test directory"); + let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create schema test disk endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("create schema test disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(error) => panic!("create metadata volume: {error}"), + } + match disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(error) => panic!("create resume metadata volume: {error}"), + } + + (temp_dir, disk) +} + +#[tokio::test] +async fn test_resume_state_creation() { + let task_id = ResumeUtils::generate_task_id(); + let buckets = vec!["bucket1".to_string(), "bucket2".to_string()]; + let state = ResumeState::new(task_id.clone(), "erasure_set".to_string(), "pool_0_set_0".to_string(), buckets); + + assert_eq!(state.task_id, task_id); + assert_eq!(state.task_type, "erasure_set"); + assert!(!state.completed); + assert_eq!(state.processed_objects, 0); + assert_eq!(state.pending_buckets.len(), 2); +} + +#[test] +fn replacement_intent_binds_a_generation_before_format() { + let state = ResumeState::replacement_intent( + "generation-a".to_string(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket-a".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ); + + assert_eq!(state.replacement_generation.as_deref(), Some("generation-a")); + assert_eq!(state.replacement_phase, ReplacementPhase::Intent); + assert_eq!(state.replacement_targets, ["replacement-a"]); + assert!(state.resume_cursor.is_none(), "a new replacement must start from the beginning"); + + let mut state = state; + state.complete_bucket("bucket-a"); + assert_eq!( + state.replacement_buckets, + ["bucket-a"], + "recovery must retain the original positional bucket plan" + ); +} + +#[tokio::test] +async fn replacement_terminal_phases_are_durable() { + use super::super::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + + let temp_dir = TempDir::new().expect("create replacement phase test directory"); + let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create test disk endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("create test disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(err) => panic!("create metadata volume for replacement phase test: {err}"), + } + + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + manager + .mark_replacement_completed_and_verified() + .await + .expect("completion and verified phase must persist together"); + + let verified = ResumeManager::load_replacement_intent(disk.clone(), &task_id) + .await + .expect("verified phase must survive a restart") + .get_state() + .await; + assert!(verified.completed); + assert_eq!(verified.replacement_phase, ReplacementPhase::Verified); + + let resumed = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["ignored-after-restart".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("verified replacement must be reopenable for terminal cleanup"); + assert_eq!( + resumed.get_state().await.replacement_buckets, + ["bucket"], + "terminal recovery must preserve the original generation bucket plan" + ); + + manager + .mark_replacement_cleanup_pending() + .await + .expect("cleanup-pending phase must persist after marker removal"); + let cleanup_pending = ResumeManager::load_replacement_intent(disk, &task_id) + .await + .expect("cleanup-pending phase must survive a restart") + .get_state() + .await; + assert!(cleanup_pending.completed); + assert_eq!(cleanup_pending.replacement_phase, ReplacementPhase::CleanupPending); +} + +#[tokio::test] +async fn replacement_proof_before_verified_state_is_reconciled_after_restart() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + + let proof = ReplacementCompletionProof::from_state(&manager.get_state().await, 42) + .expect("active replacement state should build a completion proof"); + let proof_path = replacement_completion_proof_path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + proof_path.to_str().expect("completion proof path must be UTF-8"), + serde_json::to_vec(&proof) + .expect("completion proof fixture should serialize") + .into(), + ) + .await + .expect("proof-first crash fixture should persist"); + + let recovered = ResumeManager::load_replacement_intent(disk.clone(), &task_id) + .await + .expect("matching completion proof must prevent another rebuild") + .get_state() + .await; + assert!(recovered.completed); + assert_eq!(recovered.replacement_phase, ReplacementPhase::Verified); + assert_eq!(recovered.last_update, proof.verified_at); + + let records = ResumeUtils::get_replacement_recovery_records(&disk) + .await + .expect("reconciled replacement state should be observable"); + assert_eq!(records.len(), 1); + assert_eq!(records[0].state, ReplacementRecoveryState::CleanupPending); +} + +#[tokio::test] +async fn replacement_proof_conflicting_with_active_state_fails_closed_after_restart() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + + let mut proof = ReplacementCompletionProof::from_state(&manager.get_state().await, 42) + .expect("active replacement state should build a completion proof"); + proof.set_disk_id = "pool_0_set_1".to_string(); + let proof_path = replacement_completion_proof_path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + proof_path.to_str().expect("completion proof path must be UTF-8"), + serde_json::to_vec(&proof) + .expect("completion proof fixture should serialize") + .into(), + ) + .await + .expect("conflicting proof fixture should persist"); + + let error = match ResumeManager::load_replacement_intent(disk, &task_id).await { + Ok(_) => panic!("a mismatched proof must not permit another rebuild"), + Err(error) => error, + }; + assert!(error.to_string().contains("does not match active intent")); +} + +#[tokio::test] +async fn replacement_intent_is_not_an_ordinary_resumable_task() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist in its isolated namespace"); + + assert!( + !ResumeManager::has_resume_state(&disk, &task_id).await, + "an old ordinary-resume lookup must not discover a replacement intent" + ); + assert!(ResumeManager::has_replacement_intent(&disk, &task_id).await); + assert!( + !ResumeUtils::get_resumable_tasks(&disk) + .await + .expect("ordinary resume listing should succeed") + .contains(&task_id), + "the old filename enumeration must not return replacement work" + ); + assert_eq!( + ResumeUtils::get_replacement_intent_tasks(&disk) + .await + .expect("replacement intent listing should succeed"), + vec![task_id.clone()] + ); + assert_eq!( + ResumeManager::load_replacement_intent(disk, &task_id) + .await + .expect("new replacement reader should load the isolated state") + .get_state() + .await, + manager.get_state().await + ); +} + +#[tokio::test] +async fn replacement_intent_recovers_from_torn_publication_before_formatting() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let intent_path = ResumeStateFile::ReplacementIntent.path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + intent_path.to_str().expect("replacement intent path must be UTF-8"), + b"{torn replacement intent".as_slice().into(), + ) + .await + .expect("torn replacement intent fixture should persist"); + + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("a retry must atomically replace only a torn pre-format intent"); + + let recovered = ResumeManager::load_replacement_intent(disk, &task_id) + .await + .expect("recovered intent should be readable after restart") + .get_state() + .await; + assert_eq!(recovered, manager.get_state().await); + assert_eq!(recovered.replacement_phase, ReplacementPhase::Intent); +} + +#[tokio::test] +async fn torn_intent_recovery_cas_preserves_a_concurrent_valid_binding() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let intent_path = ResumeStateFile::ReplacementIntent.path(&task_id); + let intent_path = intent_path.to_str().expect("replacement intent path must be UTF-8"); + let torn = EcstoreDiskBytes::from_static(b"{torn replacement intent"); + disk.write_all(RUSTFS_META_BUCKET, intent_path, torn) + .await + .expect("torn intent fixture should persist"); + + let expected = ResumeManager::torn_replacement_intent_bytes(&disk, &task_id) + .await + .expect("torn intent should be recoverable before a seal exists") + .expect("torn intent bytes should be retained as the CAS precondition"); + let winner = ResumeState::replacement_intent( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_winner".to_string(), + vec!["winner-bucket".to_string()], + vec!["replacement-winner".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-winner".to_string(), + canonical_path: "/mnt/replacement-winner".to_string(), + physical_device_ids: vec!["device-winner".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ); + let winner_bytes = EcstoreDiskBytes::from(serde_json::to_vec(&winner).expect("winner intent should serialize")); + disk.write_all(RUSTFS_META_BUCKET, intent_path, winner_bytes.clone()) + .await + .expect("concurrent valid intent fixture should persist"); + + let loser = ResumeManager { + disk: disk.clone(), + state: Arc::new(RwLock::new(ResumeState::replacement_intent( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_loser".to_string(), + vec!["loser-bucket".to_string()], + vec!["replacement-loser".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-loser".to_string(), + canonical_path: "/mnt/replacement-loser".to_string(), + physical_device_ids: vec!["device-loser".to_string()], + filesystem_identity: "4:5:6".to_string(), + }], + ))), + throttle: Mutex::new(PersistThrottle::new()), + state_file: ResumeStateFile::ReplacementIntent, + }; + let error = match loser.publish_new_replacement_intent(Some(expected)).await { + Ok(()) => panic!("a stale torn-intent recovery must not overwrite a concurrent valid binding"), + Err(error) => error, + }; + assert!(error.to_string().contains("changed before publication")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, intent_path) + .await + .expect("concurrent valid intent must remain durable"), + winner_bytes + ); +} + +#[tokio::test] +async fn replacement_intent_does_not_recreate_torn_state_after_seal_publication() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let intent_path = ResumeStateFile::ReplacementIntent.path(&task_id); + let intent_path = intent_path.to_str().expect("replacement intent path must be UTF-8"); + let torn = b"{torn replacement intent"; + disk.write_all(RUSTFS_META_BUCKET, intent_path, torn.as_slice().into()) + .await + .expect("torn replacement intent fixture should persist"); + let marker_path = replacement_intent_seal_path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + marker_path.to_str().expect("replacement seal path must be UTF-8"), + b"sealed".as_slice().into(), + ) + .await + .expect("replacement seal fixture should persist"); + + let error = match ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + { + Ok(_) => panic!("a seal means a torn state may have crossed the format boundary"), + Err(error) => error, + }; + assert!(error.to_string().contains("Failed to deserialize resume state")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, intent_path) + .await + .expect("torn intent must remain for operator recovery"), + torn.as_slice() + ); +} + +#[tokio::test] +async fn replacement_intent_migrates_from_legacy_resume_filename() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let legacy = ResumeState::replacement_intent( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ); + let legacy_path = ResumeStateFile::Ordinary.path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + legacy_path.to_str().expect("legacy resume path must be UTF-8"), + serde_json::to_vec(&legacy) + .expect("serialize legacy replacement state") + .into(), + ) + .await + .expect("write legacy replacement state"); + + let migrated = ResumeManager::load_replacement_intent(disk.clone(), &task_id) + .await + .expect("new binary should migrate a legacy replacement state"); + assert_eq!(migrated.get_state().await, legacy); + assert!( + !ResumeManager::has_resume_state(&disk, &task_id).await, + "migration must remove the old-binary-visible state only after the new state is durable" + ); + assert!(ResumeManager::has_replacement_intent(&disk, &task_id).await); +} + +#[tokio::test] +async fn ordinary_targeted_resume_is_not_migrated_as_a_replacement_intent() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new( + disk.clone(), + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ) + .await + .expect("ordinary targeted resume should persist"); + manager + .set_replacement_targets(vec!["manual-target".to_string()]) + .await + .expect("ordinary targeted resume should retain its target filter"); + + assert!(!ResumeManager::has_replacement_intent(&disk, &task_id).await); + assert!(ResumeManager::load_replacement_intent(disk.clone(), &task_id).await.is_err()); + assert!(ResumeManager::has_resume_state(&disk, &task_id).await); +} + +#[tokio::test] +async fn malformed_isolated_replacement_intent_is_reported_as_unknown() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let malformed = ResumeState::new( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ); + let path = ResumeStateFile::ReplacementIntent.path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + path.to_str().expect("isolated replacement path must be UTF-8"), + serde_json::to_vec(&malformed) + .expect("malformed replacement fixture should serialize") + .into(), + ) + .await + .expect("malformed isolated replacement state should persist"); + + let records = ResumeUtils::get_replacement_recovery_records(&disk) + .await + .expect("isolated replacement listing should succeed"); + assert_eq!(records.len(), 1); + assert_eq!(records[0].task_id, task_id); + assert_eq!(records[0].state, ReplacementRecoveryState::Unknown); +} + +#[tokio::test] +async fn startup_migration_moves_flat_replacement_artifacts_to_dedicated_directory() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let mut state = ResumeState::replacement_intent( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ); + state.mark_completed(); + state.replacement_phase = ReplacementPhase::Verified; + let legacy_intent = ResumeStateFile::LegacyReplacementIntent.path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + legacy_intent.to_str().expect("legacy intent path must be UTF-8"), + serde_json::to_vec(&state) + .expect("serialize legacy replacement intent") + .into(), + ) + .await + .expect("write legacy replacement intent"); + let proof = ReplacementCompletionProof::from_state(&state, state.last_update).expect("build legacy completion proof"); + let legacy_proof = legacy_replacement_completion_proof_path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + legacy_proof.to_str().expect("legacy proof path must be UTF-8"), + serde_json::to_vec(&proof).expect("serialize legacy completion proof").into(), + ) + .await + .expect("write legacy completion proof"); + + ResumeUtils::migrate_legacy_replacement_records(&disk) + .await + .expect("startup migration should move flat replacement artifacts"); + + assert_eq!( + ResumeUtils::get_replacement_intent_tasks(&disk) + .await + .expect("dedicated intent listing should succeed"), + vec![task_id.clone()] + ); + assert_eq!( + ResumeManager::load_replacement_completion_proof(disk.clone(), &task_id) + .await + .expect("dedicated completion proof should be readable"), + proof + ); + assert!(matches!( + disk.read_all(RUSTFS_META_BUCKET, legacy_intent.to_str().expect("legacy intent path must be UTF-8")) + .await, + Err(DiskError::FileNotFound) + )); + assert!(matches!( + disk.read_all(RUSTFS_META_BUCKET, legacy_proof.to_str().expect("legacy proof path must be UTF-8")) + .await, + Err(DiskError::FileNotFound) + )); +} + +#[tokio::test] +async fn startup_migration_moves_ordinary_replacement_resume_to_dedicated_directory() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let state = ResumeState::replacement_intent( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ); + let legacy_resume = ResumeStateFile::Ordinary.path(&task_id); + disk.write_all( + RUSTFS_META_BUCKET, + legacy_resume.to_str().expect("legacy resume path must be UTF-8"), + serde_json::to_vec(&state) + .expect("serialize legacy ordinary replacement state") + .into(), + ) + .await + .expect("write legacy ordinary replacement state"); + + ResumeUtils::migrate_legacy_replacement_records(&disk) + .await + .expect("startup migration should move ordinary replacement state"); + + assert_eq!( + ResumeUtils::get_replacement_intent_tasks(&disk) + .await + .expect("dedicated replacement listing should succeed"), + vec![task_id.clone()] + ); + assert_eq!( + ResumeManager::load_replacement_intent(disk.clone(), &task_id) + .await + .expect("migrated replacement intent should be readable") + .get_state() + .await, + state + ); + assert!(matches!( + disk.read_all(RUSTFS_META_BUCKET, legacy_resume.to_str().expect("legacy resume path must be UTF-8")) + .await, + Err(DiskError::FileNotFound) + )); +} + +#[tokio::test] +async fn startup_migration_reports_corrupt_ordinary_replacement_candidate() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let legacy_resume = ResumeStateFile::Ordinary.path(&task_id); + let legacy_resume = legacy_resume.to_str().expect("legacy resume path must be UTF-8"); + disk.write_all(RUSTFS_META_BUCKET, legacy_resume, b"{corrupt replacement resume".as_slice().into()) + .await + .expect("corrupt legacy replacement candidate should persist"); + + let error = ResumeUtils::migrate_legacy_replacement_records(&disk) + .await + .expect_err("a corrupt UUID-named legacy candidate must fail closed"); + + assert!(replacement_recovery_error_requires_block(&error)); + assert!(error.to_string().contains("replacement recovery corruption")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, legacy_resume) + .await + .expect("corrupt legacy state must remain for operator recovery"), + b"{corrupt replacement resume".as_slice() + ); +} + +#[tokio::test] +async fn startup_migration_reports_corrupt_flat_replacement_intent() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let legacy_intent = ResumeStateFile::LegacyReplacementIntent.path(&task_id); + let legacy_intent = legacy_intent.to_str().expect("legacy intent path must be UTF-8"); + disk.write_all(RUSTFS_META_BUCKET, legacy_intent, b"{corrupt replacement intent".as_slice().into()) + .await + .expect("corrupt legacy replacement intent should persist"); + + let error = ResumeUtils::migrate_legacy_replacement_records(&disk) + .await + .expect_err("a corrupt flat legacy intent must fail closed"); + + assert!(replacement_recovery_error_requires_block(&error)); + assert!(error.to_string().contains("replacement recovery corruption")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, legacy_intent) + .await + .expect("corrupt legacy intent must remain for operator recovery"), + b"{corrupt replacement intent".as_slice() + ); +} + +#[tokio::test] +async fn startup_migration_preserves_conflicting_dedicated_and_legacy_state() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("dedicated replacement intent should persist"); + let dedicated_path = ResumeStateFile::ReplacementIntent.path(&task_id); + let dedicated_bytes = disk + .read_all(RUSTFS_META_BUCKET, dedicated_path.to_str().expect("dedicated intent path must be UTF-8")) + .await + .expect("dedicated replacement intent should be readable"); + + let legacy_state = ResumeState::replacement_intent( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_1".to_string(), + vec!["other-bucket".to_string()], + vec!["replacement-b".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-b".to_string(), + canonical_path: "/mnt/replacement-b".to_string(), + physical_device_ids: vec!["device-b".to_string()], + filesystem_identity: "4:5:6".to_string(), + }], + ); + let legacy_path = ResumeStateFile::LegacyReplacementIntent.path(&task_id); + let legacy_bytes = serde_json::to_vec(&legacy_state).expect("serialize conflicting legacy replacement state"); + disk.write_all( + RUSTFS_META_BUCKET, + legacy_path.to_str().expect("legacy intent path must be UTF-8"), + legacy_bytes.clone().into(), + ) + .await + .expect("conflicting legacy replacement intent should persist"); + + let error = ResumeUtils::migrate_legacy_replacement_records(&disk) + .await + .expect_err("conflicting replacement states must fail closed"); + assert!(error.to_string().contains("conflicting legacy state")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, dedicated_path.to_str().expect("dedicated intent path must be UTF-8"),) + .await + .expect("dedicated state must remain after conflict"), + dedicated_bytes + ); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, legacy_path.to_str().expect("legacy intent path must be UTF-8"),) + .await + .expect("legacy state must remain after conflict"), + legacy_bytes + ); +} + +#[tokio::test] +async fn startup_migration_preserves_conflicting_dedicated_and_legacy_proof() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("dedicated replacement intent should persist"); + manager + .mark_replacement_completed_and_verified() + .await + .expect("dedicated completion proof should persist"); + let dedicated_path = replacement_completion_proof_path(&task_id); + let dedicated_bytes = disk + .read_all(RUSTFS_META_BUCKET, dedicated_path.to_str().expect("dedicated proof path must be UTF-8")) + .await + .expect("dedicated completion proof should be readable"); + + let mut legacy_proof = ReplacementCompletionProof::from_state(&manager.get_state().await, 42) + .expect("legacy completion proof fixture should build"); + legacy_proof.set_disk_id = "pool_0_set_1".to_string(); + let legacy_path = legacy_replacement_completion_proof_path(&task_id); + let legacy_bytes = serde_json::to_vec(&legacy_proof).expect("serialize conflicting legacy completion proof"); + disk.write_all( + RUSTFS_META_BUCKET, + legacy_path.to_str().expect("legacy proof path must be UTF-8"), + legacy_bytes.clone().into(), + ) + .await + .expect("conflicting legacy completion proof should persist"); + + let error = ResumeUtils::migrate_legacy_replacement_records(&disk) + .await + .expect_err("conflicting completion proofs must fail closed"); + assert!(error.to_string().contains("conflicts with legacy proof")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, dedicated_path.to_str().expect("dedicated proof path must be UTF-8"),) + .await + .expect("dedicated proof must remain after conflict"), + dedicated_bytes + ); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, legacy_path.to_str().expect("legacy proof path must be UTF-8"),) + .await + .expect("legacy proof must remain after conflict"), + legacy_bytes + ); + let error = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await { + Ok(_) => panic!("a conflicting legacy proof must not be ignored during recovery"), + Err(error) => error, + }; + assert!(error.to_string().contains("conflicts with legacy proof")); +} + +#[tokio::test] +async fn replacement_discovery_does_not_read_ordinary_resume_directory() { + let (_temp_dir, disk) = schema_test_disk().await; + for _ in 0..3 { + ResumeManager::new( + disk.clone(), + ResumeUtils::generate_task_id(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ) + .await + .expect("ordinary resume state should persist"); + } + let corrupt_task_id = ResumeUtils::generate_task_id(); + let corrupt_path = ResumeStateFile::Ordinary.path(&corrupt_task_id); + disk.write_all( + RUSTFS_META_BUCKET, + corrupt_path.to_str().expect("ordinary resume path must be UTF-8"), + b"not-json".to_vec().into(), + ) + .await + .expect("corrupt ordinary resume state should persist"); + + let replacement_task_id = ResumeUtils::generate_task_id(); + ResumeManager::new_replacement_intent( + disk.clone(), + replacement_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist in the dedicated directory"); + + assert_eq!( + ResumeUtils::get_replacement_intent_tasks(&disk) + .await + .expect("dedicated replacement listing should not parse ordinary JSON"), + vec![replacement_task_id] + ); +} + +#[tokio::test] +async fn empty_replacement_recovery_directory_is_not_an_error() { + let (_temp_dir, disk) = schema_test_disk().await; + assert!( + ResumeUtils::get_replacement_intent_tasks(&disk) + .await + .expect("missing recovery directory should be empty") + .is_empty() + ); + assert!( + ResumeUtils::get_replacement_recovery_records(&disk) + .await + .expect("missing recovery directory should have no records") + .is_empty() + ); + ResumeUtils::cleanup_expired_states(&disk, 0) + .await + .expect("missing recovery directory should not block expiry cleanup"); +} + +#[tokio::test] +async fn replacement_completion_proof_survives_resume_cleanup() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let identity = ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }; + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![identity.clone()], + ) + .await + .expect("replacement intent should persist"); + manager + .mark_replacement_completed_and_verified() + .await + .expect("verified replacement must persist proof before completion"); + + let proof = ResumeManager::load_replacement_completion_proof(disk.clone(), &task_id) + .await + .expect("completion proof must be readable from the survivor anchor"); + assert_eq!(proof.task_id, task_id); + assert_eq!(proof.replacement_generation, proof.task_id); + assert_eq!(proof.set_disk_id, "pool_0_set_0"); + assert_eq!(proof.replacement_targets, ["replacement-a"]); + assert_eq!(proof.replacement_target_identities, vec![identity]); + assert!(proof.verified_at > 0); + + manager.cleanup().await.expect("resume cleanup should succeed"); + assert!( + !ResumeManager::has_replacement_intent(&disk, &proof.task_id).await, + "completion cleanup must remove the resumable state" + ); + assert_eq!( + ResumeManager::load_replacement_completion_proof(disk, &proof.task_id) + .await + .expect("survivor proof must outlive resume cleanup"), + proof + ); +} + +#[tokio::test] +async fn replacement_recovery_records_distinguish_active_and_proven_completion() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + + let active = ResumeUtils::get_replacement_recovery_records(&disk) + .await + .expect("active replacement record should be readable"); + assert_eq!(active.len(), 1); + assert_eq!(active[0].task_id, task_id); + assert_eq!(active[0].state, ReplacementRecoveryState::WaitingForReplacement); + assert_eq!(active[0].generation.as_deref(), Some(task_id.as_str())); + + manager + .mark_replacement_completed_and_verified() + .await + .expect("completion proof should persist"); + + let cleanup_pending = ResumeUtils::get_replacement_recovery_records(&disk) + .await + .expect("cleanup-pending replacement record should be readable"); + assert_eq!(cleanup_pending.len(), 1); + assert_eq!(cleanup_pending[0].state, ReplacementRecoveryState::CleanupPending); + + manager.cleanup().await.expect("resume state cleanup should succeed"); + + let completed = ResumeUtils::get_replacement_recovery_records(&disk) + .await + .expect("completion proof should remain readable"); + assert_eq!(completed.len(), 1); + assert_eq!(completed[0].state, ReplacementRecoveryState::Completed); + assert!(completed[0].verified_at.is_some()); +} + +#[tokio::test] +async fn replacement_completion_write_failure_cannot_mark_completed() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk, + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + let proof_path = replacement_completion_proof_path(&task_id) + .to_str() + .expect("completion proof path must be UTF-8") + .to_string(); + let _failure = ReplacementProofWriteFailure::install(proof_path, DiskError::DiskAccessDenied); + + let error = manager + .mark_replacement_completed_and_verified() + .await + .expect_err("completion must fail closed when durable proof cannot be written"); + assert!(error.to_string().contains("Failed to save replacement completion proof")); + let state = manager.get_state().await; + assert!(!state.completed, "a failed proof write must not produce a completed state"); + assert_eq!(state.replacement_phase, ReplacementPhase::Intent); +} + +#[tokio::test] +async fn replacement_completion_repairs_torn_proof_without_wedging_rebuild() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let identity = ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }; + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![identity.clone()], + ) + .await + .expect("replacement intent should persist"); + manager + .mark_replacement_rebuilding(vec![identity]) + .await + .expect("replacement fixture should enter rebuilding before proof publication"); + let proof_path = replacement_completion_proof_path(&task_id); + let proof_path = proof_path.to_str().expect("completion proof path must be UTF-8"); + disk.write_all(RUSTFS_META_BUCKET, proof_path, b"{torn completion proof".as_slice().into()) + .await + .expect("torn proof fixture should persist"); + + manager + .mark_replacement_completed_and_verified() + .await + .expect("a rebuilding generation must replace only its torn completion proof"); + + let proof = ResumeManager::load_replacement_completion_proof(disk, &task_id) + .await + .expect("repaired completion proof should be durable and readable"); + assert_eq!(proof.task_id, task_id); + assert_eq!(proof.replacement_generation, proof.task_id); +} + +#[tokio::test] +async fn replacement_completion_does_not_replace_a_valid_mismatched_proof() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + let mut conflicting = ReplacementCompletionProof::from_state(&manager.get_state().await, 1) + .expect("replacement state should build a proof fixture"); + conflicting.set_disk_id = "pool_0_set_1".to_string(); + let proof_path = replacement_completion_proof_path(&task_id); + let proof_path = proof_path.to_str().expect("completion proof path must be UTF-8"); + disk.write_all( + RUSTFS_META_BUCKET, + proof_path, + serde_json::to_vec(&conflicting) + .expect("proof fixture should serialize") + .into(), + ) + .await + .expect("conflicting proof fixture should persist"); + + let error = manager + .mark_replacement_completed_and_verified() + .await + .expect_err("a distinct durable generation binding must not be overwritten"); + assert!(error.to_string().contains("does not match task")); + assert_eq!( + ResumeManager::load_replacement_completion_proof(disk, &task_id) + .await + .expect("valid conflicting proof should remain intact"), + conflicting + ); +} + +#[tokio::test] +async fn cleanup_expired_states_keeps_all_durable_replacement_phases() { + use super::super::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + + let temp_dir = TempDir::new().expect("create replacement expiry test directory"); + let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create test disk endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("create test disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(err) => panic!("create metadata volume for replacement expiry test: {err}"), + } + + let target = ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }; + let intent_task_id = ResumeUtils::generate_task_id(); + let rebuilding_task_id = ResumeUtils::generate_task_id(); + let verified_task_id = ResumeUtils::generate_task_id(); + let cleanup_pending_task_id = ResumeUtils::generate_task_id(); + let abandoned_task_id = ResumeUtils::generate_task_id(); + let ordinary_task_id = ResumeUtils::generate_task_id(); + let intent = ResumeManager::new_replacement_intent( + disk.clone(), + intent_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![target.clone()], + ) + .await + .expect("replacement intent should persist"); + let rebuilding = ResumeManager::new_replacement_intent( + disk.clone(), + rebuilding_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![target.clone()], + ) + .await + .expect("replacement rebuilding state should persist"); + rebuilding + .mark_replacement_rebuilding(vec![target.clone()]) + .await + .expect("replacement rebuilding phase should persist"); + let verified = ResumeManager::new_replacement_intent( + disk.clone(), + verified_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![target.clone()], + ) + .await + .expect("replacement verified state should persist"); + verified + .mark_replacement_completed_and_verified() + .await + .expect("replacement verified phase should persist"); + let cleanup_pending = ResumeManager::new_replacement_intent( + disk.clone(), + cleanup_pending_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![target.clone()], + ) + .await + .expect("replacement cleanup-pending state should persist"); + cleanup_pending + .mark_replacement_completed_and_verified() + .await + .expect("replacement completion should persist"); + cleanup_pending + .mark_replacement_cleanup_pending() + .await + .expect("replacement cleanup-pending phase should persist"); + let abandoned = ResumeManager::new_replacement_intent( + disk.clone(), + abandoned_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + vec!["replacement-a".to_string()], + vec![target], + ) + .await + .expect("replacement abandoned state should persist"); + abandoned + .abandon_replacement_intent() + .await + .expect("replacement abandoned phase should persist"); + let ordinary = ResumeManager::new( + disk.clone(), + ordinary_task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ) + .await + .expect("ordinary resume state should persist"); + + for manager in [&intent, &rebuilding, &verified, &cleanup_pending, &abandoned, &ordinary] { + manager.state.write().await.last_update = 0; + manager.save_state_strict().await.expect("persist expired resume state"); + } + + ResumeUtils::cleanup_expired_states(&disk, 0) + .await + .expect("replacement expiry cleanup should complete"); + + for (task_id, expected_phase) in [ + (intent_task_id.as_str(), ReplacementPhase::Intent), + (rebuilding_task_id.as_str(), ReplacementPhase::Rebuilding), + (verified_task_id.as_str(), ReplacementPhase::Verified), + (cleanup_pending_task_id.as_str(), ReplacementPhase::CleanupPending), + ] { + let state = ResumeManager::load_replacement_intent(disk.clone(), task_id) + .await + .expect("durable replacement state must survive expiry cleanup") + .get_state() + .await; + assert_eq!(state.replacement_phase, expected_phase); + } + assert!( + !ResumeManager::has_replacement_intent(&disk, &abandoned_task_id).await, + "an abandoned replacement must expire" + ); + assert!( + !ResumeManager::has_resume_state(&disk, &ordinary_task_id).await, + "an ordinary expired resume must expire" + ); +} + +#[tokio::test] +async fn test_resume_state_progress() { + let task_id = ResumeUtils::generate_task_id(); + let buckets = vec!["bucket1".to_string()]; + let mut state = ResumeState::new(task_id, "erasure_set".to_string(), "pool_0_set_0".to_string(), buckets); + + state.update_progress(10, 8, 1, 1); + assert_eq!(state.processed_objects, 10); + assert_eq!(state.successful_objects, 8); + assert_eq!(state.failed_objects, 1); + assert_eq!(state.skipped_objects, 1); + + let progress = state.get_progress_percentage(); + assert_eq!(progress, 0.0); // total_objects is 0 + + state.total_objects = 100; + let progress = state.get_progress_percentage(); + assert_eq!(progress, 10.0); +} + +#[tokio::test] +async fn replacement_intent_rejects_a_new_mount_at_the_same_endpoint() { + use super::super::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + + let temp_dir = TempDir::new().unwrap(); + let disk_path = temp_dir.path().join("resume_disk"); + std::fs::create_dir_all(&disk_path).unwrap(); + let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).unwrap(); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .unwrap(); + let _ = disk.make_volume(RUSTFS_META_BUCKET).await; + let _ = disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await; + + let task_id = ResumeUtils::generate_task_id(); + let targets = vec!["replacement-a".to_string()]; + let first = ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }; + ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket-a".to_string()], + targets.clone(), + vec![first.clone()], + ) + .await + .unwrap(); + + let reused = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket-b".to_string()], + targets.clone(), + vec![first.clone()], + ) + .await + .unwrap(); + assert_eq!( + reused.get_state().await.replacement_buckets, + ["bucket-a"], + "retries must keep the first generation's bucket plan" + ); + + let mut second = ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: first.physical_device_ids.clone(), + filesystem_identity: first.filesystem_identity.clone(), + }; + for changed_identity in [ + { + second.physical_device_ids = vec!["device-b".to_string()]; + second.clone() + }, + { + second.physical_device_ids = first.physical_device_ids.clone(); + second.filesystem_identity = "4:5:6".to_string(); + second.clone() + }, + { + second.filesystem_identity = first.filesystem_identity.clone(); + second.canonical_path = "/mnt/replacement-b".to_string(); + second.clone() + }, + ] { + let result = ResumeManager::new_replacement_intent( + disk.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket-a".to_string()], + targets.clone(), + vec![changed_identity], + ) + .await; + assert!(result.is_err(), "a new mounted instance must not reuse the old replacement cursor"); + } + temp_dir.close().unwrap(); +} + +#[test] +fn reset_for_retry_clears_progress_but_keeps_retry_budget() { + // backlog#855 / #799 B6: a retry must re-scan from the start without + // spending the retry budget's identity. + let buckets = vec!["bucket1".to_string(), "bucket2".to_string()]; + let mut state = ResumeState::new("t".to_string(), "erasure_set".to_string(), "pool_0_set_0".to_string(), buckets); + state.update_progress(10, 8, 2, 0); + state.complete_bucket("bucket1"); + state.increment_retry(); + state.mark_completed(); + + state.reset_for_retry(); + + assert!(!state.completed, "retry must un-complete the task"); + assert_eq!(state.completed_buckets.len(), 0, "all buckets must be re-scanned"); + assert_eq!(state.processed_objects, 0); + assert_eq!(state.successful_objects, 0); + assert_eq!(state.failed_objects, 0); + assert_eq!(state.skipped_objects, 0); + assert_eq!(state.retry_count, 1, "retry budget must be preserved"); +} + +#[test] +fn can_retry_is_bounded_by_max_retries() { + let mut state = ResumeState::new("t".to_string(), "erasure_set".to_string(), "pool_0_set_0".to_string(), vec![]); + assert!(state.can_retry()); + for _ in 0..state.max_retries { + assert!(state.can_retry()); + state.increment_retry(); + } + assert!(!state.can_retry(), "retries must stop after max_retries"); +} + +#[test] +fn checkpoint_reset_for_retry_rewinds_position_and_clears_sets() { + let mut checkpoint = ResumeCheckpoint::new("task".to_string()); + checkpoint.update_position(3, 42); + checkpoint.add_processed_object("bucket/a".to_string()); + checkpoint.add_failed_object("bucket/b".to_string()); + checkpoint.add_skipped_object("bucket/c".to_string()); + + checkpoint.reset_for_retry(); + + assert_eq!(checkpoint.current_bucket_index, 0); + assert_eq!(checkpoint.current_object_index, 0); + assert!(checkpoint.processed_objects.is_empty()); + assert!(checkpoint.failed_objects.is_empty()); + assert!(checkpoint.skipped_objects.is_empty()); +} + +#[tokio::test] +async fn test_resume_state_bucket_completion() { + let task_id = ResumeUtils::generate_task_id(); + let buckets = vec!["bucket1".to_string(), "bucket2".to_string()]; + let mut state = ResumeState::new(task_id, "erasure_set".to_string(), "pool_0_set_0".to_string(), buckets); + + assert_eq!(state.pending_buckets.len(), 2); + assert_eq!(state.completed_buckets.len(), 0); + + state.complete_bucket("bucket1"); + assert_eq!(state.pending_buckets.len(), 1); + assert_eq!(state.completed_buckets.len(), 1); + assert!(state.completed_buckets.contains(&"bucket1".to_string())); +} + +#[test] +fn test_checkpoint_object_sets_dedupe_and_prune() { + let mut checkpoint = ResumeCheckpoint::new("task".to_string()); + checkpoint.add_processed_object("bucket/a".to_string()); + checkpoint.add_processed_object("bucket/a".to_string()); + checkpoint.add_skipped_object("bucket/b".to_string()); + checkpoint.add_failed_object("bucket/c".to_string()); + assert_eq!(checkpoint.processed_objects.len(), 1); + assert!(checkpoint.processed_objects.contains("bucket/a")); + + checkpoint.complete_page(2, 2000); + assert_eq!(checkpoint.current_bucket_index, 2); + assert_eq!(checkpoint.current_object_index, 2000); + assert!(checkpoint.processed_objects.is_empty()); + assert!(checkpoint.skipped_objects.is_empty()); + assert!(checkpoint.failed_objects.is_empty()); +} + +#[test] +fn test_checkpoint_loads_legacy_vec_format() { + // Checkpoints written before the HashSet migration stored the object + // lists as JSON arrays (possibly with duplicates); they must still load. + let legacy = r#"{ + "task_id": "t1", + "checkpoint_time": 1700000000, + "current_bucket_index": 1, + "current_object_index": 42, + "processed_objects": ["a", "b", "a"], + "failed_objects": [], + "skipped_objects": ["c"] + }"#; + let checkpoint: ResumeCheckpoint = serde_json::from_str(legacy).unwrap(); + assert_eq!(checkpoint.current_object_index, 42); + assert_eq!(checkpoint.processed_objects.len(), 2); + assert!(checkpoint.processed_objects.contains("a")); + assert!(checkpoint.skipped_objects.contains("c")); +} + +#[test] +fn test_compose_key_injective_with_adversarial_keys() { + // Length-prefixing must keep the encoding injective even when keys + // contain the delimiter, embedded nulls, or look like a composed key. + assert_ne!(compose_key("a\0b", None), compose_key("a", Some("b"))); + assert_ne!(compose_key("3:xy", None), compose_key("x", Some("y"))); + assert_ne!(compose_key("a:b", None), compose_key("a", Some("b"))); + assert_ne!(compose_key("", Some("x")), compose_key("x", None)); + // Identical inputs must produce identical keys (stable identity). + assert_eq!(compose_key("obj", Some("v1")), compose_key("obj", Some("v1"))); +} + +#[test] +fn test_composite_key_dedup_distinguishes_versions() { + // Two versions of the same object must be distinct dedup identities, and + // the delete-marker/nil (None) version must not collide with a real one. + let mut checkpoint = ResumeCheckpoint::new("task".to_string()); + checkpoint.add_processed_object(compose_key("obj", Some("v1"))); + checkpoint.add_processed_object(compose_key("obj", Some("v2"))); + checkpoint.add_processed_object(compose_key("obj", None)); + assert_eq!(checkpoint.processed_objects.len(), 3); + assert!(checkpoint.processed_objects.contains(&compose_key("obj", Some("v1")))); + assert!(checkpoint.processed_objects.contains(&compose_key("obj", Some("v2")))); + assert!(checkpoint.processed_objects.contains(&compose_key("obj", None))); + // A different object with the same version id is still distinct. + assert!(!checkpoint.processed_objects.contains(&compose_key("other", Some("v1")))); +} + +#[tokio::test] +async fn test_resumestate_schema_v0_discarded_on_load() { + let (temp_dir, disk) = schema_test_disk().await; + + // Legacy snapshot: no schema_version, a stale positional cursor and progress. + let legacy = r#"{ + "task_id": "old-task", + "task_type": "erasure_set", + "set_disk_id": "pool_0_set_0", + "start_time": 1700000000, + "last_update": 1700000000, + "completed": true, + "total_objects": 100, + "processed_objects": 50, + "successful_objects": 40, + "failed_objects": 10, + "skipped_objects": 0, + "current_bucket": null, + "current_object": null, + "completed_buckets": ["b1"], + "pending_buckets": [], + "error_message": null, + "retry_count": 1, + "max_retries": 3, + "resume_cursor": "v1:stale-token" + }"#; + let task_id = "00000000-0000-4000-8000-000000000001"; + let legacy = legacy.replace("old-task", task_id); + let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); + disk.write_all(RUSTFS_META_BUCKET, &file_path, legacy.as_bytes().to_vec().into()) + .await + .expect("write legacy resume state"); + + let manager = ResumeManager::load_from_disk(disk.clone(), task_id).await.unwrap(); + let state = manager.get_state().await; + assert_eq!(state.schema_version, CURRENT_RESUME_SCHEMA, "schema must be stamped current"); + assert_eq!(state.resume_cursor, None, "stale cursor must be cleared"); + assert_eq!(state.processed_objects, 0); + assert_eq!(state.successful_objects, 0); + assert_eq!(state.failed_objects, 0); + assert!(!state.completed); + temp_dir.close().expect("remove schema test directory"); +} + +#[tokio::test] +async fn test_checkpoint_schema_v4_discarded_on_load() { + let (temp_dir, disk) = schema_test_disk().await; + + // The previous checkpoint schema is unsafe once its paired resume + // state is discarded: retaining either position would skip work. + let task_id = "00000000-0000-4000-8000-000000000002"; + let legacy = r#"{ + "schema_version": 4, + "task_id": "00000000-0000-4000-8000-000000000002", + "checkpoint_time": 1700000000, + "current_bucket_index": 2, + "current_object_index": 500, + "processed_objects": ["a", "b"], + "failed_objects": ["c"], + "skipped_objects": ["d"] + }"#; + let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); + disk.write_all(RUSTFS_META_BUCKET, &file_path, legacy.as_bytes().to_vec().into()) + .await + .expect("write legacy checkpoint"); + + let manager = CheckpointManager::load_from_disk(disk.clone(), task_id).await.unwrap(); + let checkpoint = manager.get_checkpoint().await; + assert_eq!(checkpoint.schema_version, CURRENT_CHECKPOINT_SCHEMA, "schema must be stamped current"); + assert_eq!(checkpoint.current_bucket_index, 0, "stale bucket position must be reset"); + assert_eq!(checkpoint.current_object_index, 0, "stale position must be reset"); + assert!(checkpoint.processed_objects.is_empty()); + assert!(checkpoint.failed_objects.is_empty()); + assert!(checkpoint.skipped_objects.is_empty()); + temp_dir.close().expect("remove schema test directory"); +} + +#[tokio::test] +async fn current_normal_resume_schema_preserves_progress() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let mut state = ResumeState::new( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket-b".to_string()], + ); + state.resume_cursor = Some("opaque-marker".to_string()); + state.processed_objects = 7; + state.successful_objects = 6; + state.failed_objects = 1; + state.completed_buckets = vec!["bucket-a".to_string()]; + let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); + let state_data = serde_json::to_vec(&state).expect("serialize current normal resume state"); + disk.write_all(RUSTFS_META_BUCKET, &file_path, state_data.into()) + .await + .expect("write current normal resume state"); + + let restored = ResumeManager::load_from_disk(disk.clone(), &task_id) + .await + .expect("load current normal resume state") + .get_state() + .await; + + assert_eq!(restored.schema_version, CURRENT_RESUME_SCHEMA); + assert_eq!(restored.resume_cursor.as_deref(), Some("opaque-marker")); + assert_eq!(restored.processed_objects, 7); + assert_eq!(restored.successful_objects, 6); + assert_eq!(restored.failed_objects, 1); + assert_eq!(restored.completed_buckets, ["bucket-a"]); + assert!(restored.replacement_targets.is_empty()); + assert_eq!(restored.replacement_generation, None); + assert_eq!(restored.replacement_phase, ReplacementPhase::None); + temp_dir.close().expect("remove schema test directory"); +} + +#[tokio::test] +async fn future_resume_and_checkpoint_schemas_are_rejected() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let mut state = ResumeState::new(task_id.clone(), "erasure_set".to_string(), "pool_0_set_0".to_string(), Vec::new()); + state.schema_version = CURRENT_RESUME_SCHEMA + 1; + let state_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); + let state_data = serde_json::to_vec(&state).expect("serialize future resume state"); + disk.write_all(RUSTFS_META_BUCKET, &state_path, state_data.into()) + .await + .expect("write future resume state"); + + let resume_error = match ResumeManager::load_from_disk(disk.clone(), &task_id).await { + Ok(_) => panic!("future resume schema must not load"), + Err(error) => error, + }; + assert!(matches!(resume_error, Error::TaskExecutionFailed { .. })); + assert!(resume_error.to_string().contains("newer than supported schema")); + + let mut checkpoint = ResumeCheckpoint::new(task_id.clone()); + checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1; + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); + let checkpoint_data = serde_json::to_vec(&checkpoint).expect("serialize future checkpoint"); + disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, checkpoint_data.into()) + .await + .expect("write future checkpoint"); + + let checkpoint_error = match CheckpointManager::load_from_disk(disk.clone(), &task_id).await { + Ok(_) => panic!("future checkpoint schema must not load"), + Err(error) => error, + }; + assert!(matches!(checkpoint_error, Error::TaskExecutionFailed { .. })); + assert!(checkpoint_error.to_string().contains("newer than supported schema")); + temp_dir.close().expect("remove schema test directory"); +} + +#[test] +fn test_persist_throttle_batches_until_threshold() { + let mut throttle = PersistThrottle::new(); + for _ in 0..PERSIST_EVERY_MUTATIONS - 1 { + assert!(!throttle.record(), "must not flush below the mutation threshold"); + } + assert!(throttle.record(), "must flush at the mutation threshold"); + throttle.mark_saved(); + assert!(!throttle.record(), "counter must reset after a save"); +} + +#[tokio::test] +async fn completion_persists_immediately_and_cleanup_propagates_delete_errors() { + use super::super::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + + let temp_dir = TempDir::new().expect("create resume persistence test directory"); + let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create test disk endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("create resume persistence test disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(err) => panic!("create metadata volume for resume persistence test: {err}"), + } + + let task_id = ResumeUtils::generate_task_id(); + let manager = ResumeManager::new( + disk.clone(), + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ) + .await + .expect("create resume manager"); + manager + .update_progress(1, 1, 0, 0) + .await + .expect("buffer progress below the persistence threshold"); + manager.mark_completed().await.expect("persist completed resume state"); + + let persisted = ResumeManager::load_from_disk(disk.clone(), &task_id) + .await + .expect("reload completed resume state") + .get_state() + .await; + assert!(persisted.completed, "completion must be persisted without waiting for the throttle"); + assert_eq!(persisted.processed_objects, 1, "the completion write must include buffered progress"); + + let state_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); + let failure = ResumeDeleteFailure::install(state_path, DiskError::DiskAccessDenied); + let error = manager + .cleanup() + .await + .expect_err("resume cleanup must propagate a real delete failure"); + assert!(matches!(error, Error::Disk(DiskError::DiskAccessDenied))); + drop(failure); + manager.cleanup().await.expect("resume cleanup must be retryable"); + manager + .cleanup() + .await + .expect("missing resume files must be idempotent success"); + + let checkpoint = CheckpointManager::new(disk.clone(), task_id.clone()) + .await + .expect("create checkpoint manager"); + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); + let failure = ResumeDeleteFailure::install(checkpoint_path, DiskError::DiskAccessDenied); + let error = checkpoint + .cleanup() + .await + .expect_err("checkpoint cleanup must propagate a real delete failure"); + assert!(matches!(error, Error::Disk(DiskError::DiskAccessDenied))); + drop(failure); + checkpoint.cleanup().await.expect("checkpoint cleanup must be retryable"); + checkpoint + .cleanup() + .await + .expect("missing checkpoint must be idempotent success"); +} + +#[tokio::test] +async fn checkpoint_rejects_a_task_id_mismatched_to_its_file_name() { + use super::super::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + + let temp_dir = TempDir::new().expect("create checkpoint binding test directory"); + let endpoint = + Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create checkpoint binding test endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("create checkpoint binding test disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(err) => panic!("create checkpoint binding metadata volume: {err}"), + } + + let requested_task_id = ResumeUtils::generate_task_id(); + let checkpoint = ResumeCheckpoint::new(ResumeUtils::generate_task_id()); + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{requested_task_id}_{RESUME_CHECKPOINT_FILE}"); + disk.write_all( + RUSTFS_META_BUCKET, + &checkpoint_path, + serde_json::to_vec(&checkpoint) + .expect("serialize mismatched checkpoint") + .into(), + ) + .await + .expect("persist mismatched checkpoint"); + + let error = match CheckpointManager::load_from_disk(disk, &requested_task_id).await { + Ok(_) => panic!("checkpoint task id must be bound to its file name"), + Err(error) => error, + }; + + assert!(error.to_string().contains("does not match")); + temp_dir.close().expect("remove checkpoint binding test directory"); +} + +#[tokio::test] +async fn test_resume_utils() { + let task_id1 = ResumeUtils::generate_task_id(); + let task_id2 = ResumeUtils::generate_task_id(); + + assert_ne!(task_id1, task_id2); + assert_eq!(task_id1.len(), 36); // UUID length + assert_eq!(task_id2.len(), 36); + assert!(validate_resume_task_id(&task_id1).is_ok()); + assert!(validate_resume_task_id(&format!("pool_0_set_0_{task_id1}")).is_err()); + assert!(validate_resume_task_id(&task_id1.to_uppercase()).is_err()); +} + +#[tokio::test] +async fn test_get_resumable_tasks_integration() { + use super::super::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + + // Create a temporary directory for testing + let temp_dir = TempDir::new().unwrap(); + let disk_path = temp_dir.path().join("test_disk"); + std::fs::create_dir_all(&disk_path).unwrap(); + + // Create a local disk for testing + let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).unwrap(); + let disk_option = DiskOption { + cleanup: false, + health_check: false, + }; + let disk = new_disk(&endpoint, &disk_option).await.unwrap(); + + // Create necessary directories first (ignore if already exist) + let _ = disk.make_volume(RUSTFS_META_BUCKET).await; + let _ = disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await; + + // Create some test resume state files + let task_ids = vec![ + ResumeUtils::generate_task_id(), + ResumeUtils::generate_task_id(), + ResumeUtils::generate_task_id(), + ]; + + // Save resume state files for each task + for task_id in &task_ids { + let state = ResumeState::new( + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket1".to_string(), "bucket2".to_string()], + ); + + let state_data = serde_json::to_vec(&state).unwrap(); + let file_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); + + disk.write_all(RUSTFS_META_BUCKET, &file_path, state_data.into()) + .await + .unwrap(); + } + + // Also create some non-resume state files to test filtering + let non_resume_files = vec![ + "other_file.txt", + "task4_ahm_checkpoint.json", + "task5_ahm_progress.json", + "_ahm_resume_state.json", // Invalid: empty task ID + "not-a-uuid_ahm_resume_state.json", + "00000000-0000-4000-8000-000000000001_extra_ahm_resume_state.json", + ]; + + for file_name in non_resume_files { + let file_path = format!("{BUCKET_META_PREFIX}/{file_name}"); + disk.write_all(RUSTFS_META_BUCKET, &file_path, b"test data".to_vec().into()) + .await + .unwrap(); + } + + // Now call get_resumable_tasks to see if it finds the correct files + let found_task_ids = ResumeUtils::get_resumable_tasks(&disk).await.unwrap(); + + // Verify that only the valid resume state files are found + assert_eq!(found_task_ids.len(), 3); + for task_id in &task_ids { + assert!(found_task_ids.contains(task_id), "Task ID {task_id} not found"); + } + + // Verify that invalid files are not included + assert!(!found_task_ids.contains(&"".to_string())); + assert!(!found_task_ids.contains(&"task4".to_string())); + assert!(!found_task_ids.contains(&"task5".to_string())); + assert!(!found_task_ids.contains(&"not-a-uuid".to_string())); + + let error = match ResumeManager::load_from_disk(disk.clone(), "../not-a-uuid").await { + Ok(_) => panic!("a traversal-like task id must be rejected before reading metadata"), + Err(error) => error, + }; + assert!(matches!( + error, + Error::TaskExecutionFailed { message } if message == "Invalid resume task id" + )); + + // Clean up + temp_dir.close().unwrap(); +} + +#[tokio::test] +async fn resume_state_rejects_filename_and_json_task_id_mismatch() { + use super::super::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + + let temp_dir = TempDir::new().expect("create resume mismatch test directory"); + let endpoint = Endpoint::try_from(temp_dir.path().to_string_lossy().as_ref()).expect("create test disk endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("create resume mismatch test disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(error) => panic!("create metadata volume for resume mismatch test: {error}"), + } + + let filename_task_id = ResumeUtils::generate_task_id(); + let state = ResumeState::new( + ResumeUtils::generate_task_id(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ); + let path = format!("{BUCKET_META_PREFIX}/{filename_task_id}_{RESUME_STATE_FILE}"); + disk.write_all( + RUSTFS_META_BUCKET, + &path, + serde_json::to_vec(&state).expect("serialize mismatched resume state").into(), + ) + .await + .expect("write mismatched resume state"); + + let error = match ResumeManager::load_from_disk(disk.clone(), &filename_task_id).await { + Ok(_) => panic!("resume state task id must match its filename"), + Err(error) => error, + }; + assert!(matches!( + error, + Error::TaskExecutionFailed { message } if message == "Resume state task id does not match filename" + )); + + let checkpoint_filename_task_id = ResumeUtils::generate_task_id(); + let checkpoint = ResumeCheckpoint::new(ResumeUtils::generate_task_id()); + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{checkpoint_filename_task_id}_{RESUME_CHECKPOINT_FILE}"); + disk.write_all( + RUSTFS_META_BUCKET, + &checkpoint_path, + serde_json::to_vec(&checkpoint) + .expect("serialize mismatched resume checkpoint") + .into(), + ) + .await + .expect("write mismatched resume checkpoint"); + + let error = match CheckpointManager::load_from_disk(disk, &checkpoint_filename_task_id).await { + Ok(_) => panic!("resume checkpoint task id must match its filename"), + Err(error) => error, + }; + assert!(matches!( + error, + Error::TaskExecutionFailed { message } if message == "Resume checkpoint task id does not match filename" + )); +} diff --git a/crates/heal/src/heal/resume/utils.rs b/crates/heal/src/heal/resume/utils.rs new file mode 100644 index 000000000..71507e7c0 --- /dev/null +++ b/crates/heal/src/heal/resume/utils.rs @@ -0,0 +1,311 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{Error, Result}; +use std::collections::HashSet; +use std::time::{SystemTime, UNIX_EPOCH}; +use tracing::{debug, warn}; +use uuid::Uuid; + +use super::super::{BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET}; +use super::replacement::{ReplacementPhase, ReplacementRecoveryRecord}; +use super::{ + EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE, + REPLACEMENT_INTENT_FILE, RESUME_STATE_FILE, ResumeManager, ResumeStateFile, is_replacement_intent, path_to_str, + replacement_recovery_corruption_for_state_load, replacement_recovery_dir, validate_resume_task_id, +}; + +/// resume utils +pub struct ResumeUtils; + +impl ResumeUtils { + /// generate unique task id + pub fn generate_task_id() -> String { + Uuid::new_v4().to_string() + } + + /// check if task can be resumed + pub async fn can_resume_task(disk: &DiskStore, task_id: &str) -> bool { + ResumeManager::has_resume_state(disk, task_id).await + } + + /// get all resumable task ids + pub async fn get_resumable_tasks(disk: &DiskStore) -> Result> { + // List all files in the buckets metadata directory + let entries = match disk.list_dir("", RUSTFS_META_BUCKET, BUCKET_META_PREFIX, -1).await { + Ok(entries) => entries, + Err(e) => { + debug!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_RESUME_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + state = "list_failed", + error = %e, + "Heal resume state listing failed" + ); + return Ok(Vec::new()); + } + }; + + let mut task_ids = Vec::new(); + + // Filter files that end with ahm_resume_state.json and extract task IDs + for entry in entries { + if entry.ends_with(&format!("_{RESUME_STATE_FILE}")) { + // Extract task ID from filename: {task_id}_ahm_resume_state.json + if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}")) + && validate_resume_task_id(task_id).is_ok() + { + task_ids.push(task_id.to_string()); + } + } + } + + debug!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_RESUME_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + task_count = task_ids.len(), + state = "listed", + "Heal resume states listed" + ); + Ok(task_ids) + } + + /// Return replacement intent task IDs from the dedicated recovery + /// directory. Periodic recovery must never enumerate the ordinary resume + /// directory, whose cardinality is unrelated to replacement work. + pub async fn get_replacement_intent_tasks(disk: &DiskStore) -> Result> { + let entries = Self::replacement_recovery_entries(disk).await?; + let suffix = format!("_{REPLACEMENT_INTENT_FILE}"); + let mut task_ids = HashSet::new(); + + for entry in entries { + if let Some(task_id) = entry.strip_suffix(&suffix) + && validate_resume_task_id(task_id).is_ok() + { + task_ids.insert(task_id.to_string()); + continue; + } + } + + let mut task_ids = task_ids.into_iter().collect::>(); + task_ids.sort_unstable(); + Ok(task_ids) + } + + async fn replacement_recovery_entries(disk: &DiskStore) -> Result> { + let recovery_dir = replacement_recovery_dir(); + let recovery_dir = path_to_str(&recovery_dir)?; + match disk.list_dir("", RUSTFS_META_BUCKET, recovery_dir, -1).await { + Ok(entries) => Ok(entries), + Err(DiskError::FileNotFound) => Ok(Vec::new()), + Err(error @ DiskError::UnformattedDisk) => Err(error.into()), + Err(error) => Err(Error::TaskExecutionFailed { + message: format!("Failed to list replacement recovery records: {error}"), + }), + } + } + + /// Migrate flat replacement artifacts from earlier builds exactly once at + /// manager startup. The normal scanner only uses the dedicated directory; + /// ordinary resume JSON is never read on its periodic path. + pub async fn migrate_legacy_replacement_records(disk: &DiskStore) -> Result<()> { + let entries = disk + .list_dir("", RUSTFS_META_BUCKET, BUCKET_META_PREFIX, -1) + .await + .map_err(|error| Error::TaskExecutionFailed { + message: format!("Failed to list legacy replacement records: {error}"), + })?; + let ordinary_suffix = format!("_{RESUME_STATE_FILE}"); + let intent_suffix = format!("_{REPLACEMENT_INTENT_FILE}"); + let proof_suffix = format!("_{REPLACEMENT_COMPLETION_PROOF_FILE}"); + let mut ordinary_task_ids = HashSet::new(); + let mut intent_task_ids = HashSet::new(); + let mut proof_task_ids = HashSet::new(); + + for entry in entries { + if let Some(task_id) = entry.strip_suffix(&intent_suffix) + && validate_resume_task_id(task_id).is_ok() + { + intent_task_ids.insert(task_id.to_string()); + continue; + } + if let Some(task_id) = entry.strip_suffix(&ordinary_suffix) + && validate_resume_task_id(task_id).is_ok() + { + ordinary_task_ids.insert(task_id.to_string()); + continue; + } + if let Some(task_id) = entry.strip_suffix(&proof_suffix) + && validate_resume_task_id(task_id).is_ok() + { + proof_task_ids.insert(task_id.to_string()); + } + } + + let mut state_task_ids = intent_task_ids.into_iter().collect::>(); + state_task_ids.extend(ordinary_task_ids); + state_task_ids.sort_unstable(); + state_task_ids.dedup(); + for task_id in state_task_ids { + let has_flat_intent = ResumeManager::has_state_file(disk, &task_id, ResumeStateFile::LegacyReplacementIntent).await; + if !has_flat_intent { + let manager = ResumeManager::load_from_disk(disk.clone(), &task_id).await.map_err(|error| { + replacement_recovery_corruption_for_state_load( + format!("Failed to load legacy replacement recovery candidate {task_id}"), + error, + ) + })?; + if !is_replacement_intent(&manager.get_state().await) { + continue; + } + } + ResumeManager::load_replacement_intent(disk.clone(), &task_id).await?; + } + + for task_id in proof_task_ids { + ResumeManager::migrate_legacy_replacement_completion_proof(disk, &task_id).await?; + } + Ok(()) + } + + /// Return all durable replacement states and completion proofs stored on + /// one survivor disk. Unlike the legacy resumable-task helper, listing + /// failures are returned to the caller so an observability surface cannot + /// silently turn an unreadable durable record into a green result. + pub async fn get_replacement_recovery_records(disk: &DiskStore) -> Result> { + let entries = Self::replacement_recovery_entries(disk).await?; + let proof_suffix = format!("_{REPLACEMENT_COMPLETION_PROOF_FILE}"); + let mut records = Vec::new(); + let mut intent_task_ids = HashSet::new(); + + for task_id in Self::get_replacement_intent_tasks(disk).await? { + let state = ResumeManager::load_replacement_intent(disk.clone(), &task_id) + .await? + .get_state() + .await; + intent_task_ids.insert(task_id.clone()); + records.push(ReplacementRecoveryRecord::from_state(state).unwrap_or_else(|| { + ReplacementRecoveryRecord::unknown( + task_id, + "isolated replacement intent violates its generation or target identity binding", + ) + })); + } + + for entry in entries { + let Some(task_id) = entry.strip_suffix(&proof_suffix) else { + continue; + }; + if validate_resume_task_id(task_id).is_err() { + continue; + } + if intent_task_ids.contains(task_id) { + continue; + } + let proof = ResumeManager::load_replacement_completion_proof(disk.clone(), task_id).await?; + records.push(ReplacementRecoveryRecord::from_completion_proof(&proof)); + } + + records.sort_by(|left, right| left.task_id.cmp(&right.task_id).then(left.state.cmp(&right.state))); + Ok(records) + } + + /// cleanup expired resume states + pub async fn cleanup_expired_states(disk: &DiskStore, max_age_hours: u64) -> Result<()> { + let task_ids = Self::get_resumable_tasks(disk).await?; + let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + + for task_id in task_ids { + if let Ok(resume_manager) = ResumeManager::load_from_disk(disk.clone(), &task_id).await { + let state = resume_manager.get_state().await; + let age_hours = current_time.saturating_sub(state.last_update) / 3600; + + if !state.completed && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) + { + continue; + } + if state.completed + && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) + { + continue; + } + + if age_hours > max_age_hours { + debug!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_RESUME_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + task_id, + age_hours, + state = "expired_cleanup_started", + "Heal resume cleanup started" + ); + if let Err(e) = resume_manager.cleanup().await { + warn!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_RESUME_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + task_id, + age_hours, + state = "expired_cleanup_failed", + error = %e, + "Heal resume state cleanup failed" + ); + } + } + } + } + + for task_id in Self::get_replacement_intent_tasks(disk).await? { + if let Ok(resume_manager) = ResumeManager::load_replacement_intent(disk.clone(), &task_id).await { + let state = resume_manager.get_state().await; + let age_hours = current_time.saturating_sub(state.last_update) / 3600; + + if !state.completed && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) + { + continue; + } + if state.completed + && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) + { + continue; + } + + if age_hours > max_age_hours + && let Err(e) = resume_manager.cleanup().await + { + warn!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_RESUME_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + task_id, + age_hours, + state = "expired_cleanup_failed", + error = %e, + "Replacement intent cleanup failed" + ); + } + } + } + + Ok(()) + } +}