diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 5c21c4704..9356f0ff0 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -78,10 +78,8 @@ pub mod bucket { pub mod recovery_disposition { pub use crate::bucket::lifecycle::recovery_disposition::{ - CreatedIlmRecoveryDisposition, IlmRecoveryDisposition, IlmRecoveryDispositionAction, IlmRecoveryDispositionError, - IlmRecoveryDispositionIdentity, IlmRecoveryDispositionOwnerLease, IlmRecoveryDispositionReasonCode, - IlmRecoveryDispositionState, ObservedIlmRecoveryDisposition, create_recovery_disposition_if_absent, - load_recovery_disposition, recovery_disposition_id, save_recovery_disposition_if_current, + IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionReasonCode, IlmRecoveryDispositionState, + dry_run_recovery_disposition, execute_recovery_disposition, }; } @@ -95,8 +93,9 @@ pub mod bucket { pub mod transition_transaction { pub use crate::bucket::lifecycle::transition_transaction::{ TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus, - delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator, - inspect_transition_transaction_for_operator, + TransitionRecoveryRetryResult, TransitionRecoveryRetryStatus, delete_transition_candidate_for_operator, + finalize_missing_transition_transaction_for_operator, inspect_transition_recovery_retry_for_operator, + inspect_transition_transaction_for_operator, retry_transition_recovery_for_operator, }; #[cfg(feature = "test-util")] pub use crate::bucket::lifecycle::transition_transaction::{ diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 426daa722..ea7a273a3 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -32,6 +32,7 @@ use crate::bucket::lifecycle::manual_transition_job::{ record_manual_transition_worker_result_with_reason, renew_manual_transition_job_lease_if_owned, save_manual_transition_job_record_if_current, save_manual_transition_task_if_absent, update_manual_transition_job_record, }; +use crate::bucket::lifecycle::recovery_disposition_runtime::run_recovery_disposition_maintenance_loop; use crate::bucket::lifecycle::replication_sink; use crate::bucket::lifecycle::replication_sink::{ DeleteReplicationConfigSnapshot, ReplicationObjectBridge, ReplicationStatusType, replication_state_to_filemeta, @@ -149,6 +150,7 @@ pub type ExpiryOpType = Box; static XXHASH_SEED: u64 = 0; static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new(); static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new(); +static RECOVERY_DISPOSITION_MAINTENANCE_STARTED: OnceLock<()> = OnceLock::new(); #[cfg(test)] #[derive(Default)] @@ -2398,9 +2400,20 @@ pub async fn init_background_expiry(api: Arc) { let _ = spawn_tier_free_version_recovery_once(api.clone(), &TIER_FREE_VERSION_RECOVERY_STARTED); spawn_tier_delete_journal_recovery_once(api.clone()); spawn_transition_transaction_recovery_once(api.clone()); + spawn_recovery_disposition_maintenance_once(api.clone()); spawn_manual_transition_job_recovery_once(api); } +fn spawn_recovery_disposition_maintenance_once(api: Arc) -> Option> { + let cancel_token = api.ctx.background_cancel_token()?; + if RECOVERY_DISPOSITION_MAINTENANCE_STARTED.set(()).is_err() { + return None; + } + Some(tokio::spawn(async move { + run_recovery_disposition_maintenance_loop(api, cancel_token).await; + })) +} + fn spawn_manual_transition_job_recovery_once(api: Arc) -> Option> { if MANUAL_TRANSITION_JOB_RECOVERY_STARTED.set(()).is_err() { return None; diff --git a/crates/ecstore/src/bucket/lifecycle/config_boundary.rs b/crates/ecstore/src/bucket/lifecycle/config_boundary.rs index 12b2e2ba5..9e2f977a9 100644 --- a/crates/ecstore/src/bucket/lifecycle/config_boundary.rs +++ b/crates/ecstore/src/bucket/lifecycle/config_boundary.rs @@ -161,20 +161,30 @@ where DeletedObject = DeletedObject, >, { - match api - .delete_object( - RUSTFS_META_BUCKET, - file, - ObjectOptions { - http_preconditions: Some(HTTPPreconditions { - if_match: Some(etag.to_string()), - ..Default::default() - }), - ..Default::default() - }, - ) - .await - { + delete_config_if_match_with_opts(api, file, etag, ObjectOptions::default()).await +} + +pub(crate) async fn delete_config_if_match_with_opts( + api: Arc, + file: &str, + etag: &str, + mut options: ObjectOptions, +) -> Result<()> +where + S: ObjectOperations< + Error = Error, + ObjectInfo = ObjectInfo, + ObjectOptions = ObjectOptions, + FileInfo = FileInfo, + ObjectToDelete = ObjectToDelete, + DeletedObject = DeletedObject, + >, +{ + options.http_preconditions = Some(HTTPPreconditions { + if_match: Some(etag.to_string()), + ..Default::default() + }); + match api.delete_object(RUSTFS_META_BUCKET, file, options).await { Ok(_) => Ok(()), Err(err) => { if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) { diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index 7197ad832..de64dbbcf 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -26,6 +26,7 @@ mod object_lock_boundary; pub use self::core as lifecycle; pub mod recovery_control; pub mod recovery_disposition; +pub(crate) mod recovery_disposition_runtime; pub mod recovery_export; mod replication_sink; pub mod rule; diff --git a/crates/ecstore/src/bucket/lifecycle/recovery_control.rs b/crates/ecstore/src/bucket/lifecycle/recovery_control.rs index de6224e82..4328f179b 100644 --- a/crates/ecstore/src/bucket/lifecycle/recovery_control.rs +++ b/crates/ecstore/src/bucket/lifecycle/recovery_control.rs @@ -485,6 +485,40 @@ impl IlmRecoveryControl { self.validate() } + pub fn abandon_for_operator(&mut self, expected_source_generation: &IlmRecoverySourceGeneration) -> Result<()> { + if self.owner.is_some() + || self.classification != IlmRecoveryClassification::RetainedAmbiguous + || &self.observed_source_generation != expected_source_generation + { + return Err(IlmRecoveryControlError::InvalidSuccessor( + "operator abandonment requires the exact ownerless retained source generation", + )); + } + self.bump_revision()?; + self.classification = IlmRecoveryClassification::Abandoned; + self.validate() + } + + pub fn retry_for_operator(&mut self, expected_source_generation: &IlmRecoverySourceGeneration) -> Result<()> { + if self.owner.is_some() + || !matches!( + self.classification, + IlmRecoveryClassification::RetainedAmbiguous | IlmRecoveryClassification::OperatorRequired + ) + || self.attempt_count == u64::MAX + || &self.observed_source_generation != expected_source_generation + { + return Err(IlmRecoveryControlError::InvalidSuccessor( + "operator retry requires the exact ownerless retained source generation", + )); + } + self.bump_revision()?; + self.classification = IlmRecoveryClassification::Retrying; + self.consecutive_failure_count = 0; + self.next_attempt_at_unix_nanos = None; + self.validate() + } + pub fn validate_successor(&self, next: &Self) -> Result<()> { self.validate()?; next.validate()?; @@ -508,12 +542,58 @@ impl IlmRecoveryControl { self.validate_failure_successor(next) } (Some(_), None) => self.validate_finish_successor(next), + (None, None) + if self.classification == IlmRecoveryClassification::RetainedAmbiguous + && next.classification == IlmRecoveryClassification::Abandoned => + { + self.validate_operator_abandon_successor(next) + } + (None, None) + if matches!( + self.classification, + IlmRecoveryClassification::RetainedAmbiguous | IlmRecoveryClassification::OperatorRequired + ) && next.classification == IlmRecoveryClassification::Retrying => + { + self.validate_operator_retry_successor(next) + } (None, None) => Err(IlmRecoveryControlError::InvalidSuccessor( "ownerless control cannot advance without a claim", )), } } + fn validate_operator_abandon_successor(&self, next: &Self) -> Result<()> { + if next.observed_source_generation != self.observed_source_generation + || next.attempt_count != self.attempt_count + || next.consecutive_failure_count != self.consecutive_failure_count + || next.first_failure_at_unix_nanos != self.first_failure_at_unix_nanos + || next.last_failure_at_unix_nanos != self.last_failure_at_unix_nanos + || next.next_attempt_at_unix_nanos != self.next_attempt_at_unix_nanos + || next.last_error_code != self.last_error_code + { + return Err(IlmRecoveryControlError::InvalidSuccessor( + "operator abandonment changed recovery history or source generation", + )); + } + Ok(()) + } + + fn validate_operator_retry_successor(&self, next: &Self) -> Result<()> { + if next.observed_source_generation != self.observed_source_generation + || next.attempt_count != self.attempt_count + || next.consecutive_failure_count != 0 + || next.first_failure_at_unix_nanos != self.first_failure_at_unix_nanos + || next.last_failure_at_unix_nanos != self.last_failure_at_unix_nanos + || next.next_attempt_at_unix_nanos.is_some() + || next.last_error_code != self.last_error_code + { + return Err(IlmRecoveryControlError::InvalidSuccessor( + "operator retry changed recovery history or source generation", + )); + } + Ok(()) + } + fn validate_claim_successor(&self, next: &Self) -> Result<()> { if self.classification != IlmRecoveryClassification::Retrying || next.classification != IlmRecoveryClassification::Retrying @@ -827,6 +907,23 @@ pub async fn observe_recovery_source( api: Arc, canonical_path: &str, source_schema: &str, +) -> EcstoreResult { + observe_recovery_source_with_options(api, canonical_path, source_schema, false).await +} + +pub(crate) async fn observe_recovery_source_no_lock( + api: Arc, + canonical_path: &str, + source_schema: &str, +) -> EcstoreResult { + observe_recovery_source_with_options(api, canonical_path, source_schema, true).await +} + +async fn observe_recovery_source_with_options( + api: Arc, + canonical_path: &str, + source_schema: &str, + no_lock: bool, ) -> EcstoreResult { validate_canonical_source_path(canonical_path).map_err(recovery_control_store_error)?; if source_schema.trim().is_empty() { @@ -837,7 +934,16 @@ pub async fn observe_recovery_source( let mut observations = Vec::new(); for set in api.all_set_disks() { let authority = format!("pool-{}/set-{}", set.pool_index, set.set_index); - match config_boundary::read_config_with_metadata(set, canonical_path, &ObjectOptions::default()).await { + match config_boundary::read_config_with_metadata( + set, + canonical_path, + &ObjectOptions { + no_lock, + ..Default::default() + }, + ) + .await + { Ok((data, metadata)) => { let etag = metadata .etag @@ -1255,6 +1361,99 @@ mod tests { )); } + #[test] + fn operator_abandonment_is_an_exact_ownerless_retained_successor() { + let mut retained = IlmRecoveryControl::new( + control().identity, + generation(), + IlmRecoveryClassification::RetainedAmbiguous, + 1_000_000_000, + IlmRecoveryErrorCode::OperatorDispositionRequired, + ) + .expect("retained control should build"); + let previous = retained.clone(); + retained + .abandon_for_operator(&previous.observed_source_generation) + .expect("exact retained generation should be abandonable"); + previous + .validate_successor(&retained) + .expect("operator abandonment should be a valid successor"); + assert_eq!(retained.classification, IlmRecoveryClassification::Abandoned); + assert_eq!(retained.revision, previous.revision + 1); + + let mut wrong_generation = previous.clone(); + let mut generation = previous.observed_source_generation.clone(); + generation.source_etag = "different".to_string(); + assert!(wrong_generation.abandon_for_operator(&generation).is_err()); + + let mut mutated_history = retained.clone(); + mutated_history.attempt_count += 1; + assert!(previous.validate_successor(&mutated_history).is_err()); + } + + #[test] + fn operator_retry_rearms_exact_retained_generation_without_resetting_history() { + for classification in [ + IlmRecoveryClassification::RetainedAmbiguous, + IlmRecoveryClassification::OperatorRequired, + ] { + let mut retained = control(); + retained + .claim("node-a", Uuid::new_v4(), 2_000_000_000, 1) + .expect("attempt should claim"); + retained + .record_retryable_failure(2_000_000_001, IlmRecoveryErrorCode::BackendTimeout) + .expect("failure should persist"); + retained.classification = classification; + retained.next_attempt_at_unix_nanos = None; + if classification == IlmRecoveryClassification::OperatorRequired { + retained.attempt_count = u64::from(MAX_RECOVERY_ATTEMPTS); + retained.consecutive_failure_count = MAX_RECOVERY_ATTEMPTS; + } + retained.validate().expect("retained control should remain valid"); + + let previous = retained.clone(); + retained + .retry_for_operator(&previous.observed_source_generation) + .expect("exact retained generation should be retryable"); + previous + .validate_successor(&retained) + .expect("operator retry should be a valid successor"); + assert_eq!(retained.classification, IlmRecoveryClassification::Retrying); + assert_eq!(retained.revision, previous.revision + 1); + assert_eq!(retained.attempt_count, previous.attempt_count); + assert_eq!(retained.first_failure_at_unix_nanos, previous.first_failure_at_unix_nanos); + assert_eq!(retained.last_failure_at_unix_nanos, previous.last_failure_at_unix_nanos); + assert_eq!(retained.last_error_code, previous.last_error_code); + assert_eq!(retained.consecutive_failure_count, 0); + assert_eq!(retained.next_attempt_at_unix_nanos, None); + assert!(retained.should_attempt_at(2_000_000_002)); + + if classification == IlmRecoveryClassification::OperatorRequired { + retained + .claim("node-b", Uuid::new_v4(), 2_000_000_002, 1) + .expect("operator retry should authorize one new bounded attempt"); + retained + .record_retryable_failure(2_000_000_003, IlmRecoveryErrorCode::BackendTimeout) + .expect("the bounded attempt failure should persist"); + assert_eq!(retained.classification, IlmRecoveryClassification::OperatorRequired); + assert_eq!(retained.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS) + 1); + assert_eq!(retained.consecutive_failure_count, 1); + } + + let mut wrong_generation = previous; + let mut changed_generation = wrong_generation.observed_source_generation.clone(); + changed_generation.source_etag = "changed".to_string(); + assert!(wrong_generation.retry_for_operator(&changed_generation).is_err()); + } + + let mut exhausted = control(); + exhausted.classification = IlmRecoveryClassification::OperatorRequired; + exhausted.attempt_count = u64::MAX; + let generation = exhausted.observed_source_generation.clone(); + assert!(exhausted.retry_for_operator(&generation).is_err()); + } + #[test] fn recovery_control_view_redacts_source_and_owner_details() { let mut control = control(); diff --git a/crates/ecstore/src/bucket/lifecycle/recovery_disposition.rs b/crates/ecstore/src/bucket/lifecycle/recovery_disposition.rs index 5721971ea..229ec3eb3 100644 --- a/crates/ecstore/src/bucket/lifecycle/recovery_disposition.rs +++ b/crates/ecstore/src/bucket/lifecycle/recovery_disposition.rs @@ -13,31 +13,79 @@ // limitations under the License. use std::sync::Arc; +#[cfg(all(test, feature = "test-util"))] +use std::sync::atomic::{AtomicU8, Ordering}; use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum}; use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; use uuid::Uuid; use super::config_boundary; use super::recovery_control::{ - IlmRecoveryClassification, IlmRecoveryProtocol, IlmRecoverySourceGeneration, load_recovery_control, observe_recovery_source, + IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryProtocol, IlmRecoverySourceCopy, IlmRecoverySourceGeneration, + MAX_ILM_RECOVERY_CONTROL_SIZE, load_recovery_control, observe_recovery_source, observe_recovery_source_no_lock, recovery_control_record_object_name, }; -use super::recovery_export::{IlmRecoveryExport, load_recovery_export, recovery_export_id}; +use super::recovery_export::{IlmRecoveryExport, IlmRecoveryExportObservation, load_recovery_export, recovery_export_id}; use super::tier_delete_journal::{ TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, validate_legacy_tier_delete_recovery_path, + validate_legacy_tier_delete_recovery_source, }; use crate::disk::RUSTFS_META_BUCKET; use crate::error::{Error, Result as EcstoreResult}; use crate::object_api::{ObjectOptions, WriteCompletion}; -use crate::storage_api_contracts::{namespace::NamespaceLocking as _, object::HTTPPreconditions}; +use crate::services::notification_sys::{ + IlmRecoveryExportFleetProofToken, acquire_ilm_recovery_export_fleet_proof, ilm_recovery_export_fleet_proof_matches, + ilm_recovery_export_member_epochs_sha256, ilm_recovery_export_topology_generation, +}; +use crate::storage_api_contracts::{ + namespace::NamespaceLocking as _, + object::{HTTPPreconditions, ObjectOperations as _}, +}; use crate::store::ECStore; -pub const ILM_RECOVERY_DISPOSITION_SCHEMA: &str = "rustfs-ilm-recovery-disposition-v1"; +const ILM_RECOVERY_DISPOSITION_SCHEMA_V1: &str = "rustfs-ilm-recovery-disposition-v1"; +pub const ILM_RECOVERY_DISPOSITION_SCHEMA: &str = "rustfs-ilm-recovery-disposition-v2"; pub const ILM_RECOVERY_DISPOSITION_PREFIX: &str = "ilm/recovery-dispositions"; pub const MAX_ILM_RECOVERY_DISPOSITION_SIZE: usize = 16 * 1024; const DISPOSITION_RETENTION_NANOS: i64 = 365 * 24 * 60 * 60 * 1_000_000_000; +const DISPOSITION_OWNER_LEASE_NANOS: i64 = 5 * 60 * 1_000_000_000; +const MAX_LEGACY_TIER_DELETE_SOURCE_SIZE: usize = 64 * 1024; const OWNER_FENCE_CHECKPOINT_DOMAIN: &[u8] = b"rustfs-ilm-recovery-disposition-owner-fence-v1"; +const LEGACY_V1_UNKNOWN_ABANDONED_CONTROL_SHA256: &str = "0000000000000000000000000000000000000000000000000000000000000000"; + +#[cfg(all(test, feature = "test-util"))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RecoveryDispositionCrashStage { + AfterLocalDelete = 1, + AfterControlAbandon = 2, +} + +#[cfg(all(test, feature = "test-util"))] +static RECOVERY_DISPOSITION_CRASH_STAGE: AtomicU8 = AtomicU8::new(0); + +#[cfg(all(test, feature = "test-util"))] +pub(crate) fn inject_recovery_disposition_crash_once(stage: RecoveryDispositionCrashStage) { + RECOVERY_DISPOSITION_CRASH_STAGE.store(stage as u8, Ordering::SeqCst); +} + +#[cfg(all(test, feature = "test-util"))] +fn maybe_inject_recovery_disposition_crash(stage: RecoveryDispositionCrashStage) -> EcstoreResult<()> { + if RECOVERY_DISPOSITION_CRASH_STAGE + .compare_exchange(stage as u8, 0, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + let message = match stage { + RecoveryDispositionCrashStage::AfterLocalDelete => "injected ILM recovery disposition crash after local delete", + RecoveryDispositionCrashStage::AfterControlAbandon => { + "injected ILM recovery disposition crash after control abandonment" + } + }; + return Err(Error::other(message)); + } + Ok(()) +} pub type Result = std::result::Result; @@ -51,6 +99,8 @@ pub enum IlmRecoveryDispositionError { ChecksumMismatch, #[error("ILM recovery disposition successor is invalid: {0}")] InvalidSuccessor(&'static str), + #[error("ILM recovery disposition v1 requires safe migration: {0}")] + LegacyV1MigrationRequired(&'static str), #[error("ILM recovery disposition json error: {0}")] Json(#[from] serde_json::Error), } @@ -94,6 +144,7 @@ pub struct IlmRecoveryDispositionIdentity { pub control_id: String, pub control_etag: String, pub control_revision: u64, + pub abandoned_control_sha256: String, pub canonical_source_path: String, pub source_generation: IlmRecoverySourceGeneration, pub admitted_topology_generation: String, @@ -109,6 +160,7 @@ impl IlmRecoveryDispositionIdentity { validate_sha256(&self.export_id, "export ID is invalid")?; validate_sha256(&self.export_content_sha256, "export content checksum is invalid")?; validate_sha256(&self.control_id, "control ID is invalid")?; + validate_sha256(&self.abandoned_control_sha256, "abandoned control content checksum is invalid")?; validate_sha256(&self.admitted_topology_generation, "admitted topology generation is invalid")?; validate_sha256(&self.admitted_member_epochs_sha256, "admitted member epoch digest is invalid")?; validate_sha256(&self.actor_sha256, "actor digest is invalid")?; @@ -137,6 +189,8 @@ impl IlmRecoveryDispositionIdentity { copy.canonical_path != self.canonical_source_path || copy.etag != self.source_generation.source_etag || copy.content_sha256 != self.source_generation.content_sha256 + || copy.encoded_len == 0 + || copy.encoded_len > 64 * 1024 }) { return Err(IlmRecoveryDispositionError::Corrupt( "copy manifest does not describe one exact source generation", @@ -195,6 +249,8 @@ pub struct IlmRecoveryDisposition { #[serde(default, skip_serializing_if = "Option::is_none")] pub owner: Option, pub confirmed_absent: Vec, + #[serde(skip)] + legacy_v1_migration_required: bool, } impl IlmRecoveryDisposition { @@ -210,6 +266,7 @@ impl IlmRecoveryDisposition { state: IlmRecoveryDispositionState::Prepared, owner: None, confirmed_absent: Vec::new(), + legacy_v1_migration_required: false, }; disposition.validate()?; Ok(disposition) @@ -484,6 +541,11 @@ impl IlmRecoveryDisposition { pub fn encode(&self) -> Result> { self.validate()?; + if self.requires_legacy_v1_migration() { + return Err(IlmRecoveryDispositionError::LegacyV1MigrationRequired( + "legacy records cannot be emitted as v2 before their control successor is bound", + )); + } let disposition_bytes = serde_json::to_vec(self)?; let persisted = PersistedIlmRecoveryDisposition { schema: ILM_RECOVERY_DISPOSITION_SCHEMA.to_string(), @@ -502,20 +564,43 @@ impl IlmRecoveryDisposition { if data.len() > MAX_ILM_RECOVERY_DISPOSITION_SIZE { return Err(IlmRecoveryDispositionError::Corrupt("encoded disposition exceeds maximum size")); } - let persisted: PersistedIlmRecoveryDisposition = serde_json::from_slice(data)?; - if persisted.schema != ILM_RECOVERY_DISPOSITION_SCHEMA { - return Err(IlmRecoveryDispositionError::UnsupportedSchema(persisted.schema)); - } - validate_sha256(&persisted.content_sha256, "content checksum is invalid")?; - let disposition_bytes = serde_json::to_vec(&persisted.disposition)?; - if hex_sha256(&disposition_bytes, ToOwned::to_owned) != persisted.content_sha256 { - return Err(IlmRecoveryDispositionError::ChecksumMismatch); - } - persisted.disposition.validate()?; - if persisted.disposition.identity.disposition_id != expected_disposition_id { + let schema: PersistedIlmRecoveryDispositionSchema = serde_json::from_slice(data)?; + let disposition = match schema.schema.as_str() { + ILM_RECOVERY_DISPOSITION_SCHEMA => { + let persisted: PersistedIlmRecoveryDisposition = serde_json::from_slice(data)?; + if persisted.schema != ILM_RECOVERY_DISPOSITION_SCHEMA { + return Err(IlmRecoveryDispositionError::UnsupportedSchema(persisted.schema)); + } + validate_sha256(&persisted.content_sha256, "content checksum is invalid")?; + let disposition_bytes = serde_json::to_vec(&persisted.disposition)?; + if hex_sha256(&disposition_bytes, ToOwned::to_owned) != persisted.content_sha256 { + return Err(IlmRecoveryDispositionError::ChecksumMismatch); + } + persisted.disposition + } + ILM_RECOVERY_DISPOSITION_SCHEMA_V1 => { + let persisted: PersistedIlmRecoveryDispositionV1 = serde_json::from_slice(data)?; + if persisted.schema != ILM_RECOVERY_DISPOSITION_SCHEMA_V1 { + return Err(IlmRecoveryDispositionError::UnsupportedSchema(persisted.schema)); + } + validate_sha256(&persisted.content_sha256, "content checksum is invalid")?; + let disposition_bytes = serde_json::to_vec(&persisted.disposition)?; + if hex_sha256(&disposition_bytes, ToOwned::to_owned) != persisted.content_sha256 { + return Err(IlmRecoveryDispositionError::ChecksumMismatch); + } + persisted.disposition.into_current() + } + _ => return Err(IlmRecoveryDispositionError::UnsupportedSchema(schema.schema)), + }; + disposition.validate()?; + if disposition.identity.disposition_id != expected_disposition_id { return Err(IlmRecoveryDispositionError::Corrupt("disposition ID does not match record key")); } - Ok(persisted.disposition) + Ok(disposition) + } + + fn requires_legacy_v1_migration(&self) -> bool { + self.legacy_v1_migration_required } fn bump_revision(&mut self) -> Result<()> { @@ -527,6 +612,91 @@ impl IlmRecoveryDisposition { } } +#[derive(Debug, Deserialize)] +struct PersistedIlmRecoveryDispositionSchema { + schema: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct IlmRecoveryDispositionIdentityV1 { + disposition_id: String, + protocol: IlmRecoveryProtocol, + action: IlmRecoveryDispositionAction, + export_id: String, + export_content_sha256: String, + control_id: String, + control_etag: String, + control_revision: u64, + canonical_source_path: String, + source_generation: IlmRecoverySourceGeneration, + admitted_topology_generation: String, + admitted_member_epochs_sha256: String, + actor_sha256: String, + reason_code: IlmRecoveryDispositionReasonCode, + confirmed_at_unix_nanos: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct IlmRecoveryDispositionV1 { + identity: IlmRecoveryDispositionIdentityV1, + created_at_unix_nanos: i64, + retain_until_unix_nanos: i64, + revision: u64, + state: IlmRecoveryDispositionState, + #[serde(default, skip_serializing_if = "Option::is_none")] + owner: Option, + confirmed_absent: Vec, +} + +impl IlmRecoveryDispositionV1 { + fn into_current(self) -> IlmRecoveryDisposition { + let retain_until_unix_nanos = if self.state == IlmRecoveryDispositionState::Completed { + // A completed v1 record cannot prove the exact control successor. + // Keep it quarantined and auditable instead of allowing retention GC. + i64::MAX + } else { + self.retain_until_unix_nanos + }; + IlmRecoveryDisposition { + identity: IlmRecoveryDispositionIdentity { + disposition_id: self.identity.disposition_id, + protocol: self.identity.protocol, + action: self.identity.action, + export_id: self.identity.export_id, + export_content_sha256: self.identity.export_content_sha256, + control_id: self.identity.control_id, + control_etag: self.identity.control_etag, + control_revision: self.identity.control_revision, + abandoned_control_sha256: LEGACY_V1_UNKNOWN_ABANDONED_CONTROL_SHA256.to_string(), + canonical_source_path: self.identity.canonical_source_path, + source_generation: self.identity.source_generation, + admitted_topology_generation: self.identity.admitted_topology_generation, + admitted_member_epochs_sha256: self.identity.admitted_member_epochs_sha256, + actor_sha256: self.identity.actor_sha256, + reason_code: self.identity.reason_code, + confirmed_at_unix_nanos: self.identity.confirmed_at_unix_nanos, + }, + created_at_unix_nanos: self.created_at_unix_nanos, + retain_until_unix_nanos, + revision: self.revision, + state: self.state, + owner: self.owner, + confirmed_absent: self.confirmed_absent, + legacy_v1_migration_required: true, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedIlmRecoveryDispositionV1 { + schema: String, + content_sha256: String, + disposition: IlmRecoveryDispositionV1, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct PersistedIlmRecoveryDisposition { @@ -549,6 +719,33 @@ pub struct CreatedIlmRecoveryDisposition { pub replayed: bool, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct IlmRecoveryDispositionDryRun { + pub disposition_id: String, + pub export_id: String, + pub export_content_sha256: String, + pub source_generation_sha256: String, + pub copy_set_sha256: String, + pub source_copy_count: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IlmRecoveryDispositionExecutionOutcome { + AcceptedForRecovery, + Completed, + Replayed, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct IlmRecoveryDispositionExecution { + pub disposition_id: String, + pub state: IlmRecoveryDispositionState, + pub outcome: IlmRecoveryDispositionExecutionOutcome, + pub confirmed_absent_copy_count: usize, + pub source_copy_count: usize, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct DecodedIlmRecoveryDispositionCheckpoint { pub(crate) disposition_id: String, @@ -630,7 +827,19 @@ pub(crate) fn decode_recovery_disposition_checkpoint( if canonical != path || disposition.identity.protocol != protocol { return Err(Error::other("ILM recovery disposition path is not canonical")); } - let identity_sha256 = checkpoint_hash(&disposition.identity)?; + let legacy_v1 = if disposition.requires_legacy_v1_migration() { + Some( + serde_json::from_slice::(data) + .map_err(IlmRecoveryDispositionError::from) + .map_err(disposition_store_error)?, + ) + } else { + None + }; + let identity_sha256 = match legacy_v1.as_ref() { + Some(persisted) => checkpoint_hash(&persisted.disposition.identity)?, + None => checkpoint_hash(&disposition.identity)?, + }; let owner_fence_sha256 = disposition .owner .as_ref() @@ -655,11 +864,13 @@ pub(crate) fn decode_recovery_disposition_checkpoint( owner_lease_acquired_at_unix_nanos: disposition.owner.as_ref().map(|owner| owner.lease_acquired_at_unix_nanos), owner_lease_expires_at_unix_nanos: disposition.owner.as_ref().map(|owner| owner.lease_expires_at_unix_nanos), confirmed_absent_sha256, - retain_until_unix_nanos: disposition.retain_until_unix_nanos, + retain_until_unix_nanos: legacy_v1.as_ref().map_or(disposition.retain_until_unix_nanos, |persisted| { + persisted.disposition.retain_until_unix_nanos + }), }) } -pub async fn create_recovery_disposition_if_absent( +pub(crate) async fn create_recovery_disposition_if_absent( api: Arc, disposition: &IlmRecoveryDisposition, ) -> EcstoreResult { @@ -688,15 +899,15 @@ pub async fn create_recovery_disposition_if_absent( .map_err(|err| Error::other(err.to_string()))?; let control_lock = api.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?; let control_guard = control_lock - .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) .await?; let source_lock = api .new_ns_lock(RUSTFS_META_BUCKET, &disposition.identity.canonical_source_path) .await?; let source_guard = source_lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?; let locks_current = || !control_guard.is_lock_lost() && !source_guard.is_lock_lost(); - validate_disposition_sources(api.clone(), disposition).await?; - let current_source = observe_recovery_source( + validate_disposition_sources(api.clone(), disposition, true).await?; + let current_source = observe_recovery_source_no_lock( api.clone(), &disposition.identity.canonical_source_path, &disposition.identity.source_generation.source_schema, @@ -738,13 +949,21 @@ pub async fn create_recovery_disposition_if_absent( } } -async fn validate_disposition_sources(api: Arc, disposition: &IlmRecoveryDisposition) -> EcstoreResult<()> { +async fn validate_disposition_sources( + api: Arc, + disposition: &IlmRecoveryDisposition, + no_lock: bool, +) -> EcstoreResult<()> { let stored_export = load_recovery_export(api.clone(), &disposition.identity.export_id).await?; if stored_export.content_sha256 != disposition.identity.export_content_sha256 { return Err(Error::PreconditionFailed); } let export = IlmRecoveryExport::decode(&stored_export.export_id, &stored_export.encoded)?; - let observed_control = load_recovery_control(api, disposition.identity.protocol, &disposition.identity.control_id).await?; + let observed_control = if no_lock { + load_recovery_control_no_lock(api, disposition).await? + } else { + load_recovery_control(api, disposition.identity.protocol, &disposition.identity.control_id).await? + }; if export.control_id != disposition.identity.control_id || export.protocol != disposition.identity.protocol || export.control_etag != disposition.identity.control_etag @@ -759,22 +978,838 @@ async fn validate_disposition_sources(api: Arc, disposition: &IlmRecove || observed_control.control.classification != IlmRecoveryClassification::RetainedAmbiguous || observed_control.control.identity.canonical_source_path != disposition.identity.canonical_source_path || observed_control.control.observed_source_generation != disposition.identity.source_generation + || expected_abandoned_control_sha256(&observed_control.control, &disposition.identity.source_generation)? + != disposition.identity.abandoned_control_sha256 { return Err(Error::PreconditionFailed); } Ok(()) } -pub async fn load_recovery_disposition( +pub async fn dry_run_recovery_disposition( + api: Arc, + observation: &IlmRecoveryExportObservation, + export_id: &str, + export_content_sha256: &str, + actor_sha256: &str, + confirmed_at_unix_nanos: i64, +) -> EcstoreResult { + let (candidate, proof) = prepare_recovery_disposition( + api.clone(), + observation, + export_id, + export_content_sha256, + actor_sha256, + confirmed_at_unix_nanos, + ) + .await?; + match load_recovery_disposition(api.clone(), candidate.identity.protocol, &candidate.identity.disposition_id).await { + Ok(existing) => { + validate_existing_request_binding(&existing.disposition, &candidate.identity)?; + if existing.disposition.requires_legacy_v1_migration() { + return Err(legacy_v1_migration_error( + "v1 records require an explicit migration before disposition execution", + )); + } + if existing.disposition.state != IlmRecoveryDispositionState::Completed { + validate_live_disposition_inputs(api, &existing.disposition, &proof, false).await?; + } + } + Err(err) if disposition_is_missing(&err) => { + validate_disposition_sources(api.clone(), &candidate, false).await?; + let observed_source = observe_recovery_source( + api, + &candidate.identity.canonical_source_path, + &candidate.identity.source_generation.source_schema, + ) + .await?; + if observed_source.canonical_data.is_none() || observed_source.generation != candidate.identity.source_generation { + return Err(Error::PreconditionFailed); + } + } + Err(err) => return Err(err), + } + if !ilm_recovery_export_fleet_proof_matches(&proof).await { + return Err(Error::PreconditionFailed); + } + Ok(IlmRecoveryDispositionDryRun { + disposition_id: candidate.identity.disposition_id, + export_id: export_id.to_string(), + export_content_sha256: export_content_sha256.to_string(), + source_generation_sha256: checkpoint_hash(&observation.source_generation)?, + copy_set_sha256: observation.source_generation.copy_set_sha256.clone(), + source_copy_count: observation.source_generation.copies.len(), + }) +} + +pub async fn execute_recovery_disposition( + api: Arc, + observation: &IlmRecoveryExportObservation, + export_id: &str, + export_content_sha256: &str, + actor_sha256: &str, + confirmed_at_unix_nanos: i64, +) -> EcstoreResult { + let (candidate, proof) = prepare_recovery_disposition( + api.clone(), + observation, + export_id, + export_content_sha256, + actor_sha256, + confirmed_at_unix_nanos, + ) + .await?; + let disposition_id = candidate.identity.disposition_id.clone(); + match load_recovery_disposition(api.clone(), candidate.identity.protocol, &disposition_id).await { + Ok(existing) => { + validate_existing_request_binding(&existing.disposition, &candidate.identity)?; + if existing.disposition.requires_legacy_v1_migration() { + return Err(legacy_v1_migration_error( + "v1 records require an explicit migration before disposition execution", + )); + } + if existing.disposition.state == IlmRecoveryDispositionState::Completed { + return Ok(disposition_execution( + &existing.disposition, + IlmRecoveryDispositionExecutionOutcome::Replayed, + )); + } + validate_live_disposition_inputs(api.clone(), &existing.disposition, &proof, false).await?; + } + Err(err) if disposition_is_missing(&err) => { + validate_disposition_sources(api.clone(), &candidate, false).await?; + let created = super::recovery_disposition_runtime::create_recovery_disposition_with_admission( + api.clone(), + &candidate, + confirmed_at_unix_nanos, + ) + .await?; + validate_existing_request_binding(&created.observed.disposition, &candidate.identity)?; + } + Err(err) => return Err(err), + } + if !ilm_recovery_export_fleet_proof_matches(&proof).await { + return Err(Error::PreconditionFailed); + } + let _execution_permit = super::recovery_disposition_runtime::acquire_recovery_disposition_execution_permit().await?; + resume_recovery_disposition(api, observation.protocol, &disposition_id, confirmed_at_unix_nanos).await +} + +pub(crate) async fn resume_recovery_disposition( api: Arc, protocol: IlmRecoveryProtocol, disposition_id: &str, + now_unix_nanos: i64, +) -> EcstoreResult { + if now_unix_nanos <= 0 { + return Err(Error::PreconditionFailed); + } + let initial = load_recovery_disposition(api.clone(), protocol, disposition_id).await?; + if initial.disposition.requires_legacy_v1_migration() { + return Err(legacy_v1_migration_error( + "v1 records require an explicit migration before disposition execution", + )); + } + if initial.disposition.state == IlmRecoveryDispositionState::Completed { + return Ok(disposition_execution( + &initial.disposition, + IlmRecoveryDispositionExecutionOutcome::Replayed, + )); + } + let proof = acquire_matching_disposition_proof(&initial.disposition).await?; + let control_object = recovery_control_record_object_name(protocol, &initial.disposition.identity.control_id) + .map_err(|_| Error::PreconditionFailed)?; + // Lock order is control record, then legacy source. Disposition revisions + // use ETag CAS and never acquire either namespace lock internally. + let control_lock = api.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?; + let control_guard = control_lock + .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let source_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &initial.disposition.identity.canonical_source_path) + .await?; + let source_guard = source_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let fences_current = || !control_guard.is_lock_lost() && !source_guard.is_lock_lost(); + if !fences_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await { + return Err(Error::PreconditionFailed); + } + + let mut current = load_recovery_disposition(api.clone(), protocol, disposition_id).await?; + if current.disposition.requires_legacy_v1_migration() { + return Err(legacy_v1_migration_error( + "v1 records require an explicit migration before disposition execution", + )); + } + if current.disposition.state == IlmRecoveryDispositionState::Completed { + return Ok(disposition_execution( + ¤t.disposition, + IlmRecoveryDispositionExecutionOutcome::Replayed, + )); + } + validate_live_disposition_inputs(api.clone(), ¤t.disposition, &proof, true).await?; + if current.disposition.owner.as_ref().is_some_and(|owner| { + owner.topology_generation != current.disposition.identity.admitted_topology_generation + || owner.member_epochs_sha256 != current.disposition.identity.admitted_member_epochs_sha256 + }) { + return Err(Error::PreconditionFailed); + } + + let owner = match current.disposition.owner.as_ref() { + Some(owner) if owner.owner_id == api.id.to_string() && owner.lease_expires_at_unix_nanos > now_unix_nanos => { + owner.clone() + } + Some(owner) if owner.lease_expires_at_unix_nanos <= now_unix_nanos => { + let replacement = new_disposition_owner(&api, &proof, now_unix_nanos)?; + let mut next = current.disposition.clone(); + next.take_over(replacement.clone()).map_err(disposition_store_error)?; + current = save_recovery_disposition_step_fenced(api.clone(), ¤t, &next, &control_guard, &source_guard, &proof) + .await?; + replacement + } + Some(_) => { + return Ok(disposition_execution( + ¤t.disposition, + IlmRecoveryDispositionExecutionOutcome::AcceptedForRecovery, + )); + } + None => { + let owner = new_disposition_owner(&api, &proof, now_unix_nanos)?; + let mut next = current.disposition.clone(); + next.claim(owner.clone()).map_err(disposition_store_error)?; + current = save_recovery_disposition_step_fenced(api.clone(), ¤t, &next, &control_guard, &source_guard, &proof) + .await?; + owner + } + }; + if current.disposition.owner.as_ref() != Some(&owner) { + return Ok(disposition_execution( + ¤t.disposition, + IlmRecoveryDispositionExecutionOutcome::AcceptedForRecovery, + )); + } + if current.disposition.state == IlmRecoveryDispositionState::Prepared { + let mut next = current.disposition.clone(); + next.begin_applying().map_err(disposition_store_error)?; + current = + save_recovery_disposition_step_fenced(api.clone(), ¤t, &next, &control_guard, &source_guard, &proof).await?; + } + + for copy in current.disposition.identity.source_generation.copies.clone() { + if current.disposition.confirmed_absent.binary_search(©.authority).is_ok() { + continue; + } + ensure_owned_execution_fence(¤t.disposition, &owner, wall_clock_unix_nanos()?)?; + if !fences_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await { + return Err(Error::PreconditionFailed); + } + delete_exact_local_recovery_copy( + api.clone(), + ©, + ¤t.disposition.identity.source_generation.source_schema, + &control_guard, + &source_guard, + ) + .await?; + #[cfg(all(test, feature = "test-util"))] + maybe_inject_recovery_disposition_crash(RecoveryDispositionCrashStage::AfterLocalDelete)?; + if !fences_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await { + return Err(Error::PreconditionFailed); + } + ensure_owned_execution_fence(¤t.disposition, &owner, wall_clock_unix_nanos()?)?; + let mut next = current.disposition.clone(); + next.confirm_absent(copy.authority).map_err(disposition_store_error)?; + current = + save_recovery_disposition_step_fenced(api.clone(), ¤t, &next, &control_guard, &source_guard, &proof).await?; + if current.disposition.owner.as_ref() != Some(&owner) { + return Ok(disposition_execution( + ¤t.disposition, + IlmRecoveryDispositionExecutionOutcome::AcceptedForRecovery, + )); + } + ensure_owned_execution_fence(¤t.disposition, &owner, wall_clock_unix_nanos()?)?; + } + if current.disposition.state != IlmRecoveryDispositionState::Completed { + ensure_owned_execution_fence(¤t.disposition, &owner, wall_clock_unix_nanos()?)?; + abandon_recovery_control_fenced(api.clone(), ¤t.disposition, &control_guard, &source_guard, &proof).await?; + #[cfg(all(test, feature = "test-util"))] + maybe_inject_recovery_disposition_crash(RecoveryDispositionCrashStage::AfterControlAbandon)?; + ensure_owned_execution_fence(¤t.disposition, &owner, wall_clock_unix_nanos()?)?; + let mut next = current.disposition.clone(); + next.complete().map_err(disposition_store_error)?; + current = save_recovery_disposition_step_fenced(api, ¤t, &next, &control_guard, &source_guard, &proof).await?; + } + let outcome = if current.disposition.state == IlmRecoveryDispositionState::Completed { + IlmRecoveryDispositionExecutionOutcome::Completed + } else { + IlmRecoveryDispositionExecutionOutcome::AcceptedForRecovery + }; + Ok(disposition_execution(¤t.disposition, outcome)) +} + +async fn prepare_recovery_disposition( + api: Arc, + observation: &IlmRecoveryExportObservation, + export_id: &str, + export_content_sha256: &str, + actor_sha256: &str, + confirmed_at_unix_nanos: i64, +) -> EcstoreResult<(IlmRecoveryDisposition, IlmRecoveryExportFleetProofToken)> { + validate_sha256(export_id, "export ID is invalid").map_err(disposition_store_error)?; + validate_sha256(export_content_sha256, "export content checksum is invalid").map_err(disposition_store_error)?; + validate_sha256(actor_sha256, "actor digest is invalid").map_err(disposition_store_error)?; + if confirmed_at_unix_nanos <= 0 { + return Err(Error::PreconditionFailed); + } + let proof = acquire_ilm_recovery_export_fleet_proof() + .await + .ok_or(Error::PreconditionFailed)?; + if ilm_recovery_export_topology_generation(&proof) != observation.topology_generation + || ilm_recovery_export_member_epochs_sha256(&proof) != observation.member_epochs_sha256 + || !ilm_recovery_export_fleet_proof_matches(&proof).await + { + return Err(Error::PreconditionFailed); + } + let stored_export = load_recovery_export(api.clone(), export_id).await?; + if stored_export.content_sha256 != export_content_sha256 { + return Err(Error::PreconditionFailed); + } + let export = IlmRecoveryExport::decode(export_id, &stored_export.encoded)?; + if !export_matches_observation(&export, observation) { + return Err(Error::PreconditionFailed); + } + let observed_control = load_recovery_control(api, observation.protocol, &observation.control_id).await?; + let retained_predecessor_is_exact = observed_control.etag == observation.control_etag + && observed_control.control.revision == observation.control_revision + && observed_control.control.classification == IlmRecoveryClassification::RetainedAmbiguous + && observed_control.control.observed_source_generation == observation.source_generation; + let abandoned_successor_is_exact = observation.control_revision.checked_add(1) == Some(observed_control.control.revision) + && observed_control.control.classification == IlmRecoveryClassification::Abandoned + && observed_control.control.owner.is_none() + && observed_control.control.identity.protocol == observation.protocol + && observed_control.control.identity.canonical_source_path == observation.canonical_source_path + && observed_control.control.observed_source_generation == observation.source_generation; + let abandoned_control_sha256 = if retained_predecessor_is_exact { + expected_abandoned_control_sha256(&observed_control.control, &observation.source_generation)? + } else if abandoned_successor_is_exact { + let encoded = observed_control.control.encode().map_err(|_| Error::PreconditionFailed)?; + hex_sha256(&encoded, ToOwned::to_owned) + } else { + return Err(Error::PreconditionFailed); + }; + let action = IlmRecoveryDispositionAction::AbandonRemoteCleanup; + let identity = IlmRecoveryDispositionIdentity { + disposition_id: recovery_disposition_id(export_id, action).map_err(disposition_store_error)?, + protocol: observation.protocol, + action, + export_id: export_id.to_string(), + export_content_sha256: export_content_sha256.to_string(), + control_id: observation.control_id.clone(), + control_etag: observation.control_etag.clone(), + control_revision: observation.control_revision, + abandoned_control_sha256, + canonical_source_path: observation.canonical_source_path.clone(), + source_generation: observation.source_generation.clone(), + admitted_topology_generation: observation.topology_generation.clone(), + admitted_member_epochs_sha256: observation.member_epochs_sha256.clone(), + actor_sha256: actor_sha256.to_string(), + reason_code: IlmRecoveryDispositionReasonCode::LegacyRemoteCleanupAbandoned, + confirmed_at_unix_nanos, + }; + Ok(( + IlmRecoveryDisposition::new(identity, confirmed_at_unix_nanos).map_err(disposition_store_error)?, + proof, + )) +} + +fn export_matches_observation(export: &IlmRecoveryExport, observation: &IlmRecoveryExportObservation) -> bool { + export.control_id == observation.control_id + && export.protocol == observation.protocol + && export.control_etag == observation.control_etag + && export.control_revision == observation.control_revision + && export.classification == observation.classification + && export.canonical_source_path == observation.canonical_source_path + && export.source_generation == observation.source_generation + && export.topology_generation == observation.topology_generation + && export.member_epochs_sha256 == observation.member_epochs_sha256 +} + +fn validate_existing_request_binding( + disposition: &IlmRecoveryDisposition, + expected_identity: &IlmRecoveryDispositionIdentity, +) -> EcstoreResult<()> { + let mut expected_identity = expected_identity.clone(); + expected_identity.confirmed_at_unix_nanos = disposition.identity.confirmed_at_unix_nanos; + if disposition.requires_legacy_v1_migration() { + expected_identity.abandoned_control_sha256 = disposition.identity.abandoned_control_sha256.clone(); + } + if disposition.identity != expected_identity { + return Err(Error::PreconditionFailed); + } + Ok(()) +} + +fn expected_abandoned_control_sha256( + control: &IlmRecoveryControl, + source_generation: &IlmRecoverySourceGeneration, +) -> EcstoreResult { + let mut abandoned = control.clone(); + abandoned + .abandon_for_operator(source_generation) + .map_err(|_| Error::PreconditionFailed)?; + control + .validate_successor(&abandoned) + .map_err(|_| Error::PreconditionFailed)?; + let encoded = abandoned.encode().map_err(|_| Error::PreconditionFailed)?; + Ok(hex_sha256(&encoded, ToOwned::to_owned)) +} + +fn legacy_v1_migration_error(message: &'static str) -> Error { + Error::other(IlmRecoveryDispositionError::LegacyV1MigrationRequired(message)) +} + +async fn acquire_matching_disposition_proof( + disposition: &IlmRecoveryDisposition, +) -> EcstoreResult { + let proof = acquire_ilm_recovery_export_fleet_proof() + .await + .ok_or(Error::PreconditionFailed)?; + if ilm_recovery_export_topology_generation(&proof) != disposition.identity.admitted_topology_generation + || ilm_recovery_export_member_epochs_sha256(&proof) != disposition.identity.admitted_member_epochs_sha256 + || !ilm_recovery_export_fleet_proof_matches(&proof).await + { + return Err(Error::PreconditionFailed); + } + Ok(proof) +} + +async fn validate_live_disposition_inputs( + api: Arc, + disposition: &IlmRecoveryDisposition, + proof: &IlmRecoveryExportFleetProofToken, + no_lock: bool, +) -> EcstoreResult<()> { + if ilm_recovery_export_topology_generation(proof) != disposition.identity.admitted_topology_generation + || ilm_recovery_export_member_epochs_sha256(proof) != disposition.identity.admitted_member_epochs_sha256 + || !ilm_recovery_export_fleet_proof_matches(proof).await + { + return Err(Error::PreconditionFailed); + } + let stored_export = load_recovery_export(api.clone(), &disposition.identity.export_id).await?; + if stored_export.content_sha256 != disposition.identity.export_content_sha256 { + return Err(Error::PreconditionFailed); + } + let export = IlmRecoveryExport::decode(&stored_export.export_id, &stored_export.encoded)?; + if export.control_id != disposition.identity.control_id + || export.protocol != disposition.identity.protocol + || export.control_etag != disposition.identity.control_etag + || export.control_revision != disposition.identity.control_revision + || export.classification != IlmRecoveryClassification::RetainedAmbiguous + || export.canonical_source_path != disposition.identity.canonical_source_path + || export.source_generation != disposition.identity.source_generation + || export.topology_generation != disposition.identity.admitted_topology_generation + || export.member_epochs_sha256 != disposition.identity.admitted_member_epochs_sha256 + { + return Err(Error::PreconditionFailed); + } + let observed_control = if no_lock { + load_recovery_control_no_lock(api.clone(), disposition).await? + } else { + load_recovery_control(api.clone(), disposition.identity.protocol, &disposition.identity.control_id).await? + }; + let retained_generation_is_exact = observed_control.etag == disposition.identity.control_etag + && observed_control.control.revision == disposition.identity.control_revision + && observed_control.control.classification == IlmRecoveryClassification::RetainedAmbiguous + && observed_control.control.identity.canonical_source_path == disposition.identity.canonical_source_path + && observed_control.control.observed_source_generation == disposition.identity.source_generation; + let abandonment_is_durable = disposition.state == IlmRecoveryDispositionState::Applying + && disposition.confirmed_absent.len() == disposition.identity.source_generation.copies.len() + && control_is_abandoned_successor(disposition, &observed_control.control); + if !retained_generation_is_exact && !abandonment_is_durable { + return Err(Error::PreconditionFailed); + } + for copy in &disposition.identity.source_generation.copies { + let observed = read_local_recovery_copy(api.clone(), copy, no_lock).await?; + if disposition.confirmed_absent.binary_search(©.authority).is_ok() { + if observed.is_some() { + return Err(Error::PreconditionFailed); + } + } else { + match observed.as_ref() { + Some((data, etag)) + if etag == ©.etag + && u64::try_from(data.len()).ok() == Some(copy.encoded_len) + && hex_sha256(data, ToOwned::to_owned) == copy.content_sha256 => + { + validate_legacy_tier_delete_recovery_source( + ©.canonical_path, + &disposition.identity.source_generation.source_schema, + data, + )?; + } + None if disposition.state == IlmRecoveryDispositionState::Applying => { + // Applying is durable authorization for local-only removal. + // A crash may happen after delete but before its CAS progress + // revision, so restart may confirm that absence here. + } + _ => return Err(Error::PreconditionFailed), + } + } + } + if !ilm_recovery_export_fleet_proof_matches(proof).await { + return Err(Error::PreconditionFailed); + } + Ok(()) +} + +async fn load_recovery_control_no_lock( + api: Arc, + disposition: &IlmRecoveryDisposition, +) -> EcstoreResult { + let object = recovery_control_record_object_name(disposition.identity.protocol, &disposition.identity.control_id) + .map_err(|_| Error::PreconditionFailed)?; + let (encoded, metadata) = config_boundary::read_config_limited_preserve_empty_with_metadata( + api, + &object, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + MAX_ILM_RECOVERY_CONTROL_SIZE, + ) + .await?; + let etag = metadata + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or(Error::PreconditionFailed)?; + let control = + IlmRecoveryControl::decode(&disposition.identity.control_id, &encoded).map_err(|_| Error::PreconditionFailed)?; + Ok(super::recovery_control::ObservedIlmRecoveryControl { control, etag }) +} + +fn control_is_abandoned_successor(disposition: &IlmRecoveryDisposition, control: &IlmRecoveryControl) -> bool { + disposition.identity.control_revision.checked_add(1) == Some(control.revision) + && control.classification == IlmRecoveryClassification::Abandoned + && control.owner.is_none() + && control.identity.protocol == disposition.identity.protocol + && control.identity.canonical_source_path == disposition.identity.canonical_source_path + && control.observed_source_generation == disposition.identity.source_generation + && control + .encode() + .ok() + .is_some_and(|encoded| hex_sha256(&encoded, ToOwned::to_owned) == disposition.identity.abandoned_control_sha256) +} + +async fn abandon_recovery_control_fenced( + api: Arc, + disposition: &IlmRecoveryDisposition, + control_guard: &rustfs_lock::NamespaceLockGuard, + source_guard: &rustfs_lock::NamespaceLockGuard, + proof: &IlmRecoveryExportFleetProofToken, +) -> EcstoreResult<()> { + if disposition.state != IlmRecoveryDispositionState::Applying + || disposition.confirmed_absent.len() != disposition.identity.source_generation.copies.len() + { + return Err(Error::PreconditionFailed); + } + let current = load_recovery_control_no_lock(api.clone(), disposition).await?; + let object = recovery_control_record_object_name(disposition.identity.protocol, &disposition.identity.control_id) + .map_err(|_| Error::PreconditionFailed)?; + if control_is_abandoned_successor(disposition, ¤t.control) { + if control_guard.is_lock_lost() || source_guard.is_lock_lost() || !ilm_recovery_export_fleet_proof_matches(proof).await { + return Err(Error::PreconditionFailed); + } + let encoded = current.control.encode().map_err(|_| Error::PreconditionFailed)?; + api.record_durable_ilm_decommission_terminal(&object, &encoded).await?; + if control_guard.is_lock_lost() || source_guard.is_lock_lost() { + return Err(Error::PreconditionFailed); + } + return Ok(()); + } + if current.etag != disposition.identity.control_etag + || current.control.revision != disposition.identity.control_revision + || current.control.classification != IlmRecoveryClassification::RetainedAmbiguous + || current.control.identity.protocol != disposition.identity.protocol + || current.control.identity.canonical_source_path != disposition.identity.canonical_source_path + || current.control.observed_source_generation != disposition.identity.source_generation + { + return Err(Error::PreconditionFailed); + } + let mut next = current.control.clone(); + next.abandon_for_operator(&disposition.identity.source_generation) + .map_err(|_| Error::PreconditionFailed)?; + current + .control + .validate_successor(&next) + .map_err(|_| Error::PreconditionFailed)?; + if control_guard.is_lock_lost() || source_guard.is_lock_lost() || !ilm_recovery_export_fleet_proof_matches(proof).await { + return Err(Error::PreconditionFailed); + } + let encoded = next.encode().map_err(|_| Error::PreconditionFailed)?; + let mut options = ObjectOptions { + max_parity: true, + no_lock: true, + write_completion: WriteCompletion::TailDrained, + http_preconditions: Some(HTTPPreconditions { + if_match: Some(current.etag), + ..Default::default() + }), + ..Default::default() + }; + options.add_namespace_lock_guard(control_guard); + options.add_namespace_lock_guard(source_guard); + let write_result = config_boundary::save_config_with_opts(api.clone(), &object, encoded, &options).await; + let observed = match load_recovery_control_no_lock(api.clone(), disposition).await { + Ok(observed) if control_is_abandoned_successor(disposition, &observed.control) => observed, + Ok(_) => return Err(Error::PreconditionFailed), + Err(read_err) => return Err(write_result.err().unwrap_or(read_err)), + }; + if control_guard.is_lock_lost() || source_guard.is_lock_lost() || !ilm_recovery_export_fleet_proof_matches(proof).await { + return Err(Error::PreconditionFailed); + } + let observed_encoded = observed.control.encode().map_err(|_| Error::PreconditionFailed)?; + api.record_durable_ilm_decommission_terminal(&object, &observed_encoded) + .await?; + if control_guard.is_lock_lost() || source_guard.is_lock_lost() { + return Err(Error::PreconditionFailed); + } + Ok(()) +} + +async fn read_local_recovery_copy( + api: Arc, + copy: &IlmRecoverySourceCopy, + no_lock: bool, +) -> EcstoreResult, String)>> { + let set = recovery_copy_set(&api, ©.authority)?; + match config_boundary::read_config_limited_preserve_empty_with_metadata( + set, + ©.canonical_path, + &ObjectOptions { + no_lock, + ..Default::default() + }, + MAX_LEGACY_TIER_DELETE_SOURCE_SIZE, + ) + .await + { + Ok((data, metadata)) => { + let etag = metadata + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or(Error::PreconditionFailed)?; + Ok(Some((data, etag))) + } + Err(err) if disposition_is_missing(&err) => Ok(None), + Err(err) => Err(err), + } +} + +fn recovery_copy_set(api: &ECStore, authority: &str) -> EcstoreResult> { + let (pool_index, set_index) = parse_recovery_copy_authority(authority)?; + api.all_set_disks() + .into_iter() + .find(|set| set.pool_index == pool_index && set.set_index == set_index) + .ok_or(Error::PreconditionFailed) +} + +fn parse_recovery_copy_authority(authority: &str) -> EcstoreResult<(usize, usize)> { + let Some((pool, set)) = authority.strip_prefix("pool-").and_then(|value| value.split_once("/set-")) else { + return Err(Error::PreconditionFailed); + }; + let pool_index = pool.parse::().map_err(|_| Error::PreconditionFailed)?; + let set_index = set.parse::().map_err(|_| Error::PreconditionFailed)?; + if format!("pool-{pool_index}/set-{set_index}") != authority { + return Err(Error::PreconditionFailed); + } + Ok((pool_index, set_index)) +} + +async fn delete_exact_local_recovery_copy( + api: Arc, + copy: &IlmRecoverySourceCopy, + source_schema: &str, + control_guard: &rustfs_lock::NamespaceLockGuard, + source_guard: &rustfs_lock::NamespaceLockGuard, +) -> EcstoreResult<()> { + let set = recovery_copy_set(&api, ©.authority)?; + let Some((data, etag)) = read_local_recovery_copy(api.clone(), copy, true).await? else { + return Ok(()); + }; + if etag != copy.etag + || u64::try_from(data.len()).ok() != Some(copy.encoded_len) + || hex_sha256(&data, ToOwned::to_owned) != copy.content_sha256 + { + return Err(Error::PreconditionFailed); + } + validate_legacy_tier_delete_recovery_source(©.canonical_path, source_schema, &data)?; + let mut options = ObjectOptions { + no_lock: true, + http_preconditions: Some(HTTPPreconditions { + if_match: Some(copy.etag.clone()), + ..Default::default() + }), + ..Default::default() + }; + options.add_namespace_lock_guard(control_guard); + options.add_namespace_lock_guard(source_guard); + let delete = set.delete_object(RUSTFS_META_BUCKET, ©.canonical_path, options).await; + match read_local_recovery_copy(api, copy, true).await { + Ok(None) => Ok(()), + Ok(Some((observed, observed_etag))) => { + if observed_etag != copy.etag || observed != data { + Err(Error::PreconditionFailed) + } else { + Err(delete.err().unwrap_or(Error::PreconditionFailed)) + } + } + Err(read_err) => Err(delete.err().unwrap_or(read_err)), + } +} + +fn new_disposition_owner( + api: &ECStore, + proof: &IlmRecoveryExportFleetProofToken, + now_unix_nanos: i64, +) -> EcstoreResult { + let lease_expires_at_unix_nanos = now_unix_nanos + .checked_add(DISPOSITION_OWNER_LEASE_NANOS) + .ok_or(Error::PreconditionFailed)?; + Ok(IlmRecoveryDispositionOwnerLease { + owner_id: api.id.to_string(), + owner_epoch: Uuid::new_v4(), + lease_acquired_at_unix_nanos: now_unix_nanos, + lease_expires_at_unix_nanos, + topology_generation: ilm_recovery_export_topology_generation(proof), + member_epochs_sha256: ilm_recovery_export_member_epochs_sha256(proof), + }) +} + +fn ensure_owned_execution_fence( + disposition: &IlmRecoveryDisposition, + owner: &IlmRecoveryDispositionOwnerLease, + now_unix_nanos: i64, +) -> EcstoreResult<()> { + if disposition.state != IlmRecoveryDispositionState::Applying + || disposition.owner.as_ref() != Some(owner) + || owner.lease_expires_at_unix_nanos <= now_unix_nanos + { + return Err(Error::PreconditionFailed); + } + Ok(()) +} + +async fn save_recovery_disposition_step_fenced( + api: Arc, + current: &ObservedIlmRecoveryDisposition, + next: &IlmRecoveryDisposition, + control_guard: &rustfs_lock::NamespaceLockGuard, + source_guard: &rustfs_lock::NamespaceLockGuard, + proof: &IlmRecoveryExportFleetProofToken, +) -> EcstoreResult { + current + .disposition + .validate_successor(next) + .map_err(disposition_store_error)?; + let protocol = current.disposition.identity.protocol; + let disposition_id = ¤t.disposition.identity.disposition_id; + let object = recovery_disposition_record_object_name(protocol, disposition_id).map_err(disposition_store_error)?; + let authoritative = load_recovery_disposition(api.clone(), protocol, disposition_id).await?; + if authoritative != *current { + if authoritative.disposition.is_same_or_later_generation_of(next) { + if control_guard.is_lock_lost() + || source_guard.is_lock_lost() + || !ilm_recovery_export_fleet_proof_matches(proof).await + { + return Err(Error::PreconditionFailed); + } + record_disposition_decommission_checkpoint(api.as_ref(), &object, &authoritative).await?; + if control_guard.is_lock_lost() || source_guard.is_lock_lost() { + return Err(Error::PreconditionFailed); + } + return Ok(authoritative); + } + return Err(Error::PreconditionFailed); + } + if control_guard.is_lock_lost() || source_guard.is_lock_lost() || !ilm_recovery_export_fleet_proof_matches(proof).await { + return Err(Error::PreconditionFailed); + } + let encoded = next.encode().map_err(disposition_store_error)?; + let mut options = ObjectOptions { + max_parity: true, + write_completion: WriteCompletion::TailDrained, + http_preconditions: Some(HTTPPreconditions { + if_match: Some(current.etag.clone()), + ..Default::default() + }), + ..Default::default() + }; + options.add_namespace_lock_guard(control_guard); + options.add_namespace_lock_guard(source_guard); + let write_result = config_boundary::save_config_with_opts(api.clone(), &object, encoded, &options).await; + let observed = match load_recovery_disposition(api.clone(), protocol, disposition_id).await { + Ok(observed) if observed.disposition.is_same_or_later_generation_of(next) => observed, + Ok(_) => return Err(Error::PreconditionFailed), + Err(read_err) => return Err(write_result.err().unwrap_or(read_err)), + }; + if control_guard.is_lock_lost() || source_guard.is_lock_lost() || !ilm_recovery_export_fleet_proof_matches(proof).await { + return Err(Error::PreconditionFailed); + } + record_disposition_decommission_checkpoint(api.as_ref(), &object, &observed).await?; + if control_guard.is_lock_lost() || source_guard.is_lock_lost() { + return Err(Error::PreconditionFailed); + } + Ok(observed) +} + +fn disposition_execution( + disposition: &IlmRecoveryDisposition, + outcome: IlmRecoveryDispositionExecutionOutcome, +) -> IlmRecoveryDispositionExecution { + IlmRecoveryDispositionExecution { + disposition_id: disposition.identity.disposition_id.clone(), + state: disposition.state, + outcome, + confirmed_absent_copy_count: disposition.confirmed_absent.len(), + source_copy_count: disposition.identity.source_generation.copies.len(), + } +} + +fn wall_clock_unix_nanos() -> EcstoreResult { + i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()).map_err(|_| Error::PreconditionFailed) +} + +pub(crate) async fn load_recovery_disposition( + api: Arc, + protocol: IlmRecoveryProtocol, + disposition_id: &str, +) -> EcstoreResult { + load_recovery_disposition_with_options(api, protocol, disposition_id, false).await +} + +pub(crate) async fn load_recovery_disposition_no_lock( + api: Arc, + protocol: IlmRecoveryProtocol, + disposition_id: &str, +) -> EcstoreResult { + load_recovery_disposition_with_options(api, protocol, disposition_id, true).await +} + +async fn load_recovery_disposition_with_options( + api: Arc, + protocol: IlmRecoveryProtocol, + disposition_id: &str, + no_lock: bool, ) -> EcstoreResult { let object = recovery_disposition_record_object_name(protocol, disposition_id).map_err(disposition_store_error)?; let (encoded, metadata) = config_boundary::read_config_limited_preserve_empty_with_metadata( api, &object, - &ObjectOptions::default(), + &ObjectOptions { + no_lock, + ..Default::default() + }, MAX_ILM_RECOVERY_DISPOSITION_SIZE, ) .await?; @@ -794,52 +1829,6 @@ pub async fn load_recovery_disposition( }) } -pub async fn save_recovery_disposition_if_current( - api: Arc, - current: &ObservedIlmRecoveryDisposition, - next: &IlmRecoveryDisposition, -) -> EcstoreResult { - current - .disposition - .validate_successor(next) - .map_err(disposition_store_error)?; - let protocol = current.disposition.identity.protocol; - let disposition_id = ¤t.disposition.identity.disposition_id; - let object = recovery_disposition_record_object_name(protocol, disposition_id).map_err(disposition_store_error)?; - let authoritative = load_recovery_disposition(api.clone(), protocol, disposition_id).await?; - if authoritative != *current { - if authoritative.disposition.is_same_or_later_generation_of(next) { - record_disposition_decommission_checkpoint(api.as_ref(), &object, &authoritative).await?; - return Ok(authoritative); - } - return Err(Error::PreconditionFailed); - } - let encoded = next.encode().map_err(disposition_store_error)?; - let write_result = config_boundary::save_config_with_opts( - api.clone(), - &object, - encoded.clone(), - &ObjectOptions { - max_parity: true, - write_completion: WriteCompletion::TailDrained, - http_preconditions: Some(HTTPPreconditions { - if_match: Some(current.etag.clone()), - ..Default::default() - }), - ..Default::default() - }, - ) - .await; - match load_recovery_disposition(api.clone(), protocol, disposition_id).await { - Ok(observed) if observed.disposition.is_same_or_later_generation_of(next) => { - record_disposition_decommission_checkpoint(api.as_ref(), &object, &observed).await?; - Ok(observed) - } - Ok(_) => Err(Error::PreconditionFailed), - Err(read_err) => Err(write_result.err().unwrap_or(read_err)), - } -} - async fn record_disposition_decommission_checkpoint( api: &ECStore, object: &str, @@ -904,7 +1893,6 @@ fn disposition_is_missing(err: &Error) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::bucket::lifecycle::recovery_control::IlmRecoverySourceCopy; fn digest(value: &[u8]) -> String { hex_sha256(value, ToOwned::to_owned) @@ -947,6 +1935,7 @@ mod tests { control_id, control_etag: "control-etag".to_string(), control_revision: 1, + abandoned_control_sha256: digest(b"abandoned-control"), canonical_source_path: source_path, source_generation, admitted_topology_generation: digest(b"topology"), @@ -958,6 +1947,41 @@ mod tests { IlmRecoveryDisposition::new(identity, 1_000_000_001).expect("disposition should be valid") } + fn encode_v1_fixture(disposition: &IlmRecoveryDisposition) -> Vec { + let legacy = IlmRecoveryDispositionV1 { + identity: IlmRecoveryDispositionIdentityV1 { + disposition_id: disposition.identity.disposition_id.clone(), + protocol: disposition.identity.protocol, + action: disposition.identity.action, + export_id: disposition.identity.export_id.clone(), + export_content_sha256: disposition.identity.export_content_sha256.clone(), + control_id: disposition.identity.control_id.clone(), + control_etag: disposition.identity.control_etag.clone(), + control_revision: disposition.identity.control_revision, + canonical_source_path: disposition.identity.canonical_source_path.clone(), + source_generation: disposition.identity.source_generation.clone(), + admitted_topology_generation: disposition.identity.admitted_topology_generation.clone(), + admitted_member_epochs_sha256: disposition.identity.admitted_member_epochs_sha256.clone(), + actor_sha256: disposition.identity.actor_sha256.clone(), + reason_code: disposition.identity.reason_code, + confirmed_at_unix_nanos: disposition.identity.confirmed_at_unix_nanos, + }, + created_at_unix_nanos: disposition.created_at_unix_nanos, + retain_until_unix_nanos: disposition.retain_until_unix_nanos, + revision: disposition.revision, + state: disposition.state, + owner: disposition.owner.clone(), + confirmed_absent: disposition.confirmed_absent.clone(), + }; + let disposition_bytes = serde_json::to_vec(&legacy).expect("v1 disposition should encode"); + serde_json::to_vec(&PersistedIlmRecoveryDispositionV1 { + schema: ILM_RECOVERY_DISPOSITION_SCHEMA_V1.to_string(), + content_sha256: digest(&disposition_bytes), + disposition: legacy, + }) + .expect("v1 envelope should encode") + } + fn owner(epoch: u128, acquired: i64) -> IlmRecoveryDispositionOwnerLease { IlmRecoveryDispositionOwnerLease { owner_id: "node-a".to_string(), @@ -992,6 +2016,10 @@ mod tests { fn checksum_envelope_round_trips_and_rejects_tampering() { let disposition = sample_disposition(); let encoded = disposition.encode().expect("disposition should encode"); + assert_eq!( + serde_json::from_slice::(&encoded).unwrap()["schema"], + ILM_RECOVERY_DISPOSITION_SCHEMA + ); assert_eq!( IlmRecoveryDisposition::decode(&disposition.identity.disposition_id, &encoded).expect("disposition should decode"), disposition @@ -1036,6 +2064,49 @@ mod tests { assert!(rebound.validate().is_err(), "export ID must bind the exact control and source generation"); } + #[test] + fn v1_envelope_decodes_strictly_and_requires_safe_migration() { + let disposition = sample_disposition(); + let encoded = encode_v1_fixture(&disposition); + let decoded = IlmRecoveryDisposition::decode(&disposition.identity.disposition_id, &encoded) + .expect("valid v1 records should remain distinguishable from corruption"); + assert!(decoded.requires_legacy_v1_migration()); + assert!(matches!(decoded.encode(), Err(IlmRecoveryDispositionError::LegacyV1MigrationRequired(_)))); + let path = + recovery_disposition_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, &disposition.identity.disposition_id) + .unwrap(); + let checkpoint = decode_recovery_disposition_checkpoint(&path, &encoded).unwrap(); + let persisted: PersistedIlmRecoveryDispositionV1 = serde_json::from_slice(&encoded).unwrap(); + assert_eq!(checkpoint.identity_sha256, checkpoint_hash(&persisted.disposition.identity).unwrap()); + assert_eq!(checkpoint.retain_until_unix_nanos, persisted.disposition.retain_until_unix_nanos); + + let mut tampered: serde_json::Value = serde_json::from_slice(&encoded).unwrap(); + tampered["disposition"]["identity"]["actor_sha256"] = serde_json::Value::String(digest(b"other-actor")); + assert!(matches!( + IlmRecoveryDisposition::decode(&disposition.identity.disposition_id, &serde_json::to_vec(&tampered).unwrap()), + Err(IlmRecoveryDispositionError::ChecksumMismatch) + )); + + let mut unknown: serde_json::Value = serde_json::from_slice(&encoded).unwrap(); + unknown["disposition"]["identity"]["unknown"] = serde_json::Value::Bool(true); + assert!(matches!( + IlmRecoveryDisposition::decode(&disposition.identity.disposition_id, &serde_json::to_vec(&unknown).unwrap()), + Err(IlmRecoveryDispositionError::Json(_)) + )); + + let mut completed = disposition; + completed.claim(owner(1, 2_000_000_000)).unwrap(); + completed.begin_applying().unwrap(); + completed.confirm_absent("pool-0/set-0").unwrap(); + completed.confirm_absent("pool-1/set-0").unwrap(); + completed.complete().unwrap(); + let completed = IlmRecoveryDisposition::decode(&completed.identity.disposition_id, &encode_v1_fixture(&completed)) + .expect("completed v1 record should decode into fail-closed quarantine"); + assert!(completed.requires_legacy_v1_migration()); + assert_eq!(completed.state, IlmRecoveryDispositionState::Completed); + assert_eq!(completed.retain_until_unix_nanos, i64::MAX); + } + #[test] fn state_and_absence_progress_successors_are_strict() { let prepared = sample_disposition(); @@ -1163,6 +2234,112 @@ mod tests { assert!(disposition.validate().is_err()); } + #[test] + fn recovery_copy_authority_parser_is_strict() { + assert_eq!( + parse_recovery_copy_authority("pool-0/set-17").expect("canonical authority should parse"), + (0, 17) + ); + for invalid in [ + "pool-00/set-17", + "pool-0/set-017", + "pool-0/set-17/extra", + "pool-x/set-1", + "pool-1/set-x", + "set-0/pool-1", + "", + ] { + assert!(parse_recovery_copy_authority(invalid).is_err(), "authority must fail closed: {invalid}"); + } + } + + #[test] + fn execution_fence_requires_exact_live_owner() { + let mut disposition = sample_disposition(); + let active_owner = owner(1, 2_000_000_000); + disposition.claim(active_owner.clone()).expect("owner claim should succeed"); + disposition.begin_applying().expect("applying transition should succeed"); + assert!(ensure_owned_execution_fence(&disposition, &active_owner, 2_000_000_999).is_ok()); + assert!(ensure_owned_execution_fence(&disposition, &active_owner, 2_000_001_000).is_err()); + assert!(ensure_owned_execution_fence(&disposition, &owner(2, 2_000_000_000), 2_000_000_999).is_err()); + } + + #[test] + fn source_copy_size_is_bounded_before_destructive_execution() { + let mut disposition = sample_disposition(); + disposition.identity.source_generation.copies[0].encoded_len = 64 * 1024 + 1; + assert!(disposition.validate().is_err()); + } + + #[test] + fn abandoned_control_is_a_valid_crash_recovery_bridge_to_completion() { + use crate::bucket::lifecycle::recovery_control::{IlmRecoveryControlIdentity, IlmRecoveryErrorCode}; + + let mut disposition = sample_disposition(); + disposition + .claim(owner(1, 2_000_000_000)) + .expect("owner claim should succeed"); + disposition.begin_applying().expect("applying transition should succeed"); + disposition + .confirm_absent("pool-0/set-0") + .expect("first absence should be recorded"); + disposition + .confirm_absent("pool-1/set-0") + .expect("second absence should be recorded"); + + let mut control = IlmRecoveryControl::new( + IlmRecoveryControlIdentity { + protocol: IlmRecoveryProtocol::TierDeleteJournal, + canonical_source_path: disposition.identity.canonical_source_path.clone(), + stable_operation_identity: "legacy-operation".to_string(), + record_class: "tier_delete_journal_v2".to_string(), + }, + disposition.identity.source_generation.clone(), + IlmRecoveryClassification::RetainedAmbiguous, + 1_000_000_000, + IlmRecoveryErrorCode::OperatorDispositionRequired, + ) + .expect("retained control should build"); + assert_eq!(control.revision, disposition.identity.control_revision); + control + .abandon_for_operator(&disposition.identity.source_generation) + .expect("exact source generation should be abandoned"); + disposition.identity.abandoned_control_sha256 = + hex_sha256(&control.encode().expect("abandoned control should encode"), ToOwned::to_owned); + assert!(control_is_abandoned_successor(&disposition, &control)); + + let mut completed = disposition.clone(); + completed.complete().expect("completed transition should succeed"); + disposition + .validate_successor(&completed) + .expect("completed disposition should be a valid successor"); + } + + #[test] + fn completed_replay_binding_requires_the_original_actor() { + let disposition = sample_disposition(); + assert!(validate_existing_request_binding(&disposition, &disposition.identity).is_ok()); + + let mut later_retry = disposition.identity.clone(); + later_retry.confirmed_at_unix_nanos += 1; + assert!( + validate_existing_request_binding(&disposition, &later_retry).is_ok(), + "server request time must not make a stable replay identity diverge" + ); + + let mut wrong_identity = disposition.identity.clone(); + wrong_identity.actor_sha256 = digest(b"different-actor"); + assert!(validate_existing_request_binding(&disposition, &wrong_identity).is_err()); + + let mut wrong_identity = disposition.identity.clone(); + wrong_identity.abandoned_control_sha256 = digest(b"different-abandoned-control"); + assert!(validate_existing_request_binding(&disposition, &wrong_identity).is_err()); + + let mut wrong_identity = disposition.identity.clone(); + wrong_identity.admitted_member_epochs_sha256 = digest(b"different-fleet"); + assert!(validate_existing_request_binding(&disposition, &wrong_identity).is_err()); + } + #[test] fn encoded_disposition_is_bounded_to_sixteen_kibibytes() { let mut disposition = sample_disposition(); diff --git a/crates/ecstore/src/bucket/lifecycle/recovery_disposition_runtime.rs b/crates/ecstore/src/bucket/lifecycle/recovery_disposition_runtime.rs new file mode 100644 index 000000000..e9398262e --- /dev/null +++ b/crates/ecstore/src/bucket/lifecycle/recovery_disposition_runtime.rs @@ -0,0 +1,1133 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashSet; +use std::future::Future; +use std::sync::{ + Arc, LazyLock, + atomic::{AtomicUsize, Ordering}, +}; +use std::time::Duration; + +use futures::{StreamExt, stream}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, TryAcquireError}; +use tokio_util::sync::CancellationToken; + +use super::config_boundary; +use super::recovery_control::{IlmRecoveryClassification, IlmRecoveryProtocol, list_recovery_controls}; +use super::recovery_disposition::{ + CreatedIlmRecoveryDisposition, ILM_RECOVERY_DISPOSITION_PREFIX, IlmRecoveryDisposition, IlmRecoveryDispositionState, + ObservedIlmRecoveryDisposition, create_recovery_disposition_if_absent, load_recovery_disposition, + load_recovery_disposition_no_lock, recovery_disposition_id_from_record_object_name, recovery_disposition_record_object_name, + resume_recovery_disposition, +}; +use crate::disk::RUSTFS_META_BUCKET; +use crate::error::{Error, Result}; +use crate::object_api::ObjectOptions; +use crate::storage_api_contracts::{list::ListOperations as _, namespace::NamespaceLocking as _}; +use crate::store::ECStore; + +pub(crate) const MAX_ILM_RECOVERY_DISPOSITIONS: usize = 10_000; +pub(crate) const MAX_ILM_RECOVERY_DISPOSITION_BYTES: u64 = 10_000_u64 * 16_u64 * 1024_u64; +pub(crate) const MAX_ACTOR_DISPOSITIONS_PER_MINUTE: usize = 10; +pub(crate) const MAX_CLUSTER_DISPOSITIONS_PER_MINUTE: usize = 100; +pub(crate) const MAX_ACTIVE_RECOVERY_DISPOSITIONS: usize = 8; +pub(crate) const DEFAULT_RECOVERY_DISPOSITION_PASS_LIMIT: usize = MAX_ACTIVE_RECOVERY_DISPOSITIONS; + +const RECOVERY_DISPOSITION_ADMISSION_LOCK: &str = "ilm/recovery-admission/disposition.lock"; +const RECOVERY_DISPOSITION_RECENT_WINDOW_NANOS: i64 = 60 * 1_000_000_000; +const RECOVERY_DISPOSITION_PAGE_LIST_TIMEOUT: Duration = Duration::from_secs(15); +const RECOVERY_DISPOSITION_ENTRY_TIMEOUT: Duration = Duration::from_secs(30); +const RECOVERY_DISPOSITION_INVENTORY_TIMEOUT: Duration = Duration::from_secs(60); +const RECOVERY_DISPOSITION_PASS_TIMEOUT: Duration = Duration::from_secs(60); +const RECOVERY_DISPOSITION_MAINTENANCE_INTERVAL: Duration = Duration::from_secs(60); +const RECOVERY_DISPOSITION_METRICS_INTERVAL: Duration = Duration::from_secs(5 * 60); +const METRIC_RECOVERY_DISPOSITION_RECORDS: &str = "rustfs_ilm_recovery_disposition_records"; +const METRIC_RECOVERY_DISPOSITION_BYTES: &str = "rustfs_ilm_recovery_disposition_bytes"; +const METRIC_RECOVERY_DISPOSITION_INFLIGHT: &str = "rustfs_ilm_recovery_disposition_inflight"; +const METRIC_RECOVERY_DISPOSITION_OPERATIONS_TOTAL: &str = "rustfs_ilm_recovery_disposition_operations_total"; +const METRIC_RECOVERY_CONTROLS: &str = "rustfs_ilm_recovery_controls"; +const METRIC_RECOVERY_CONTROL_SCAN_INCOMPLETE: &str = "rustfs_ilm_recovery_control_scan_incomplete"; + +static RECOVERY_DISPOSITION_EXECUTION_LIMIT: LazyLock> = + LazyLock::new(|| Arc::new(Semaphore::new(MAX_ACTIVE_RECOVERY_DISPOSITIONS))); +static RECOVERY_DISPOSITION_INFLIGHT: AtomicUsize = AtomicUsize::new(0); + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct IlmRecoveryDispositionInventory { + pub(crate) count: usize, + pub(crate) bytes: u64, + pub(crate) prepared: usize, + pub(crate) applying: usize, + pub(crate) completed: usize, + pub(crate) corrupt: usize, + pub(crate) incomplete: bool, + creations: Vec<(i64, String)>, +} + +impl IlmRecoveryDispositionInventory { + fn active(&self) -> usize { + self.prepared.saturating_add(self.applying) + } + + fn check_admission(&self, actor_sha256: &str, candidate_len: usize, now_unix_nanos: i64) -> Result<()> { + if self.incomplete || self.corrupt > 0 || self.creations.iter().any(|(created_at, _)| *created_at > now_unix_nanos) { + return Err(Error::PreconditionFailed); + } + let recent_after = now_unix_nanos.saturating_sub(RECOVERY_DISPOSITION_RECENT_WINDOW_NANOS); + let cluster_recent = self + .creations + .iter() + .filter(|(created_at, _)| *created_at > recent_after) + .count(); + let actor_recent = self + .creations + .iter() + .filter(|(created_at, actor)| *created_at > recent_after && actor == actor_sha256) + .count(); + check_recovery_disposition_admission(self.count, self.bytes, self.active(), actor_recent, cluster_recent, candidate_len) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct IlmRecoveryControlClassificationCounts { + pub(crate) retrying: usize, + pub(crate) retained_ambiguous: usize, + pub(crate) corrupt: usize, + pub(crate) operator_required: usize, + pub(crate) abandoned: usize, + pub(crate) terminal: usize, + pub(crate) incomplete: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct RecoveryDispositionMaintenanceStats { + pub(crate) scanned: u64, + pub(crate) resumed: u64, + pub(crate) completed: u64, + pub(crate) replayed: u64, + pub(crate) garbage_collected: u64, + pub(crate) retained: u64, + pub(crate) corrupt: u64, + pub(crate) failed: u64, + pub(crate) next_marker: Option, + pub(crate) truncated: bool, +} + +#[derive(Debug)] +enum BoundedEntryTaskResult { + Completed(T), + TimedOut, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RecoveryDispositionMaintenanceEntryOutcome { + Missing, + Corrupt, + Retained, + GarbageCollected, + Failed, + Resumed, + Completed, + Replayed, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct RecoveryDispositionMaintenanceCursor { + marker: Option, + wait_for_interval: bool, +} + +pub(crate) struct RecoveryDispositionExecutionPermit<'a> { + _permit: OwnedSemaphorePermit, + inflight: &'a AtomicUsize, + record_metrics: bool, +} + +impl Drop for RecoveryDispositionExecutionPermit<'_> { + fn drop(&mut self) { + let inflight = self.inflight.fetch_sub(1, Ordering::AcqRel).saturating_sub(1); + if self.record_metrics { + metrics::gauge!(METRIC_RECOVERY_DISPOSITION_INFLIGHT).set(metric_count(inflight)); + } + } +} + +fn try_acquire_recovery_disposition_execution_permit( + execution_limit: Arc, + inflight: &AtomicUsize, + record_metrics: bool, +) -> Result> { + let permit = execution_limit.try_acquire_owned().map_err(|err| match err { + TryAcquireError::NoPermits => { + if record_metrics { + metrics::counter!( + METRIC_RECOVERY_DISPOSITION_OPERATIONS_TOTAL, + "operation" => "execution_admission", + "outcome" => "rejected" + ) + .increment(1); + } + Error::SlowDown + } + TryAcquireError::Closed => Error::other("ILM recovery disposition execution limiter is closed"), + })?; + let current_inflight = inflight.fetch_add(1, Ordering::AcqRel).saturating_add(1); + if record_metrics { + metrics::gauge!(METRIC_RECOVERY_DISPOSITION_INFLIGHT).set(metric_count(current_inflight)); + } + Ok(RecoveryDispositionExecutionPermit { + _permit: permit, + inflight, + record_metrics, + }) +} + +pub(crate) async fn acquire_recovery_disposition_execution_permit() -> Result> { + try_acquire_recovery_disposition_execution_permit( + Arc::clone(&RECOVERY_DISPOSITION_EXECUTION_LIMIT), + &RECOVERY_DISPOSITION_INFLIGHT, + true, + ) +} + +async fn run_bounded_entry_tasks( + tasks: I, + concurrency: usize, + timeout: Duration, + cancel_token: &CancellationToken, +) -> Result>> +where + I: IntoIterator + Send + 'static, + I::IntoIter: Send, + F: Future + Send + 'static, + T: Send + 'static, +{ + if concurrency == 0 { + return Err(Error::other("ILM recovery disposition task concurrency must be positive")); + } + let mut pending = stream::iter(tasks) + .map(|task| async move { + match tokio::time::timeout(timeout, task).await { + Ok(result) => BoundedEntryTaskResult::Completed(result), + Err(_) => BoundedEntryTaskResult::TimedOut, + } + }) + .buffer_unordered(concurrency); + let mut results = Vec::new(); + loop { + let next = tokio::select! { + biased; + _ = cancel_token.cancelled() => return Err(Error::OperationCanceled), + next = pending.next() => next, + }; + match next { + Some(result) => results.push(result), + None => return Ok(results), + } + } +} + +/// Creates or replays a disposition under the cluster-wide admission lock. +/// +/// Lock order is admission, then the control/source locks acquired by +/// `create_recovery_disposition_if_absent`. Existing records bypass inventory +/// admission so an idempotent replay can still converge when the namespace is +/// at its limit. +pub(crate) async fn create_recovery_disposition_with_admission( + api: Arc, + disposition: &IlmRecoveryDisposition, + now_unix_nanos: i64, +) -> Result { + disposition.validate().map_err(Error::other)?; + let protocol = disposition.identity.protocol; + let disposition_id = &disposition.identity.disposition_id; + match load_recovery_disposition(api.clone(), protocol, disposition_id).await { + Ok(_) => return create_recovery_disposition_if_absent(api, disposition).await, + Err(err) if disposition_is_missing(&err) => {} + Err(err) => return Err(err), + } + + let lock = api + .new_ns_lock(RUSTFS_META_BUCKET, RECOVERY_DISPOSITION_ADMISSION_LOCK) + .await?; + let admission_guard = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?; + if admission_guard.is_lock_lost() { + return Err(Error::PreconditionFailed); + } + match load_recovery_disposition(api.clone(), protocol, disposition_id).await { + Ok(_) => return create_recovery_disposition_if_absent(api, disposition).await, + Err(err) if disposition_is_missing(&err) => {} + Err(err) => return Err(err), + } + + let inventory = tokio::time::timeout( + RECOVERY_DISPOSITION_INVENTORY_TIMEOUT, + collect_recovery_disposition_inventory(api.clone()), + ) + .await + .map_err(|_| Error::Timeout)??; + record_recovery_disposition_inventory_metrics(&inventory); + let candidate_len = disposition.encode().map_err(Error::other)?.len(); + if let Err(err) = inventory.check_admission(&disposition.identity.actor_sha256, candidate_len, now_unix_nanos) { + metrics::counter!( + METRIC_RECOVERY_DISPOSITION_OPERATIONS_TOTAL, + "operation" => "create_admission", + "outcome" => "rejected" + ) + .increment(1); + return Err(err); + } + if admission_guard.is_lock_lost() { + return Err(Error::PreconditionFailed); + } + let result = create_recovery_disposition_if_absent(api, disposition).await; + metrics::counter!( + METRIC_RECOVERY_DISPOSITION_OPERATIONS_TOTAL, + "operation" => "create", + "outcome" => if result.is_ok() { "success" } else { "failed" } + ) + .increment(1); + result +} + +pub(crate) async fn collect_recovery_disposition_inventory(api: Arc) -> Result { + let mut marker = None; + let mut seen_markers = HashSet::new(); + let mut inventory = IlmRecoveryDispositionInventory::default(); + loop { + let page = api + .clone() + .list_objects_v2( + RUSTFS_META_BUCKET, + &format!("{ILM_RECOVERY_DISPOSITION_PREFIX}/"), + marker.clone(), + None, + 1_000, + false, + None, + false, + ) + .await?; + let mut load_candidates = Vec::with_capacity(page.objects.len()); + for object in page.objects { + inventory.count = inventory + .count + .checked_add(1) + .ok_or_else(|| Error::other("ILM recovery disposition count overflow"))?; + if object.size < 0 { + inventory.incomplete = true; + } else { + inventory.bytes = inventory + .bytes + .checked_add( + u64::try_from(object.size) + .map_err(|_| Error::other("ILM recovery disposition object size does not fit u64"))?, + ) + .ok_or_else(|| Error::other("ILM recovery disposition byte total overflow"))?; + } + let Ok((protocol, disposition_id)) = recovery_disposition_id_from_record_object_name(&object.name) else { + inventory.corrupt = inventory.corrupt.saturating_add(1); + inventory.incomplete = true; + continue; + }; + load_candidates.push((protocol, disposition_id)); + } + let inventory_cancel = CancellationToken::new(); + let page_api = api.clone(); + let loads = load_candidates.into_iter().map(move |(protocol, disposition_id)| { + let api = page_api.clone(); + async move { load_recovery_disposition(api, protocol, &disposition_id).await } + }); + for result in run_bounded_entry_tasks( + loads, + MAX_ACTIVE_RECOVERY_DISPOSITIONS, + RECOVERY_DISPOSITION_ENTRY_TIMEOUT, + &inventory_cancel, + ) + .await? + { + match result { + BoundedEntryTaskResult::Completed(Ok(observed)) => { + inventory.creations.push(( + observed.disposition.created_at_unix_nanos, + observed.disposition.identity.actor_sha256.clone(), + )); + match observed.disposition.state { + IlmRecoveryDispositionState::Prepared => inventory.prepared = inventory.prepared.saturating_add(1), + IlmRecoveryDispositionState::Applying => inventory.applying = inventory.applying.saturating_add(1), + IlmRecoveryDispositionState::Completed => inventory.completed = inventory.completed.saturating_add(1), + } + } + BoundedEntryTaskResult::Completed(Err(err)) if disposition_is_missing(&err) => { + inventory.incomplete = true; + } + BoundedEntryTaskResult::Completed(Err(_)) => { + inventory.corrupt = inventory.corrupt.saturating_add(1); + inventory.incomplete = true; + } + BoundedEntryTaskResult::TimedOut => inventory.incomplete = true, + } + } + if !page.is_truncated { + break; + } + let next = page + .next_continuation_token + .ok_or_else(|| Error::other("ILM recovery disposition inventory omitted its continuation marker"))?; + marker = Some(record_inventory_marker(&mut seen_markers, next)?); + } + Ok(inventory) +} + +pub(crate) async fn refresh_recovery_control_metrics(api: Arc) -> Result { + let mut counts = IlmRecoveryControlClassificationCounts::default(); + for protocol in IlmRecoveryProtocol::all() { + let mut marker = None; + let mut seen_markers = HashSet::new(); + loop { + let page = list_recovery_controls(api.clone(), protocol, None, 1_000, marker.clone()).await?; + counts.incomplete |= page.incomplete; + for record in page.records { + match record.classification { + IlmRecoveryClassification::Retrying => counts.retrying = counts.retrying.saturating_add(1), + IlmRecoveryClassification::RetainedAmbiguous => { + counts.retained_ambiguous = counts.retained_ambiguous.saturating_add(1) + } + IlmRecoveryClassification::Corrupt => counts.corrupt = counts.corrupt.saturating_add(1), + IlmRecoveryClassification::OperatorRequired => { + counts.operator_required = counts.operator_required.saturating_add(1) + } + IlmRecoveryClassification::Abandoned => counts.abandoned = counts.abandoned.saturating_add(1), + IlmRecoveryClassification::Terminal => counts.terminal = counts.terminal.saturating_add(1), + } + } + if !page.truncated { + break; + } + let next = page + .next_marker + .ok_or_else(|| Error::other("ILM recovery control metrics omitted its continuation marker"))?; + marker = Some(record_control_metrics_marker(&mut seen_markers, next)?); + } + } + metrics::gauge!(METRIC_RECOVERY_CONTROLS, "classification" => "retrying").set(metric_count(counts.retrying)); + metrics::gauge!(METRIC_RECOVERY_CONTROLS, "classification" => "retained_ambiguous") + .set(metric_count(counts.retained_ambiguous)); + metrics::gauge!(METRIC_RECOVERY_CONTROLS, "classification" => "corrupt").set(metric_count(counts.corrupt)); + metrics::gauge!(METRIC_RECOVERY_CONTROLS, "classification" => "operator_required") + .set(metric_count(counts.operator_required)); + metrics::gauge!(METRIC_RECOVERY_CONTROLS, "classification" => "abandoned").set(metric_count(counts.abandoned)); + metrics::gauge!(METRIC_RECOVERY_CONTROLS, "classification" => "terminal").set(metric_count(counts.terminal)); + metrics::gauge!(METRIC_RECOVERY_CONTROL_SCAN_INCOMPLETE).set(if counts.incomplete { 1.0 } else { 0.0 }); + Ok(counts) +} + +async fn process_recovery_disposition_maintenance_entry( + api: Arc, + object_name: String, + now_unix_nanos: i64, +) -> RecoveryDispositionMaintenanceEntryOutcome { + let (protocol, disposition_id) = match recovery_disposition_id_from_record_object_name(&object_name) { + Ok(parsed) => parsed, + Err(_) => return RecoveryDispositionMaintenanceEntryOutcome::Corrupt, + }; + let observed = match load_recovery_disposition(api.clone(), protocol, &disposition_id).await { + Ok(observed) => observed, + Err(err) if disposition_is_missing(&err) => return RecoveryDispositionMaintenanceEntryOutcome::Missing, + Err(_) => return RecoveryDispositionMaintenanceEntryOutcome::Corrupt, + }; + if observed.disposition.state == IlmRecoveryDispositionState::Completed { + if !disposition_is_gc_eligible(observed.disposition.state, observed.disposition.retain_until_unix_nanos, now_unix_nanos) { + return RecoveryDispositionMaintenanceEntryOutcome::Retained; + } + return match garbage_collect_completed_recovery_disposition(api, &observed, now_unix_nanos).await { + Ok(true) => RecoveryDispositionMaintenanceEntryOutcome::GarbageCollected, + Ok(false) => RecoveryDispositionMaintenanceEntryOutcome::Retained, + Err(_) => RecoveryDispositionMaintenanceEntryOutcome::Failed, + }; + } + + let _permit = match acquire_recovery_disposition_execution_permit().await { + Ok(permit) => permit, + Err(_) => return RecoveryDispositionMaintenanceEntryOutcome::Failed, + }; + match resume_recovery_disposition(api, protocol, &disposition_id, now_unix_nanos).await { + Ok(execution) => { + use super::recovery_disposition::IlmRecoveryDispositionExecutionOutcome::{AcceptedForRecovery, Completed, Replayed}; + match execution.outcome { + AcceptedForRecovery => RecoveryDispositionMaintenanceEntryOutcome::Resumed, + Completed => RecoveryDispositionMaintenanceEntryOutcome::Completed, + Replayed => RecoveryDispositionMaintenanceEntryOutcome::Replayed, + } + } + Err(_) => RecoveryDispositionMaintenanceEntryOutcome::Failed, + } +} + +fn record_maintenance_entry_outcome( + stats: &mut RecoveryDispositionMaintenanceStats, + outcome: RecoveryDispositionMaintenanceEntryOutcome, +) { + match outcome { + RecoveryDispositionMaintenanceEntryOutcome::Missing => {} + RecoveryDispositionMaintenanceEntryOutcome::Corrupt => stats.corrupt = stats.corrupt.saturating_add(1), + RecoveryDispositionMaintenanceEntryOutcome::Retained => stats.retained = stats.retained.saturating_add(1), + RecoveryDispositionMaintenanceEntryOutcome::GarbageCollected => { + stats.garbage_collected = stats.garbage_collected.saturating_add(1) + } + RecoveryDispositionMaintenanceEntryOutcome::Failed => stats.failed = stats.failed.saturating_add(1), + RecoveryDispositionMaintenanceEntryOutcome::Resumed => stats.resumed = stats.resumed.saturating_add(1), + RecoveryDispositionMaintenanceEntryOutcome::Completed => stats.completed = stats.completed.saturating_add(1), + RecoveryDispositionMaintenanceEntryOutcome::Replayed => stats.replayed = stats.replayed.saturating_add(1), + } +} + +fn advance_recovery_disposition_maintenance_cursor( + stats: &RecoveryDispositionMaintenanceStats, + seen_markers: &mut HashSet, +) -> Result { + if !stats.truncated { + seen_markers.clear(); + return Ok(RecoveryDispositionMaintenanceCursor { + marker: None, + wait_for_interval: true, + }); + } + let next = stats + .next_marker + .clone() + .ok_or_else(|| Error::other("ILM recovery disposition maintenance omitted its continuation marker"))?; + Ok(RecoveryDispositionMaintenanceCursor { + marker: Some(record_maintenance_marker(seen_markers, next)?), + wait_for_interval: false, + }) +} + +pub(crate) async fn run_recovery_disposition_maintenance_pass( + api: Arc, + cancel_token: &CancellationToken, + limit: usize, + marker: Option, + now_unix_nanos: i64, +) -> Result { + if !(1..=1_000).contains(&limit) { + return Err(Error::other("ILM recovery disposition pass limit must be between 1 and 1000")); + } + let page = tokio::time::timeout( + RECOVERY_DISPOSITION_PAGE_LIST_TIMEOUT, + api.clone().list_objects_v2( + RUSTFS_META_BUCKET, + &format!("{ILM_RECOVERY_DISPOSITION_PREFIX}/"), + marker, + None, + i32::try_from(limit).unwrap_or(1_000), + false, + None, + false, + ), + ) + .await + .map_err(|_| Error::Timeout)??; + if page.is_truncated && page.next_continuation_token.is_none() { + return Err(Error::other("ILM recovery disposition pass omitted its continuation marker")); + } + let scanned = + u64::try_from(page.objects.len()).map_err(|_| Error::other("ILM recovery disposition page length does not fit u64"))?; + let mut stats = RecoveryDispositionMaintenanceStats { + scanned, + next_marker: page.next_continuation_token, + truncated: page.is_truncated, + ..Default::default() + }; + let entries = page.objects.into_iter().map(move |object| { + let api = api.clone(); + async move { process_recovery_disposition_maintenance_entry(api, object.name, now_unix_nanos).await } + }); + for result in run_bounded_entry_tasks( + entries, + MAX_ACTIVE_RECOVERY_DISPOSITIONS, + RECOVERY_DISPOSITION_ENTRY_TIMEOUT, + cancel_token, + ) + .await? + { + match result { + BoundedEntryTaskResult::Completed(outcome) => record_maintenance_entry_outcome(&mut stats, outcome), + BoundedEntryTaskResult::TimedOut => stats.failed = stats.failed.saturating_add(1), + } + } + metrics::counter!( + METRIC_RECOVERY_DISPOSITION_OPERATIONS_TOTAL, + "operation" => "maintenance_pass", + "outcome" => if stats.failed == 0 { "success" } else { "failed" } + ) + .increment(1); + Ok(stats) +} + +pub(crate) async fn run_recovery_disposition_maintenance_loop(api: Arc, cancel_token: CancellationToken) { + let mut interval = tokio::time::interval(RECOVERY_DISPOSITION_MAINTENANCE_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut marker = None; + let mut seen_markers = HashSet::new(); + let mut last_metrics_refresh = None; + let mut wait_for_interval = true; + loop { + if wait_for_interval { + tokio::select! { + biased; + _ = cancel_token.cancelled() => return, + _ = interval.tick() => {}, + } + } else if cancel_token.is_cancelled() { + return; + } + let now_unix_nanos = match now_unix_nanos() { + Ok(now) => now, + Err(_) => { + metrics::counter!( + METRIC_RECOVERY_DISPOSITION_OPERATIONS_TOTAL, + "operation" => "maintenance_loop", + "outcome" => "clock_error" + ) + .increment(1); + wait_for_interval = true; + interval.reset(); + continue; + } + }; + let pass = run_recovery_disposition_maintenance_pass( + api.clone(), + &cancel_token, + DEFAULT_RECOVERY_DISPOSITION_PASS_LIMIT, + marker.clone(), + now_unix_nanos, + ); + match tokio::time::timeout(RECOVERY_DISPOSITION_PASS_TIMEOUT, pass).await { + Ok(Ok(stats)) => match advance_recovery_disposition_maintenance_cursor(&stats, &mut seen_markers) { + Ok(cursor) => { + marker = cursor.marker; + wait_for_interval = cursor.wait_for_interval; + } + Err(_) => { + metrics::counter!( + METRIC_RECOVERY_DISPOSITION_OPERATIONS_TOTAL, + "operation" => "maintenance_loop", + "outcome" => "marker_cycle" + ) + .increment(1); + seen_markers.clear(); + marker = None; + wait_for_interval = true; + } + }, + Ok(Err(_)) | Err(_) => { + metrics::counter!( + METRIC_RECOVERY_DISPOSITION_OPERATIONS_TOTAL, + "operation" => "maintenance_loop", + "outcome" => "failed" + ) + .increment(1); + seen_markers.clear(); + marker = None; + wait_for_interval = true; + } + } + let metrics_refresh_due = + last_metrics_refresh.is_none_or(|last: tokio::time::Instant| last.elapsed() >= RECOVERY_DISPOSITION_METRICS_INTERVAL); + if marker.is_none() && metrics_refresh_due { + if let Ok(Ok(inventory)) = tokio::time::timeout( + RECOVERY_DISPOSITION_INVENTORY_TIMEOUT, + collect_recovery_disposition_inventory(api.clone()), + ) + .await + { + record_recovery_disposition_inventory_metrics(&inventory); + } + let _ = + tokio::time::timeout(RECOVERY_DISPOSITION_INVENTORY_TIMEOUT, refresh_recovery_control_metrics(api.clone())).await; + last_metrics_refresh = Some(tokio::time::Instant::now()); + } + if wait_for_interval { + // A full round (or a failed page) gets one fresh backoff. Resetting + // after metrics work avoids an overdue interval tick turning the + // next round into an immediate retry. + interval.reset(); + } + } +} + +pub(crate) async fn garbage_collect_completed_recovery_disposition( + api: Arc, + observed: &ObservedIlmRecoveryDisposition, + now_unix_nanos: i64, +) -> Result { + if !disposition_is_gc_eligible(observed.disposition.state, observed.disposition.retain_until_unix_nanos, now_unix_nanos) { + return Ok(false); + } + let protocol = observed.disposition.identity.protocol; + let disposition_id = &observed.disposition.identity.disposition_id; + let object = recovery_disposition_record_object_name(protocol, disposition_id).map_err(Error::other)?; + let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &object).await?; + let guard = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?; + let authoritative_result = load_recovery_disposition_no_lock(api.clone(), protocol, disposition_id).await; + if guard.is_lock_lost() { + return Err(Error::PreconditionFailed); + } + let authoritative = match authoritative_result { + Ok(authoritative) => authoritative, + Err(err) if disposition_is_missing(&err) => return Ok(true), + Err(err) => return Err(err), + }; + if &authoritative != observed + || authoritative.disposition.state != IlmRecoveryDispositionState::Completed + || authoritative.disposition.retain_until_unix_nanos > now_unix_nanos + || guard.is_lock_lost() + { + return Err(Error::PreconditionFailed); + } + api.record_durable_ilm_decommission_terminal(&object, &authoritative.encoded) + .await?; + if guard.is_lock_lost() { + return Err(Error::PreconditionFailed); + } + let mut delete_options = ObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + no_lock: true, + ..Default::default() + }; + delete_options.add_namespace_lock_guard(&guard); + let delete_result = + config_boundary::delete_config_if_match_with_opts(api.clone(), &object, &authoritative.etag, delete_options).await; + if guard.is_lock_lost() { + return Err(Error::PreconditionFailed); + } + let confirmation = load_recovery_disposition_no_lock(api, protocol, disposition_id).await; + if guard.is_lock_lost() { + return Err(Error::PreconditionFailed); + } + match confirmation { + Err(err) if disposition_is_missing(&err) => { + metrics::counter!( + METRIC_RECOVERY_DISPOSITION_OPERATIONS_TOTAL, + "operation" => "gc", + "outcome" => "deleted" + ) + .increment(1); + Ok(true) + } + Ok(_) => Err(delete_result.err().unwrap_or(Error::PreconditionFailed)), + Err(confirm_err) => Err(delete_result.err().unwrap_or(confirm_err)), + } +} + +fn check_recovery_disposition_admission( + count: usize, + bytes: u64, + active: usize, + actor_recent: usize, + cluster_recent: usize, + candidate_len: usize, +) -> Result<()> { + let candidate_len = + u64::try_from(candidate_len).map_err(|_| Error::other("ILM recovery disposition size does not fit u64"))?; + if count >= MAX_ILM_RECOVERY_DISPOSITIONS + || bytes + .checked_add(candidate_len) + .is_none_or(|total| total > MAX_ILM_RECOVERY_DISPOSITION_BYTES) + || active >= MAX_ACTIVE_RECOVERY_DISPOSITIONS + || actor_recent >= MAX_ACTOR_DISPOSITIONS_PER_MINUTE + || cluster_recent >= MAX_CLUSTER_DISPOSITIONS_PER_MINUTE + { + return Err(Error::SlowDown); + } + Ok(()) +} + +fn disposition_is_gc_eligible(state: IlmRecoveryDispositionState, retain_until_unix_nanos: i64, now_unix_nanos: i64) -> bool { + state == IlmRecoveryDispositionState::Completed && retain_until_unix_nanos <= now_unix_nanos +} + +fn record_recovery_disposition_inventory_metrics(inventory: &IlmRecoveryDispositionInventory) { + metrics::gauge!(METRIC_RECOVERY_DISPOSITION_RECORDS, "state" => "prepared").set(metric_count(inventory.prepared)); + metrics::gauge!(METRIC_RECOVERY_DISPOSITION_RECORDS, "state" => "applying").set(metric_count(inventory.applying)); + metrics::gauge!(METRIC_RECOVERY_DISPOSITION_RECORDS, "state" => "completed").set(metric_count(inventory.completed)); + metrics::gauge!(METRIC_RECOVERY_DISPOSITION_RECORDS, "state" => "corrupt").set(metric_count(inventory.corrupt)); + metrics::gauge!(METRIC_RECOVERY_DISPOSITION_BYTES).set(metric_u64(inventory.bytes)); +} + +fn metric_count(value: usize) -> f64 { + f64::from(u32::try_from(value).unwrap_or(u32::MAX)) +} + +fn metric_u64(value: u64) -> f64 { + value.to_string().parse::().unwrap_or(f64::MAX) +} + +fn record_inventory_marker(seen_markers: &mut HashSet, next: String) -> Result { + if !seen_markers.insert(next.clone()) { + return Err(Error::other("ILM recovery disposition inventory repeated its continuation marker")); + } + Ok(next) +} + +fn record_control_metrics_marker(seen_markers: &mut HashSet, next: String) -> Result { + if !seen_markers.insert(next.clone()) { + return Err(Error::other("ILM recovery control metrics repeated its continuation marker")); + } + Ok(next) +} + +fn record_maintenance_marker(seen_markers: &mut HashSet, next: String) -> Result { + if !seen_markers.insert(next.clone()) { + return Err(Error::other("ILM recovery disposition maintenance repeated its continuation marker")); + } + Ok(next) +} + +fn now_unix_nanos() -> Result { + i64::try_from(time::OffsetDateTime::now_utc().unix_timestamp_nanos()) + .map_err(|_| Error::other("ILM recovery disposition timestamp does not fit i64")) +} + +fn disposition_is_missing(err: &Error) -> bool { + matches!( + err, + Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::VersionNotFound(_, _, _) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_maintenance_page_fits_strictly_inside_pass_deadline() { + const { + assert!(DEFAULT_RECOVERY_DISPOSITION_PASS_LIMIT <= MAX_ACTIVE_RECOVERY_DISPOSITIONS); + } + assert!(RECOVERY_DISPOSITION_PAGE_LIST_TIMEOUT + RECOVERY_DISPOSITION_ENTRY_TIMEOUT < RECOVERY_DISPOSITION_PASS_TIMEOUT); + } + + #[test] + fn truncated_maintenance_pages_advance_without_per_page_interval_waits() { + let page_count = MAX_ILM_RECOVERY_DISPOSITIONS.div_ceil(DEFAULT_RECOVERY_DISPOSITION_PASS_LIMIT); + let mut seen_markers = HashSet::new(); + let mut interval_waits = 0; + for page_index in 0..page_count { + let truncated = page_index + 1 < page_count; + let stats = RecoveryDispositionMaintenanceStats { + next_marker: truncated.then(|| format!("page-{}", page_index + 1)), + truncated, + ..Default::default() + }; + let cursor = advance_recovery_disposition_maintenance_cursor(&stats, &mut seen_markers) + .expect("unique maintenance markers should advance"); + interval_waits += usize::from(cursor.wait_for_interval); + if truncated { + assert!(cursor.marker.is_some(), "a truncated page must continue immediately"); + } else { + assert!(cursor.marker.is_none(), "a completed round must restart from the beginning"); + } + } + assert_eq!(interval_waits, 1, "ten thousand records must incur one interval wait per full round"); + } + + #[test] + fn timed_out_entry_does_not_add_a_page_interval_before_tail_or_retry() { + let mut stats = RecoveryDispositionMaintenanceStats { + failed: 1, + next_marker: Some("tail-page".to_string()), + truncated: true, + ..Default::default() + }; + let mut seen_markers = HashSet::new(); + let tail = advance_recovery_disposition_maintenance_cursor(&stats, &mut seen_markers) + .expect("a timed-out entry must not block pagination"); + assert!(!tail.wait_for_interval); + assert_eq!(tail.marker.as_deref(), Some("tail-page")); + + stats.next_marker = None; + stats.truncated = false; + let next_round = advance_recovery_disposition_maintenance_cursor(&stats, &mut seen_markers) + .expect("the completed round should schedule one bounded retry interval"); + assert!(next_round.wait_for_interval); + assert!(next_round.marker.is_none()); + } + + #[test] + fn execution_permit_rejects_saturation_and_releases_capacity_on_drop() { + let execution_limit = Arc::new(Semaphore::new(2)); + let inflight = AtomicUsize::new(0); + + let first = try_acquire_recovery_disposition_execution_permit(Arc::clone(&execution_limit), &inflight, false) + .expect("first execution permit should be available"); + let second = try_acquire_recovery_disposition_execution_permit(Arc::clone(&execution_limit), &inflight, false) + .expect("second execution permit should be available"); + assert_eq!(inflight.load(Ordering::Acquire), 2); + + let saturated = try_acquire_recovery_disposition_execution_permit(Arc::clone(&execution_limit), &inflight, false); + assert!(matches!(saturated, Err(Error::SlowDown))); + assert_eq!(inflight.load(Ordering::Acquire), 2); + + drop(first); + assert_eq!(inflight.load(Ordering::Acquire), 1); + let replacement = try_acquire_recovery_disposition_execution_permit(execution_limit, &inflight, false) + .expect("dropping a permit should immediately restore capacity"); + assert_eq!(inflight.load(Ordering::Acquire), 2); + + drop(second); + drop(replacement); + assert_eq!(inflight.load(Ordering::Acquire), 0); + } + + #[tokio::test(start_paused = true)] + async fn bounded_tasks_complete_fast_peer_and_release_timed_out_permit() { + type EntryTask = std::pin::Pin + Send + 'static>>; + + let execution_limit = Arc::new(Semaphore::new(1)); + let inflight = Arc::new(AtomicUsize::new(0)); + let fast_completed = Arc::new(AtomicUsize::new(0)); + let slow_started = Arc::new(tokio::sync::Notify::new()); + let fast_finished = Arc::new(tokio::sync::Notify::new()); + let slow_limit = Arc::clone(&execution_limit); + let slow_inflight = Arc::clone(&inflight); + let slow_signal = Arc::clone(&slow_started); + let slow: EntryTask = Box::pin(async move { + let _permit = try_acquire_recovery_disposition_execution_permit(slow_limit, slow_inflight.as_ref(), false) + .expect("slow task should acquire its isolated execution permit"); + slow_signal.notify_one(); + std::future::pending::().await + }); + let fast_signal = Arc::clone(&fast_completed); + let fast_notification = Arc::clone(&fast_finished); + let fast: EntryTask = Box::pin(async move { + fast_signal.store(1, Ordering::Release); + fast_notification.notify_one(); + 7 + }); + let cancel_token = CancellationToken::new(); + let runner = run_bounded_entry_tasks(vec![slow, fast], 2, RECOVERY_DISPOSITION_ENTRY_TIMEOUT, &cancel_token); + tokio::pin!(runner); + + tokio::select! { + _ = slow_started.notified() => {} + result = &mut runner => panic!("bounded runner completed before the slow task started: {result:?}"), + } + tokio::select! { + _ = fast_finished.notified() => {} + result = &mut runner => panic!("bounded runner completed before the fast peer: {result:?}"), + } + assert_eq!(fast_completed.load(Ordering::Acquire), 1); + assert_eq!(inflight.load(Ordering::Acquire), 1); + + tokio::time::advance(RECOVERY_DISPOSITION_ENTRY_TIMEOUT).await; + let results = runner.await.expect("entry timeout should be reported as an outcome"); + assert_eq!( + results + .iter() + .filter(|result| matches!(result, BoundedEntryTaskResult::Completed(7))) + .count(), + 1 + ); + assert_eq!( + results + .iter() + .filter(|result| matches!(result, BoundedEntryTaskResult::TimedOut)) + .count(), + 1 + ); + assert_eq!(inflight.load(Ordering::Acquire), 0); + } + + #[tokio::test(start_paused = true)] + async fn bounded_task_cancellation_drops_execution_permit() { + let execution_limit = Arc::new(Semaphore::new(1)); + let inflight = Arc::new(AtomicUsize::new(0)); + let task_started = Arc::new(tokio::sync::Notify::new()); + let task_limit = Arc::clone(&execution_limit); + let task_inflight = Arc::clone(&inflight); + let task_signal = Arc::clone(&task_started); + let pending = async move { + let _permit = try_acquire_recovery_disposition_execution_permit(task_limit, task_inflight.as_ref(), false) + .expect("pending task should acquire its isolated execution permit"); + task_signal.notify_one(); + std::future::pending::<()>().await; + }; + let cancel_token = CancellationToken::new(); + let runner = run_bounded_entry_tasks(std::iter::once(pending), 1, RECOVERY_DISPOSITION_ENTRY_TIMEOUT, &cancel_token); + tokio::pin!(runner); + + tokio::select! { + _ = task_started.notified() => {} + result = &mut runner => panic!("bounded runner completed before the pending task started: {result:?}"), + } + assert_eq!(inflight.load(Ordering::Acquire), 1); + cancel_token.cancel(); + let result = runner.await; + assert!(matches!(result, Err(Error::OperationCanceled))); + assert_eq!(inflight.load(Ordering::Acquire), 0); + } + + #[test] + fn disposition_admission_enforces_each_exact_boundary() { + assert!( + check_recovery_disposition_admission( + MAX_ILM_RECOVERY_DISPOSITIONS - 1, + MAX_ILM_RECOVERY_DISPOSITION_BYTES - 1, + MAX_ACTIVE_RECOVERY_DISPOSITIONS - 1, + MAX_ACTOR_DISPOSITIONS_PER_MINUTE - 1, + MAX_CLUSTER_DISPOSITIONS_PER_MINUTE - 1, + 1, + ) + .is_ok() + ); + assert!(check_recovery_disposition_admission(MAX_ILM_RECOVERY_DISPOSITIONS, 0, 0, 0, 0, 1).is_err()); + assert!(check_recovery_disposition_admission(0, MAX_ILM_RECOVERY_DISPOSITION_BYTES, 0, 0, 0, 1).is_err()); + assert!(check_recovery_disposition_admission(0, 0, MAX_ACTIVE_RECOVERY_DISPOSITIONS, 0, 0, 1).is_err()); + assert!(check_recovery_disposition_admission(0, 0, 0, MAX_ACTOR_DISPOSITIONS_PER_MINUTE, 0, 1).is_err()); + assert!(check_recovery_disposition_admission(0, 0, 0, 0, MAX_CLUSTER_DISPOSITIONS_PER_MINUTE, 1).is_err()); + } + + #[test] + fn disposition_admission_enforces_exact_byte_boundary_and_overflow() { + let candidate_len = 16 * 1024; + assert!( + check_recovery_disposition_admission( + 0, + MAX_ILM_RECOVERY_DISPOSITION_BYTES - u64::try_from(candidate_len).expect("candidate fits u64"), + 0, + 0, + 0, + candidate_len, + ) + .is_ok() + ); + assert!(matches!( + check_recovery_disposition_admission( + 0, + MAX_ILM_RECOVERY_DISPOSITION_BYTES - u64::try_from(candidate_len).expect("candidate fits u64") + 1, + 0, + 0, + 0, + candidate_len, + ), + Err(Error::SlowDown) + )); + assert!(matches!( + check_recovery_disposition_admission(0, u64::MAX, 0, 0, 0, 1), + Err(Error::SlowDown) + )); + } + + #[test] + fn inventory_enforces_actor_and_cluster_rate_limits_independently() { + let now = 120 * 1_000_000_000; + let recent = now - RECOVERY_DISPOSITION_RECENT_WINDOW_NANOS + 1; + let actor_below_limit = IlmRecoveryDispositionInventory { + creations: vec![(recent, "actor-below-limit".to_string()); MAX_ACTOR_DISPOSITIONS_PER_MINUTE - 1], + ..Default::default() + }; + assert!(actor_below_limit.check_admission("actor-below-limit", 1, now).is_ok()); + + let actor_at_limit = IlmRecoveryDispositionInventory { + creations: vec![(recent, "actor-at-limit".to_string()); MAX_ACTOR_DISPOSITIONS_PER_MINUTE], + ..Default::default() + }; + assert!(matches!(actor_at_limit.check_admission("actor-at-limit", 1, now), Err(Error::SlowDown))); + assert!(actor_at_limit.check_admission("different-actor", 1, now).is_ok()); + + let cluster_below_limit = IlmRecoveryDispositionInventory { + creations: (0..MAX_CLUSTER_DISPOSITIONS_PER_MINUTE - 1) + .map(|index| (recent, format!("actor-{index}"))) + .collect(), + ..Default::default() + }; + assert!(cluster_below_limit.check_admission("new-actor", 1, now).is_ok()); + + let cluster_at_limit = IlmRecoveryDispositionInventory { + creations: (0..MAX_CLUSTER_DISPOSITIONS_PER_MINUTE) + .map(|index| (recent, format!("actor-{index}"))) + .collect(), + ..Default::default() + }; + assert!(matches!(cluster_at_limit.check_admission("new-actor", 1, now), Err(Error::SlowDown))); + } + + #[test] + fn inventory_rejects_future_creation_even_when_rate_limits_are_clear() { + let now = 120 * 1_000_000_000; + let inventory = IlmRecoveryDispositionInventory { + creations: vec![(now + 1, "other-actor".to_string())], + ..Default::default() + }; + + assert!(matches!(inventory.check_admission("actor", 1, now), Err(Error::PreconditionFailed))); + } + + #[test] + fn inventory_marker_rejects_cycles() { + let mut seen = HashSet::new(); + assert_eq!( + record_inventory_marker(&mut seen, "next".to_string()).expect("new marker should pass"), + "next" + ); + assert!(record_inventory_marker(&mut seen, "next".to_string()).is_err()); + } + + #[test] + fn control_metrics_marker_rejects_cycles() { + let mut seen = HashSet::new(); + assert_eq!( + record_control_metrics_marker(&mut seen, "next".to_string()).expect("new marker should pass"), + "next" + ); + assert!(record_control_metrics_marker(&mut seen, "next".to_string()).is_err()); + } + + #[test] + fn incomplete_inventory_fails_closed() { + let inventory = IlmRecoveryDispositionInventory { + incomplete: true, + ..Default::default() + }; + assert!(inventory.check_admission("actor", 1, 1_000_000_000).is_err()); + } + + #[test] + fn inventory_rate_window_is_exact_and_rejects_future_timestamps() { + let now = 120 * 1_000_000_000; + let mut inventory = IlmRecoveryDispositionInventory { + creations: vec![(now - RECOVERY_DISPOSITION_RECENT_WINDOW_NANOS, "actor".to_string())], + ..Default::default() + }; + assert!(inventory.check_admission("actor", 1, now).is_ok()); + + inventory.creations = vec![(now - RECOVERY_DISPOSITION_RECENT_WINDOW_NANOS + 1, "actor".to_string()); 10]; + assert!(inventory.check_admission("actor", 1, now).is_err()); + + inventory.creations = vec![(now + 1, "actor".to_string())]; + assert!(inventory.check_admission("actor", 1, now).is_err()); + } + + #[test] + fn garbage_collection_requires_terminal_state_and_elapsed_retention() { + assert!(!disposition_is_gc_eligible(IlmRecoveryDispositionState::Prepared, 10, 11)); + assert!(!disposition_is_gc_eligible(IlmRecoveryDispositionState::Applying, 10, 11)); + assert!(!disposition_is_gc_eligible(IlmRecoveryDispositionState::Completed, 11, 10)); + assert!(disposition_is_gc_eligible(IlmRecoveryDispositionState::Completed, 10, 10)); + } +} diff --git a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs index 426956d13..a950a6a5b 100644 --- a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs +++ b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs @@ -34,7 +34,7 @@ use crate::bucket::lifecycle::tier_sweeper::{ }; use crate::disk::RUSTFS_META_BUCKET; use crate::error::{Error, Result as EcstoreResult}; -use crate::object_api::ObjectOptions; +use crate::object_api::{ObjectInfo, ObjectOptions}; use crate::services::tier::{tier::TierConfigMgr, warm_backend::TransitionCandidateProbe}; use crate::storage_api_contracts::{ list::ListOperations as _, @@ -954,6 +954,10 @@ pub enum TransitionOperatorError { expected: String, actual: TransitionOperatorProbe, }, + #[error("transition recovery control is stale")] + StaleRecoveryControl, + #[error("transition recovery control is not eligible for operator retry")] + RetryNotAllowed, #[error("transition transaction store failed: {0}")] Store(#[source] Error), #[error("remote tier reconciliation failed: {0}")] @@ -962,6 +966,179 @@ pub enum TransitionOperatorError { type TransitionOperatorResult = std::result::Result; +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct TransitionRecoveryRetryStatus { + pub control_id: String, + pub transaction_id: Uuid, + pub state: TransitionTransactionState, + pub classification: IlmRecoveryClassification, + pub control_revision: u64, + pub attempt_count: u64, + pub consecutive_failure_count: u32, + pub last_error_code: IlmRecoveryErrorCode, + pub source_generation_sha256: String, + pub copy_set_sha256: String, + pub retry_ready: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry_not_ready_reason: Option<&'static str>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct TransitionRecoveryRetryResult { + pub control_id: String, + pub transaction_id: Uuid, + pub previous_revision: u64, + pub revision: u64, + pub classification: IlmRecoveryClassification, + pub attempt_count: u64, + pub source_generation_sha256: String, +} + +struct TransitionRecoveryRetryContext { + observed: ObservedIlmRecoveryControl, + transaction: TransitionTransaction, + source_generation_sha256: String, +} + +fn transition_recovery_retry_readiness(control: &IlmRecoveryControl) -> (bool, Option<&'static str>) { + if control.owner.is_some() { + return (false, Some("attempt_owned")); + } + match control.classification { + IlmRecoveryClassification::RetainedAmbiguous | IlmRecoveryClassification::OperatorRequired => (true, None), + IlmRecoveryClassification::Retrying => (false, Some("already_retrying")), + IlmRecoveryClassification::Corrupt => (false, Some("source_corrupt")), + IlmRecoveryClassification::Abandoned => (false, Some("source_abandoned")), + IlmRecoveryClassification::Terminal => (false, Some("source_terminal")), + } +} + +async fn load_transition_recovery_retry_context( + api: Arc, + control_id: &str, +) -> TransitionOperatorResult { + let observed = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await { + Ok(observed) => observed, + Err(Error::ConfigNotFound) => return Err(TransitionOperatorError::NotFound), + Err(err) => return Err(TransitionOperatorError::Store(err)), + }; + let transaction_id = Uuid::parse_str(&observed.control.identity.stable_operation_identity) + .ok() + .filter(|transaction_id| !transaction_id.is_nil()) + .ok_or(TransitionOperatorError::StaleRecoveryControl)?; + let canonical_path = transition_transaction_record_object_name(transaction_id) + .map_err(|err| TransitionOperatorError::Store(Error::other(err)))?; + if observed.control.identity.canonical_source_path != canonical_path + || observed.control.identity.record_class != "transition_transaction_v1" + { + return Err(TransitionOperatorError::StaleRecoveryControl); + } + let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await { + Ok(transaction) => transaction, + Err(Error::ConfigNotFound) => return Err(TransitionOperatorError::NotFound), + Err(err) => return Err(TransitionOperatorError::Store(err)), + }; + let source = observe_recovery_source(api, &canonical_path, TRANSITION_TRANSACTION_SCHEMA) + .await + .map_err(TransitionOperatorError::Store)?; + let exact_source = source.is_consistent() + && source.generation == observed.control.observed_source_generation + && source + .canonical_data + .as_deref() + .is_some_and(|data| TransitionTransaction::decode(transaction_id, data).is_ok_and(|decoded| decoded == transaction)); + if !exact_source { + return Err(TransitionOperatorError::StaleRecoveryControl); + } + let generation = serde_json::to_vec(&observed.control.observed_source_generation) + .map_err(|err| TransitionOperatorError::Store(Error::other(err)))?; + Ok(TransitionRecoveryRetryContext { + observed, + transaction, + source_generation_sha256: hex_sha256(&generation, ToOwned::to_owned), + }) +} + +pub async fn inspect_transition_recovery_retry_for_operator( + api: Arc, + control_id: &str, +) -> TransitionOperatorResult { + let context = load_transition_recovery_retry_context(api, control_id).await?; + let (retry_ready, retry_not_ready_reason) = transition_recovery_retry_readiness(&context.observed.control); + Ok(TransitionRecoveryRetryStatus { + control_id: control_id.to_string(), + transaction_id: context.transaction.transaction_id, + state: context.transaction.state, + classification: context.observed.control.classification, + control_revision: context.observed.control.revision, + attempt_count: context.observed.control.attempt_count, + consecutive_failure_count: context.observed.control.consecutive_failure_count, + last_error_code: context.observed.control.last_error_code, + source_generation_sha256: context.source_generation_sha256, + copy_set_sha256: context.observed.control.observed_source_generation.copy_set_sha256.clone(), + retry_ready, + retry_not_ready_reason, + }) +} + +pub async fn retry_transition_recovery_for_operator( + api: Arc, + control_id: &str, + expected_control_revision: u64, + expected_source_generation_sha256: &str, +) -> TransitionOperatorResult { + let control_object = recovery_control_record_object_name(IlmRecoveryProtocol::TransitionTransaction, control_id) + .map_err(|err| TransitionOperatorError::Store(Error::other(err)))?; + let retry_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &format!("{control_object}.recovery-lock")) + .await + .map_err(TransitionOperatorError::Store)?; + let retry_guard = retry_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await + .map_err(|err| TransitionOperatorError::Store(Error::other(err)))?; + let context = load_transition_recovery_retry_context(api.clone(), control_id).await?; + let (retry_ready, _) = transition_recovery_retry_readiness(&context.observed.control); + if !retry_ready { + return Err(TransitionOperatorError::RetryNotAllowed); + } + if retry_guard.is_lock_lost() + || expected_control_revision == 0 + || context.observed.control.revision != expected_control_revision + || context.source_generation_sha256 != expected_source_generation_sha256 + { + return Err(TransitionOperatorError::StaleRecoveryControl); + } + let previous_revision = context.observed.control.revision; + let mut next = context.observed.control.clone(); + next.retry_for_operator(&context.observed.control.observed_source_generation) + .map_err(|_| TransitionOperatorError::RetryNotAllowed)?; + if retry_guard.is_lock_lost() { + return Err(TransitionOperatorError::StaleRecoveryControl); + } + save_recovery_control_if_current(api.clone(), &context.observed, &next) + .await + .map_err(|err| match err { + Error::PreconditionFailed => TransitionOperatorError::StaleRecoveryControl, + err => TransitionOperatorError::Store(err), + })?; + let persisted = load_recovery_control(api, IlmRecoveryProtocol::TransitionTransaction, control_id) + .await + .map_err(TransitionOperatorError::Store)?; + if retry_guard.is_lock_lost() || persisted.control != next { + return Err(TransitionOperatorError::StaleRecoveryControl); + } + Ok(TransitionRecoveryRetryResult { + control_id: control_id.to_string(), + transaction_id: context.transaction.transaction_id, + previous_revision, + revision: persisted.control.revision, + classification: persisted.control.classification, + attempt_count: persisted.control.attempt_count, + source_generation_sha256: context.source_generation_sha256, + }) +} + fn validate_operator_reconcile_transaction( transaction: &TransitionTransaction, now_unix_nanos: i128, @@ -1706,12 +1883,27 @@ async fn local_commit_matches_transaction(api: Arc, transaction: &Trans .get_object_info(&transaction.source.bucket, &transaction.source.object, &opts) .await?; let transitioned = &object.transitioned_object; - Ok(transitioned.status == TRANSITION_COMPLETE + Ok(local_object_matches_transition_source(&object, &transaction.source) + && transitioned.status == TRANSITION_COMPLETE && transitioned.name == transaction.remote_object && transitioned.tier == transaction.tier_name && transitioned.version_id == transaction.remote_version.tier_delete_version_id().unwrap_or_default()) } +fn local_object_matches_transition_source(object: &ObjectInfo, source: &TransitionSourceIdentity) -> bool { + let observed_version_id = object.version_id.filter(|version_id| !version_id.is_nil()); + let observed_mod_time = object + .mod_time + .and_then(|mod_time| i64::try_from(mod_time.unix_timestamp_nanos()).ok()); + object.bucket == source.bucket + && object.name == source.object + && observed_version_id == source.version_id + && object.data_dir == Some(source.data_dir) + && observed_mod_time == Some(source.mod_time_unix_nanos) + && object.size == source.size + && object.etag.as_deref() == Some(source.etag.as_str()) +} + fn transition_source_lookup_options(transaction: &TransitionTransaction) -> ObjectOptions { ObjectOptions { version_id: match transaction.source.version_mode { @@ -2183,6 +2375,41 @@ mod tests { } } + #[test] + fn local_commit_proof_requires_the_complete_source_identity() { + let source = source_identity(TransitionSourceVersionMode::Versioned); + let exact = ObjectInfo { + bucket: source.bucket.clone(), + name: source.object.clone(), + version_id: source.version_id, + data_dir: Some(source.data_dir), + mod_time: Some( + time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(source.mod_time_unix_nanos)) + .expect("source timestamp should be valid"), + ), + size: source.size, + etag: Some(source.etag.clone()), + ..Default::default() + }; + assert!(local_object_matches_transition_source(&exact, &source)); + + let mut changed = exact.clone(); + changed.version_id = Some(Uuid::new_v4()); + assert!(!local_object_matches_transition_source(&changed, &source)); + changed = exact.clone(); + changed.data_dir = Some(Uuid::new_v4()); + assert!(!local_object_matches_transition_source(&changed, &source)); + changed = exact.clone(); + changed.mod_time = changed.mod_time.map(|value| value + Duration::from_nanos(1)); + assert!(!local_object_matches_transition_source(&changed, &source)); + changed = exact.clone(); + changed.size += 1; + assert!(!local_object_matches_transition_source(&changed, &source)); + changed = exact; + changed.etag = Some("different-etag".to_string()); + assert!(!local_object_matches_transition_source(&changed, &source)); + } + fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof { TransitionCleanupProof { transaction_id: transaction.transaction_id, diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 068d412f7..087e25a5f 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -828,8 +828,14 @@ mod tests { recovery_control::{ IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, list_recovery_controls, load_recovery_control, - observe_recovery_source, save_recovery_control_if_absent, + observe_recovery_source, recovery_control_record_object_name, save_recovery_control_if_absent, }, + recovery_disposition::{ + IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionState, RecoveryDispositionCrashStage, + dry_run_recovery_disposition, execute_recovery_disposition, inject_recovery_disposition_crash_once, + load_recovery_disposition, + }, + recovery_disposition_runtime::garbage_collect_completed_recovery_disposition, recovery_export::{ create_recovery_export, inspect_recovery_export_observation, load_recovery_export, recovery_export_record_object_name, @@ -856,9 +862,10 @@ mod tests { TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier, TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator, - finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator, - load_transition_transaction_record, recover_transition_transaction_records, - recover_transition_transaction_records_at, save_transition_transaction_record, + finalize_missing_transition_transaction_for_operator, inspect_transition_recovery_retry_for_operator, + inspect_transition_transaction_for_operator, load_transition_transaction_record, + recover_transition_transaction_records, recover_transition_transaction_records_at, + retry_transition_recovery_for_operator, save_transition_transaction_record, save_transition_transaction_record_if_current, transition_recovery_control_id, transition_transaction_record_object_name, }, @@ -16936,6 +16943,301 @@ mod tests { ); } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn legacy_recovery_disposition_removes_only_local_journals_and_replays() { + Box::pin(legacy_recovery_disposition_removes_only_local_journals_and_replays_case()).await; + } + + #[cfg(feature = "test-util")] + async fn legacy_recovery_disposition_removes_only_local_journals_and_replays_case() { + let temp_dir = tempfile::tempdir().expect("create legacy disposition store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-recovery-disposition", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let tier_name = "LEGACY-DISPOSITION"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("legacy disposition tier lease should resolve") + .backend_identity(); + let fixtures = [ + serde_json::json!({ + "version": 1, + "obj_name": "legacy/disposition-v1", + "version_id": "opaque-disposition-v1", + "tier_name": tier_name, + }), + serde_json::json!({ + "version": 2, + "obj_name": "legacy/disposition-v2", + "version_id": "opaque-disposition-v2", + "tier_name": tier_name, + "backend_identity": backend_identity, + }), + ]; + let mut journal_paths = Vec::new(); + for fixture in &fixtures { + let data = serde_json::to_vec(fixture).expect("legacy disposition fixture should encode"); + let entry = crate::bucket::lifecycle::tier_delete_journal::decode_tier_delete_journal_entry(&data) + .expect("legacy disposition fixture should decode"); + let path = tier_delete_journal_object_name(&entry); + com::save_config(store.clone(), &path, data) + .await + .expect("legacy disposition fixture should persist"); + journal_paths.push(path); + } + + let recovered = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("legacy disposition recovery scan should finish"); + assert_eq!((recovered.scanned, recovered.deleted, recovered.failed), (2, 0, 0)); + assert_eq!(tier_delete_journal_count(store.clone()).await, 2); + + let mut controls = list_recovery_controls( + store.clone(), + IlmRecoveryProtocol::TierDeleteJournal, + Some(IlmRecoveryClassification::RetainedAmbiguous), + 100, + None, + ) + .await + .expect("legacy disposition controls should be listable") + .records; + controls.sort_by(|left, right| left.control_id.cmp(&right.control_id)); + assert_eq!(controls.len(), 2, "both legacy schemas must support disposition"); + + let actor_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-disposition-actor", ToOwned::to_owned); + let wrong_actor_sha256 = rustfs_utils::crypto::hex_sha256(b"different-disposition-actor", ToOwned::to_owned); + let wrong_export_sha256 = "ff".repeat(32); + for (index, control) in controls.iter().enumerate() { + let observation = inspect_recovery_export_observation(store.clone(), &control.control_id) + .await + .expect("legacy disposition source should be observable"); + let export = create_recovery_export(store.clone(), &observation, &actor_sha256) + .await + .expect("legacy disposition export should persist"); + let confirmed_at_unix_nanos = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()) + .expect("legacy disposition timestamp should fit i64"); + + if index == 0 { + let wrong_hash = Box::pin(execute_recovery_disposition( + store.clone(), + &observation, + &export.export_id, + &wrong_export_sha256, + &actor_sha256, + confirmed_at_unix_nanos, + )) + .await + .expect_err("a mismatched export checksum must fail before local deletion"); + assert_eq!(wrong_hash, Error::PreconditionFailed); + assert_eq!(tier_delete_journal_count(store.clone()).await, 2); + } + + let dry_run = dry_run_recovery_disposition( + store.clone(), + &observation, + &export.export_id, + &export.content_sha256, + &actor_sha256, + confirmed_at_unix_nanos, + ) + .await + .expect("legacy disposition dry-run should validate exact local state"); + assert_eq!(dry_run.source_copy_count, observation.source_generation.copies.len()); + assert_eq!( + tier_delete_journal_count(store.clone()).await, + fixtures.len() - index, + "dry-run must not delete a legacy journal" + ); + assert!( + matches!( + load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id,) + .await, + Err(Error::ConfigNotFound) + ), + "dry-run must not persist a disposition record" + ); + + if index == 0 { + inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterLocalDelete); + Box::pin(execute_recovery_disposition( + store.clone(), + &observation, + &export.export_id, + &export.content_sha256, + &actor_sha256, + confirmed_at_unix_nanos, + )) + .await + .expect_err("the injected crash must stop after local delete commits"); + assert_eq!(tier_delete_journal_count(store.clone()).await, 1); + let interrupted = + load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id) + .await + .expect("the applying disposition must survive the post-delete crash"); + assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying); + assert!( + interrupted.disposition.confirmed_absent.is_empty(), + "the crash must occur before absence progress is persisted" + ); + } else { + inject_recovery_disposition_crash_once(RecoveryDispositionCrashStage::AfterControlAbandon); + Box::pin(execute_recovery_disposition( + store.clone(), + &observation, + &export.export_id, + &export.content_sha256, + &actor_sha256, + confirmed_at_unix_nanos, + )) + .await + .expect_err("the injected crash must stop after control abandonment commits"); + let interrupted = + load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &dry_run.disposition_id) + .await + .expect("the applying disposition must survive the post-control crash"); + assert_eq!(interrupted.disposition.state, IlmRecoveryDispositionState::Applying); + assert_eq!( + interrupted.disposition.confirmed_absent.len(), + interrupted.disposition.identity.source_generation.copies.len() + ); + + let abandoned = + load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id) + .await + .expect("the abandoned control must survive the injected crash"); + let exact_abandoned = abandoned.control.encode().expect("the exact abandoned control should encode"); + let mut wrong_history = abandoned.control; + wrong_history.last_error_code = IlmRecoveryErrorCode::CleanupFailed; + let control_path = recovery_control_record_object_name(observation.protocol, &observation.control_id) + .expect("control path should remain canonical"); + com::save_config( + store.clone(), + &control_path, + wrong_history + .encode() + .expect("the alternate valid control history should encode"), + ) + .await + .expect("the alternate control history fixture should persist"); + let wrong_history_err = Box::pin(execute_recovery_disposition( + store.clone(), + &observation, + &export.export_id, + &export.content_sha256, + &actor_sha256, + confirmed_at_unix_nanos + 1, + )) + .await + .expect_err("a different abandoned control history must not bridge to completion"); + assert_eq!(wrong_history_err, Error::PreconditionFailed); + com::save_config(store.clone(), &control_path, exact_abandoned) + .await + .expect("the exact abandoned control fixture should be restored"); + } + + let replay_confirmed_at_unix_nanos = confirmed_at_unix_nanos + 2; + let executed = Box::pin(execute_recovery_disposition( + store.clone(), + &observation, + &export.export_id, + &export.content_sha256, + &actor_sha256, + replay_confirmed_at_unix_nanos, + )) + .await + .expect("a later request must resume and complete the interrupted disposition"); + assert_eq!(executed.state, IlmRecoveryDispositionState::Completed); + assert_eq!(executed.outcome, IlmRecoveryDispositionExecutionOutcome::Completed); + assert_eq!(executed.confirmed_absent_copy_count, executed.source_copy_count); + assert_eq!(tier_delete_journal_count(store.clone()).await, fixtures.len() - index - 1); + assert!(matches!( + com::read_config(store.clone(), &observation.canonical_source_path).await, + Err(Error::ConfigNotFound) + )); + if index == 0 { + let untouched = journal_paths + .iter() + .find(|path| *path != &observation.canonical_source_path) + .expect("the other legacy journal should remain"); + com::read_config(store.clone(), untouched) + .await + .expect("disposition must not remove a different legacy journal"); + } + + let abandoned = load_recovery_control(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id) + .await + .expect("abandoned recovery control should remain inspectable"); + assert_eq!(abandoned.control.classification, IlmRecoveryClassification::Abandoned); + assert_eq!(abandoned.control.revision, observation.control_revision + 1); + assert_eq!(abandoned.control.observed_source_generation, observation.source_generation); + + let persisted = + load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id) + .await + .expect("completed disposition should remain durable"); + assert_eq!(persisted.disposition.state, IlmRecoveryDispositionState::Completed); + + let replayed = Box::pin(execute_recovery_disposition( + store.clone(), + &observation, + &export.export_id, + &export.content_sha256, + &actor_sha256, + replay_confirmed_at_unix_nanos + 1, + )) + .await + .expect("same actor should replay the completed disposition"); + assert_eq!(replayed.state, IlmRecoveryDispositionState::Completed); + assert_eq!(replayed.outcome, IlmRecoveryDispositionExecutionOutcome::Replayed); + + let wrong_actor = Box::pin(execute_recovery_disposition( + store.clone(), + &observation, + &export.export_id, + &export.content_sha256, + &wrong_actor_sha256, + replay_confirmed_at_unix_nanos + 2, + )) + .await + .expect_err("a different actor must not replay a completed disposition"); + assert_eq!(wrong_actor, Error::PreconditionFailed); + + assert!( + !Box::pin(garbage_collect_completed_recovery_disposition( + store.clone(), + &persisted, + persisted.disposition.retain_until_unix_nanos - 1, + )) + .await + .expect("completed disposition should remain before retention expires") + ); + assert!( + Box::pin(garbage_collect_completed_recovery_disposition( + store.clone(), + &persisted, + persisted.disposition.retain_until_unix_nanos, + )) + .await + .expect("expired completed disposition should be garbage collected") + ); + assert!(matches!( + load_recovery_disposition(store.clone(), IlmRecoveryProtocol::TierDeleteJournal, &executed.disposition_id).await, + Err(Error::ConfigNotFound) + )); + assert_eq!(backend.remove_count().await, 0, "legacy disposition must not call the remote tier"); + assert_eq!(backend.exact_remove_count(), 0, "legacy disposition must not issue exact remote DELETE"); + assert!( + backend.op_log().await.is_empty(), + "legacy disposition must not invoke any backend operation" + ); + } + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] @@ -20328,7 +20630,7 @@ mod tests { bucket: bucket.to_string(), object: object.to_string(), version_id: None, - data_dir: uuid::Uuid::new_v4(), + data_dir: original.data_dir.expect("source object should have data_dir"), mod_time_unix_nanos: original .mod_time .expect("source object should have mod_time") @@ -20462,7 +20764,7 @@ mod tests { bucket: bucket.to_string(), object: object.to_string(), version_id: None, - data_dir: uuid::Uuid::new_v4(), + data_dir: original.data_dir.expect("source object should have data_dir"), mod_time_unix_nanos: original .mod_time .expect("source object should have mod_time") @@ -20603,10 +20905,77 @@ mod tests { IlmRecoveryClassification::RetainedAmbiguous ); let local_commit_control = - load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id) + load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id) .await .expect("local-commit control should persist"); assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired); + + let upload_status = inspect_transition_recovery_retry_for_operator(store.clone(), &upload_started_control_id) + .await + .expect("retained upload should be inspectable for a bounded retry"); + let local_status = inspect_transition_recovery_retry_for_operator(store.clone(), &local_commit_control_id) + .await + .expect("operator-required local commit should be inspectable for a bounded retry"); + assert!(upload_status.retry_ready); + assert!(local_status.retry_ready); + assert!(matches!( + retry_transition_recovery_for_operator( + store.clone(), + &upload_started_control_id, + upload_status.control_revision + 1, + &upload_status.source_generation_sha256, + ) + .await, + Err(TransitionOperatorError::StaleRecoveryControl) + )); + + let put_count_before_retry = backend.put_count().await; + let get_count_before_retry = backend.get_count().await; + let remove_count_before_retry = backend.remove_count().await; + let upload_retry = retry_transition_recovery_for_operator( + store.clone(), + &upload_started_control_id, + upload_status.control_revision, + &upload_status.source_generation_sha256, + ) + .await + .expect("exact retained upload generation should be rearmed"); + let local_retry = retry_transition_recovery_for_operator( + store.clone(), + &local_commit_control_id, + local_status.control_revision, + &local_status.source_generation_sha256, + ) + .await + .expect("exact operator-required local commit generation should be rearmed"); + assert_eq!(upload_retry.classification, IlmRecoveryClassification::Retrying); + assert_eq!(local_retry.classification, IlmRecoveryClassification::Retrying); + assert_eq!(upload_retry.attempt_count, upload_status.attempt_count); + assert_eq!(local_retry.attempt_count, local_status.attempt_count); + assert_eq!(backend.put_count().await, put_count_before_retry); + assert_eq!(backend.get_count().await, get_count_before_retry); + assert_eq!(backend.remove_count().await, remove_count_before_retry); + assert_eq!(backend.exact_remove_count(), 0, "operator retry must not directly issue remote DELETE"); + + let retried = recover_transition_transaction_records(store.clone(), 100, None) + .await + .expect("rearmed records should be re-evaluated through normal recovery"); + assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (2, 0, 2, 0)); + let upload_retained = + load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id) + .await + .expect("upload retry result should persist"); + let local_retained = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id) + .await + .expect("local commit retry result should persist"); + assert_eq!(upload_retained.control.classification, IlmRecoveryClassification::RetainedAmbiguous); + assert_eq!(local_retained.control.classification, IlmRecoveryClassification::OperatorRequired); + assert_eq!(upload_retained.control.attempt_count, upload_status.attempt_count + 1); + assert_eq!(local_retained.control.attempt_count, local_status.attempt_count + 1); + assert_eq!(backend.put_count().await, put_count_before_retry); + assert_eq!(backend.get_count().await, get_count_before_retry); + assert_eq!(backend.remove_count().await, remove_count_before_retry); + assert_eq!(backend.exact_remove_count(), 0); } #[cfg(feature = "test-util")] diff --git a/docs/architecture/ilm-tiering-persistence-contracts.md b/docs/architecture/ilm-tiering-persistence-contracts.md index 4234edd9a..0becd45af 100644 --- a/docs/architecture/ilm-tiering-persistence-contracts.md +++ b/docs/architecture/ilm-tiering-persistence-contracts.md @@ -28,7 +28,7 @@ These are approved-target invariants. A protocol's explicitly labeled current ex | Phase | Authoritative owner | May issue remote DELETE? | Ownership transfer evidence | |---|---|---:|---| | Remote PUT is in flight or its response is unknown | Transition transaction | Only cleanup of its own canonical candidate, subject to the transaction recovery predicate | Durable transaction identity plus a known remote-version state; the approved target also requires expiry and durable takeover of the creator fence | -| Local transition commit is complete | Exact transitioned version in `xl.meta` | No | Current recovery finds the transaction's logical bucket/object/version and checks `TRANSITION_COMPLETE` plus the same remote object, tier, and remote version. It does not compare the recorded data directory, modification time, size, or ETag; the approved target adds that full source comparison | +| Local transition commit is complete | Exact transitioned version in `xl.meta` | No | Recovery finds the transaction's logical bucket/object/version and requires the complete recorded source identity (version ID, data directory, modification time, size, and ETag), `TRANSITION_COMPLETE`, and the same remote object, tier, and remote version before removing only the transaction record | | An ordinary delete removes that transitioned version | Hidden `xl.meta` free-version | Yes | Metadata quorum atomically removes the visible version and preserves its exact tier tuple in the free-version | | A recursive prefix/delete-all operation cannot preserve per-object markers | v6 journal bound to an immutable single dispatch manifest or a chunk-parent-bound child manifest | Yes, but only after child/manifest completion and all-pool absence proof | `DispatchAuthorized`, exact local destructive mutation, every journal `Committed`, then child/manifest `Completed`; a chunk parent advances only after that child completion | | Tier configuration mutation, manual job, or decommission receipt | Intent/admission/copy proof only | No | These records gate configuration, scheduling, or migration; they never become remote-object cleanup owners | @@ -134,7 +134,7 @@ The creator owns the canonical remote candidate until local metadata commits the | `UploadOutcomeUnknown`; probe returns `VersionedPresent` whose identifier is a nil UUID | Transaction recovery retains ownership evidence | Retain | A nil identifier is invalid exact-version evidence and never becomes unversioned or remote-delete authority | | `UploadOutcomeUnknown`; probe ambiguous, unsupported, or errors | Transaction recovery retains ownership evidence | Retain | No destructive action; operator reconcile may inspect after expiry | | `Uploaded` | Originating transition attempt until expiry; after expiry, the worker that wins `Uploaded -> CleanupPending` by exact ETag CAS | Retain while active; after expiry, persist `CleanupPending`, then recheck and delete the unreferenced candidate or record | Current CAS fences the predecessor, but approved v2 also requires a durable recovery lease, full all-pool source/free-version proof, and before/after fence checks | -| `LocalCommitStarted`; logical source lookup returns `TRANSITION_COMPLETE` with the same remote object, tier, and remote version | Transition committer until ownership transfers to `xl.meta` | Delete transaction record | Current recovery treats this tuple as ownership transfer. The approved target additionally compares recorded source version ID, data directory, modification time, size, and ETag before conditional terminal cleanup | +| `LocalCommitStarted`; logical source lookup returns the complete recorded source identity and `TRANSITION_COMPLETE` with the same remote object, tier, and remote version | Transition committer until ownership transfers to `xl.meta` | Delete transaction record | Recovery treats only the full source and remote tuple as ownership transfer; a mismatch remains operator-required and cannot authorize remote deletion | | `LocalCommitStarted`; logical source is missing, its transition tuple differs, or the read is uncertain | Transaction record/recovery | Retain | No remote delete without a separate durable cleanup proof | | `CleanupPending`; logical source lookup returns the same current transition predicate | `xl.meta` is remote reachability owner; recovery owns only record cleanup | Delete transaction record | `xl.meta` is owner; do not delete remote. The approved target adds the full recorded source comparison | | `CleanupPending`; logical source is absent or its transition tuple differs | Transaction recovery | Delete exact candidate, then record | Cleanup proof, known version state, exact backend lease, durable owner fence, and before/after identity checks | diff --git a/docs/operations/tier-ilm-debugging.md b/docs/operations/tier-ilm-debugging.md index d5da40af3..52e6c680b 100644 --- a/docs/operations/tier-ilm-debugging.md +++ b/docs/operations/tier-ilm-debugging.md @@ -173,7 +173,7 @@ Historical transition transactions in `upload_outcome_unknown` state can use an ## Inspect and disposition retained recovery records -This section describes an **approved target that is not implemented yet**. Current servers do not expose the routes below and continue to quarantine tier-delete journal v1/v2 records. Do not remove internal metadata objects by hand: that loses ETag, all-pool, decommission, export, and audit guarantees. +Current servers expose the routes below for retained recovery controls. Do not remove internal metadata objects by hand: that loses ETag, all-pool, decommission, export, and audit guarantees. The approved read-only inventory is bounded and paginated: @@ -223,6 +223,39 @@ Malformed/unsupported records and journal v3-v6 cannot use abandon. Known-versio Automatic retry state survives restart. Retryable transport/quorum failures use a 60-second exponential base capped at one hour and a deterministic 80-to-100-percent multiplier, so jitter never increases the capped delay. After 32 consecutive failures or seven days from the first persisted failure, automatic work stops at `operator_required`. Unsupported or ambiguous evidence goes directly to `retained_ambiguous`/`operator_required`; age alone never deletes it. Resolved controls, immutable exports, and completed disposition receipts have minimum 30-day, 90-day, and 365-day retention respectively, and are collected only after exact source absence, decommission, successor, and audit checks. +### Retry a retained transition transaction + +For a `transition_transaction` control, inspect returns an additional `transition_retry` object when the exact transaction source and recovery-control generation are still consistent. It contains `retry_ready`, `control_revision`, `source_generation_sha256`, the current classification and counters, and a bounded refusal reason. A missing `transition_retry` with `transition_retry_not_ready_reason=source_or_control_not_ready` means the server could not reconstruct exact live evidence; do not retry from an older response. + +First perform a dry-run with the exact revision and source-generation digest returned by the latest inspect: + +```json +POST /rustfs/admin/v3/ilm/recovery/records/ +{ + "action": "retry_transition_recovery", + "mode": "dry_run", + "expected_control_revision": 7, + "expected_source_generation_sha256": "" +} +``` + +After repairing the reported storage, tier, or capability problem, repeat inspect and dry-run, then execute with the newly observed values: + +```json +POST /rustfs/admin/v3/ilm/recovery/records/ +{ + "action": "retry_transition_recovery", + "mode": "execute", + "expected_control_revision": 7, + "expected_source_generation_sha256": "", + "confirm": true +} +``` + +Execution performs one ETag-CAS update of the exact ownerless `retained_ambiguous` or `operator_required` control to `retrying`. It preserves the lifetime attempt count and failure history, clears only the consecutive-failure backoff, and does not mutate the transaction source or issue a tier PUT, GET, probe, or DELETE. The normal recovery worker then acquires a fresh bounded owner lease and repeats every source and remote proof before any side effect. + +A historical v1 `UploadStarted` record can return to `retained_ambiguous` because its bytes do not prove whether PUT reached the provider. `LocalCommitStarted` becomes terminal only when the local object still matches the recorded version ID, data directory, modification time, size, ETag, and exact transitioned remote tuple; otherwise it returns to `operator_required`. Retrying is therefore a bounded re-evaluation after an underlying repair, not an override of missing evidence. + The full schema, lease, mixed-version, retry, privacy, and metric requirements are in [../architecture/ilm-tiering-persistence-contracts.md](../architecture/ilm-tiering-persistence-contracts.md#bounded-recovery-control-and-operator-disposition). ## Reconcile legacy transition-version metadata diff --git a/rustfs/src/admin/handlers/ilm_transition.rs b/rustfs/src/admin/handlers/ilm_transition.rs index 23c545a3c..87595ff8e 100644 --- a/rustfs/src/admin/handlers/ilm_transition.rs +++ b/rustfs/src/admin/handlers/ilm_transition.rs @@ -18,18 +18,21 @@ use crate::admin::runtime_sources::{current_action_credentials, object_store_fro use crate::admin::storage_api::bucket::is_reserved_or_invalid_bucket; use crate::admin::storage_api::error::StorageError; use crate::admin::storage_api::lifecycle::{ - IlmRecoveryClassification, IlmRecoveryControlView, IlmRecoveryExportObservation, IlmRecoveryProtocol, - ManualTransitionCancelCheck, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink, - ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission, - ManualTransitionScopeAdmissionClaim, TransitionOperatorDeleteResult, TransitionOperatorError, + IlmRecoveryClassification, IlmRecoveryControlView, IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionReasonCode, + IlmRecoveryDispositionState, IlmRecoveryExportObservation, IlmRecoveryProtocol, ManualTransitionCancelCheck, + ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink, ManualTransitionQueueSnapshot, + ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission, ManualTransitionScopeAdmissionClaim, + TransitionOperatorDeleteResult, TransitionOperatorError, TransitionRecoveryRetryResult, TransitionRecoveryRetryStatus, claim_manual_transition_scope_admission, create_recovery_export, delete_manual_transition_scope_admission_if_current, - delete_transition_candidate_for_operator, enqueue_transition_for_existing_objects_scoped, - finalize_missing_transition_transaction_for_operator, inspect_recovery_control, inspect_recovery_export_observation, + delete_transition_candidate_for_operator, dry_run_recovery_disposition, enqueue_transition_for_existing_objects_scoped, + execute_recovery_disposition, finalize_missing_transition_transaction_for_operator, inspect_recovery_control, + inspect_recovery_export_observation, inspect_transition_recovery_retry_for_operator, inspect_transition_transaction_for_operator, list_recovery_controls, load_manual_transition_job_record, load_manual_transition_scope_admission, load_recovery_export, manual_transition_job_lease_expired, manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired, persist_manual_transition_job_progress_if_owned, renew_manual_transition_job_lease_if_owned, - request_manual_transition_job_cancel, save_manual_transition_job_record, update_manual_transition_job_record, + request_manual_transition_job_cancel, retry_transition_recovery_for_operator, save_manual_transition_job_record, + update_manual_transition_job_record, }; use crate::admin::storage_api::runtime::ECStore; use crate::admin::storage_api::s3::{S3ErrorCode as AdminS3ErrorCode, error as admin_s3_error}; @@ -257,7 +260,7 @@ pub fn register_ilm_transition_route(r: &mut S3Router) -> std::i r.insert( Method::POST, format!("{ADMIN_PREFIX}/v3/ilm/recovery/records/{{control_id}}").as_str(), - AdminOperation(&IlmRecoveryExportCreateHandler {}), + AdminOperation(&IlmRecoveryRecordMutationHandler {}), )?; r.insert( Method::GET, @@ -540,6 +543,16 @@ fn map_recovery_export_error(err: StorageError) -> S3Error { } } +fn map_recovery_disposition_error(err: StorageError) -> S3Error { + if err == StorageError::ConfigNotFound { + admin_s3_error(AdminS3ErrorCode::NoSuchKey, "ILM recovery export or disposition not found") + } else if err == StorageError::SlowDown { + admin_s3_error(AdminS3ErrorCode::SlowDown, "ILM recovery disposition admission capacity is exhausted") + } else { + admin_s3_error(AdminS3ErrorCode::OperationAborted, "ILM recovery disposition request cannot proceed") + } +} + fn recovery_export_download_headers(export_id: &str, encoded_len: usize) -> S3Result { let mut headers = HeaderMap::new(); headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/json")); @@ -603,12 +616,14 @@ struct IlmRecoveryControlInspectResponse { observation_receipt: Option, #[serde(skip_serializing_if = "Option::is_none")] observation_receipt_expires_at_unix_nanos: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -enum IlmRecoveryDispositionReasonCode { - LegacyRemoteCleanupAbandoned, + #[serde(skip_serializing_if = "Option::is_none")] + disposition_dry_run_receipt: Option, + #[serde(skip_serializing_if = "Option::is_none")] + disposition_dry_run_receipt_expires_at_unix_nanos: Option, + #[serde(skip_serializing_if = "Option::is_none")] + transition_retry: Option, + #[serde(skip_serializing_if = "Option::is_none")] + transition_retry_not_ready_reason: Option<&'static str>, } #[derive(Debug, Deserialize)] @@ -628,6 +643,13 @@ enum IlmRecoveryRecordMutationRequest { #[serde(default)] acknowledge_remote_cleanup_abandoned: Option, }, + RetryTransitionRecovery { + mode: IlmRecoveryReceiptMode, + expected_control_revision: u64, + expected_source_generation_sha256: String, + #[serde(default)] + confirm: Option, + }, } fn parse_recovery_record_mutation_request(body: &[u8]) -> S3Result { @@ -650,10 +672,22 @@ fn parse_recovery_record_mutation_request(body: &[u8]) -> S3Result { Export { observation_receipt: &'a str, @@ -670,6 +704,14 @@ enum ValidatedIlmRecoveryRecordMutation<'a> { export_sha256: &'a str, reason_code: IlmRecoveryDispositionReasonCode, }, + RetryTransitionDryRun { + expected_control_revision: u64, + expected_source_generation_sha256: &'a str, + }, + RetryTransitionExecute { + expected_control_revision: u64, + expected_source_generation_sha256: &'a str, + }, } fn validate_recovery_record_mutation_request( @@ -725,9 +767,62 @@ fn validate_recovery_record_mutation_request( )), } } + IlmRecoveryRecordMutationRequest::RetryTransitionRecovery { + mode, + expected_control_revision, + expected_source_generation_sha256, + confirm, + } => { + if *expected_control_revision == 0 { + return Err(admin_s3_error( + AdminS3ErrorCode::InvalidArgument, + "transition recovery retry requires a nonzero expected control revision", + )); + } + validate_recovery_sha256( + expected_source_generation_sha256, + "invalid transition recovery source generation checksum", + )?; + match mode { + IlmRecoveryReceiptMode::DryRun if confirm.is_none() => { + Ok(ValidatedIlmRecoveryRecordMutation::RetryTransitionDryRun { + expected_control_revision: *expected_control_revision, + expected_source_generation_sha256, + }) + } + IlmRecoveryReceiptMode::DryRun => Err(admin_s3_error( + AdminS3ErrorCode::InvalidArgument, + "ILM recovery retry dry-run must not include confirm", + )), + IlmRecoveryReceiptMode::Execute if *confirm == Some(true) => { + Ok(ValidatedIlmRecoveryRecordMutation::RetryTransitionExecute { + expected_control_revision: *expected_control_revision, + expected_source_generation_sha256, + }) + } + IlmRecoveryReceiptMode::Execute => Err(admin_s3_error( + AdminS3ErrorCode::InvalidRequest, + "transition recovery retry requires confirm=true", + )), + } + } } } +#[derive(Debug, Serialize)] +struct IlmTransitionRecoveryRetryDryRunResponse { + action: &'static str, + mode: IlmRecoveryReceiptMode, + status: TransitionRecoveryRetryStatus, +} + +#[derive(Debug, Serialize)] +struct IlmTransitionRecoveryRetryExecuteResponse { + action: &'static str, + mode: IlmRecoveryReceiptMode, + result: TransitionRecoveryRetryResult, +} + #[derive(Debug, Serialize)] struct IlmRecoveryExportCreateResponse { export_id: String, @@ -736,33 +831,30 @@ struct IlmRecoveryExportCreateResponse { outcome: &'static str, } -// These response envelopes pin the future disposition wire contract before -// its storage state machine is connected to this handler. -#[allow(dead_code)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] enum IlmRecoveryDispositionDryRunStatus { Ready, } -#[allow(dead_code)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] -enum IlmRecoveryDispositionState { +enum IlmRecoveryDispositionResponseState { Applying, Completed, } -#[allow(dead_code)] -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -enum IlmRecoveryDispositionOutcome { - AcceptedForRecovery, - Completed, - Replayed, +fn recovery_disposition_response_state(state: IlmRecoveryDispositionState) -> S3Result { + match state { + IlmRecoveryDispositionState::Applying => Ok(IlmRecoveryDispositionResponseState::Applying), + IlmRecoveryDispositionState::Completed => Ok(IlmRecoveryDispositionResponseState::Completed), + IlmRecoveryDispositionState::Prepared => Err(admin_s3_error( + AdminS3ErrorCode::OperationAborted, + "ILM recovery disposition is accepted but not yet applying", + )), + } } -#[allow(dead_code)] #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct IlmRecoveryDispositionDryRunResponse { @@ -779,15 +871,14 @@ struct IlmRecoveryDispositionDryRunResponse { observation_receipt_expires_at_unix_nanos: i64, } -#[allow(dead_code)] #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct IlmRecoveryDispositionExecuteResponse { action: IlmRecoveryReceiptAction, mode: IlmRecoveryReceiptMode, disposition_id: String, - state: IlmRecoveryDispositionState, - outcome: IlmRecoveryDispositionOutcome, + state: IlmRecoveryDispositionResponseState, + outcome: IlmRecoveryDispositionExecutionOutcome, confirmed_absent_copy_count: usize, source_copy_count: usize, } @@ -928,6 +1019,14 @@ fn map_transition_operator_error(err: TransitionOperatorError) -> S3Error { TransitionOperatorError::CandidateVersionMismatch { .. } => { s3_error!(OperationAborted, "remote candidate version does not match requested exact version") } + TransitionOperatorError::StaleRecoveryControl => admin_s3_error( + AdminS3ErrorCode::OperationAborted, + "transition recovery control or source generation changed", + ), + TransitionOperatorError::RetryNotAllowed => admin_s3_error( + AdminS3ErrorCode::OperationAborted, + "transition recovery control is not eligible for operator retry", + ), TransitionOperatorError::Store(_) | TransitionOperatorError::Remote(_) => { s3_error!(InternalError, "transition reconciliation failed") } @@ -1531,21 +1630,58 @@ impl Operation for IlmRecoveryControlInspectHandler { let control = inspect_recovery_control(store.clone(), &control_id) .await .map_err(map_recovery_control_error)?; + let (transition_retry, transition_retry_not_ready_reason) = + if control.protocol == IlmRecoveryProtocol::TransitionTransaction { + match inspect_transition_recovery_retry_for_operator(store.clone(), &control_id).await { + Ok(status) => (Some(status), None), + Err(_) => (None, Some("source_or_control_not_ready")), + } + } else { + (None, None) + }; let now = OffsetDateTime::now_utc(); - let (export_ready, export_not_ready_reason, observation_receipt, expires_at) = - match inspect_recovery_export_observation(store, &control_id).await { - Ok(observation) => match issue_recovery_observation_receipt( - observation, - actor_sha256, + let ( + export_ready, + export_not_ready_reason, + observation_receipt, + observation_receipt_expires_at_unix_nanos, + disposition_dry_run_receipt, + disposition_dry_run_receipt_expires_at_unix_nanos, + ) = match inspect_recovery_export_observation(store, &control_id).await { + Ok(observation) => { + let export_receipt = issue_recovery_observation_receipt( + observation.clone(), + actor_sha256.clone(), IlmRecoveryReceiptAction::Export, IlmRecoveryReceiptMode::Execute, now, - ) { + ); + let disposition_receipt = issue_recovery_observation_receipt( + observation, + actor_sha256, + IlmRecoveryReceiptAction::AbandonRemoteCleanup, + IlmRecoveryReceiptMode::DryRun, + now, + ); + let (export_ready, export_not_ready_reason, observation_receipt, export_expires_at) = match export_receipt { Ok((token, expires_at)) => (true, None, Some(token), Some(expires_at)), Err(_) => (false, Some("receipt_key_unavailable"), None, None), - }, - Err(_) => (false, Some("fleet_or_source_not_ready"), None, None), - }; + }; + let (disposition_receipt, disposition_expires_at) = match disposition_receipt { + Ok((token, expires_at)) => (Some(token), Some(expires_at)), + Err(_) => (None, None), + }; + ( + export_ready, + export_not_ready_reason, + observation_receipt, + export_expires_at, + disposition_receipt, + disposition_expires_at, + ) + } + Err(_) => (false, Some("fleet_or_source_not_ready"), None, None, None, None), + }; json_response( StatusCode::OK, &IlmRecoveryControlInspectResponse { @@ -1553,16 +1689,20 @@ impl Operation for IlmRecoveryControlInspectHandler { export_ready, export_not_ready_reason, observation_receipt, - observation_receipt_expires_at_unix_nanos: expires_at, + observation_receipt_expires_at_unix_nanos, + disposition_dry_run_receipt, + disposition_dry_run_receipt_expires_at_unix_nanos, + transition_retry, + transition_retry_not_ready_reason, }, ) } } -pub struct IlmRecoveryExportCreateHandler {} +pub struct IlmRecoveryRecordMutationHandler {} #[async_trait::async_trait] -impl Operation for IlmRecoveryExportCreateHandler { +impl Operation for IlmRecoveryRecordMutationHandler { async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { let actor_sha256 = authorize_recovery_admin_request(&req, AdminAction::SetTierAction).await?; let control_id = recovery_control_id_from_params(¶ms)?; @@ -1570,35 +1710,161 @@ impl Operation for IlmRecoveryExportCreateHandler { return Err(admin_s3_error(AdminS3ErrorCode::InternalError, "object store is not initialized")); }; let body = req.input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await.map_err(|_| { - admin_s3_error(AdminS3ErrorCode::InvalidRequest, "ILM recovery export body is too large or unreadable") + admin_s3_error(AdminS3ErrorCode::InvalidRequest, "ILM recovery request body is too large or unreadable") })?; let request = parse_recovery_record_mutation_request(&body)?; - let ValidatedIlmRecoveryRecordMutation::Export { observation_receipt } = - validate_recovery_record_mutation_request(&request)? - else { - return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "unsupported ILM recovery action")); - }; - let receipt = decode_recovery_receipt(observation_receipt, &recovery_receipt_credentials()?)?; - let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()) + let mutation = validate_recovery_record_mutation_request(&request)?; + let now = OffsetDateTime::now_utc(); + let now_unix_nanos = i64::try_from(now.unix_timestamp_nanos()) .map_err(|_| admin_s3_error(AdminS3ErrorCode::InternalError, "ILM recovery receipt timestamp is invalid"))?; - let observation = validate_recovery_observation_receipt( - receipt, - &actor_sha256, - &control_id, - IlmRecoveryReceiptAction::Export, - IlmRecoveryReceiptMode::Execute, - now, - )?; - let created = create_recovery_export(store, &observation, &actor_sha256) - .await - .map_err(map_recovery_export_error)?; - let response = IlmRecoveryExportCreateResponse { - download_url: format!("{ADMIN_PREFIX}/v3/ilm/recovery/exports/{}", created.export_id), - outcome: if created.replayed { "replayed" } else { "created" }, - export_id: created.export_id, - export_sha256: created.content_sha256, - }; - json_response(StatusCode::OK, &response) + match mutation { + ValidatedIlmRecoveryRecordMutation::Export { observation_receipt } => { + let receipt_credentials = recovery_receipt_credentials()?; + let receipt = decode_recovery_receipt(observation_receipt, &receipt_credentials)?; + let observation = validate_recovery_observation_receipt( + receipt, + &actor_sha256, + &control_id, + IlmRecoveryReceiptAction::Export, + IlmRecoveryReceiptMode::Execute, + now_unix_nanos, + )?; + let created = create_recovery_export(store, &observation, &actor_sha256) + .await + .map_err(map_recovery_export_error)?; + let response = IlmRecoveryExportCreateResponse { + download_url: format!("{ADMIN_PREFIX}/v3/ilm/recovery/exports/{}", created.export_id), + outcome: if created.replayed { "replayed" } else { "created" }, + export_id: created.export_id, + export_sha256: created.content_sha256, + }; + json_response(StatusCode::OK, &response) + } + ValidatedIlmRecoveryRecordMutation::AbandonDryRun { + observation_receipt, + export_id, + export_sha256, + reason_code: IlmRecoveryDispositionReasonCode::LegacyRemoteCleanupAbandoned, + } => { + let receipt_credentials = recovery_receipt_credentials()?; + let receipt = decode_recovery_receipt(observation_receipt, &receipt_credentials)?; + let observation = validate_recovery_observation_receipt( + receipt, + &actor_sha256, + &control_id, + IlmRecoveryReceiptAction::AbandonRemoteCleanup, + IlmRecoveryReceiptMode::DryRun, + now_unix_nanos, + )?; + let dry_run = + dry_run_recovery_disposition(store, &observation, export_id, export_sha256, &actor_sha256, now_unix_nanos) + .await + .map_err(map_recovery_disposition_error)?; + let execute_receipt_now = OffsetDateTime::now_utc(); + let (execute_receipt, execute_receipt_expires_at_unix_nanos) = issue_recovery_observation_receipt( + observation, + actor_sha256, + IlmRecoveryReceiptAction::AbandonRemoteCleanup, + IlmRecoveryReceiptMode::Execute, + execute_receipt_now, + )?; + json_response( + StatusCode::OK, + &IlmRecoveryDispositionDryRunResponse { + action: IlmRecoveryReceiptAction::AbandonRemoteCleanup, + mode: IlmRecoveryReceiptMode::DryRun, + status: IlmRecoveryDispositionDryRunStatus::Ready, + disposition_id: dry_run.disposition_id, + export_id: dry_run.export_id, + export_sha256: dry_run.export_content_sha256, + source_generation_sha256: dry_run.source_generation_sha256, + copy_set_sha256: dry_run.copy_set_sha256, + source_copy_count: dry_run.source_copy_count, + observation_receipt: execute_receipt, + observation_receipt_expires_at_unix_nanos: execute_receipt_expires_at_unix_nanos, + }, + ) + } + ValidatedIlmRecoveryRecordMutation::AbandonExecute { + observation_receipt, + export_id, + export_sha256, + reason_code: IlmRecoveryDispositionReasonCode::LegacyRemoteCleanupAbandoned, + } => { + let receipt_credentials = recovery_receipt_credentials()?; + let receipt = decode_recovery_receipt(observation_receipt, &receipt_credentials)?; + let observation = validate_recovery_observation_receipt( + receipt, + &actor_sha256, + &control_id, + IlmRecoveryReceiptAction::AbandonRemoteCleanup, + IlmRecoveryReceiptMode::Execute, + now_unix_nanos, + )?; + let execution = + execute_recovery_disposition(store, &observation, export_id, export_sha256, &actor_sha256, now_unix_nanos) + .await + .map_err(map_recovery_disposition_error)?; + let state = recovery_disposition_response_state(execution.state)?; + json_response( + StatusCode::OK, + &IlmRecoveryDispositionExecuteResponse { + action: IlmRecoveryReceiptAction::AbandonRemoteCleanup, + mode: IlmRecoveryReceiptMode::Execute, + disposition_id: execution.disposition_id, + state, + outcome: execution.outcome, + confirmed_absent_copy_count: execution.confirmed_absent_copy_count, + source_copy_count: execution.source_copy_count, + }, + ) + } + ValidatedIlmRecoveryRecordMutation::RetryTransitionDryRun { + expected_control_revision, + expected_source_generation_sha256, + } => { + let status = inspect_transition_recovery_retry_for_operator(store, &control_id) + .await + .map_err(map_transition_operator_error)?; + if !status.retry_ready { + return Err(map_transition_operator_error(TransitionOperatorError::RetryNotAllowed)); + } + if status.control_revision != expected_control_revision + || status.source_generation_sha256 != expected_source_generation_sha256 + { + return Err(map_transition_operator_error(TransitionOperatorError::StaleRecoveryControl)); + } + json_response( + StatusCode::OK, + &IlmTransitionRecoveryRetryDryRunResponse { + action: "retry_transition_recovery", + mode: IlmRecoveryReceiptMode::DryRun, + status, + }, + ) + } + ValidatedIlmRecoveryRecordMutation::RetryTransitionExecute { + expected_control_revision, + expected_source_generation_sha256, + } => { + let result = retry_transition_recovery_for_operator( + store, + &control_id, + expected_control_revision, + expected_source_generation_sha256, + ) + .await + .map_err(map_transition_operator_error)?; + json_response( + StatusCode::OK, + &IlmTransitionRecoveryRetryExecuteResponse { + action: "retry_transition_recovery", + mode: IlmRecoveryReceiptMode::Execute, + result, + }, + ) + } + } } } @@ -1812,17 +2078,74 @@ mod tests { assert!(!token.contains("actor-a")); assert!(!token.contains("ilm/tier-delete-journal")); assert_eq!(decode_recovery_receipt(&token, &credentials).unwrap(), payload); - assert!( - validate_recovery_observation_receipt( - payload.clone(), - &payload.actor_sha256, - &payload.observation.control_id, - IlmRecoveryReceiptAction::Export, + let receipt_classes = [ + (payload.clone(), IlmRecoveryReceiptAction::Export, IlmRecoveryReceiptMode::Execute), + ( + IlmRecoveryObservationReceipt { + action: IlmRecoveryReceiptAction::AbandonRemoteCleanup, + mode: IlmRecoveryReceiptMode::DryRun, + ..payload.clone() + }, + IlmRecoveryReceiptAction::AbandonRemoteCleanup, + IlmRecoveryReceiptMode::DryRun, + ), + ( + IlmRecoveryObservationReceipt { + action: IlmRecoveryReceiptAction::AbandonRemoteCleanup, + mode: IlmRecoveryReceiptMode::Execute, + ..payload.clone() + }, + IlmRecoveryReceiptAction::AbandonRemoteCleanup, IlmRecoveryReceiptMode::Execute, - payload.issued_at_unix_nanos, + ), + ]; + let expected_classes = [ + (IlmRecoveryReceiptAction::Export, IlmRecoveryReceiptMode::Execute), + (IlmRecoveryReceiptAction::AbandonRemoteCleanup, IlmRecoveryReceiptMode::DryRun), + (IlmRecoveryReceiptAction::AbandonRemoteCleanup, IlmRecoveryReceiptMode::Execute), + ]; + for (receipt, actual_action, actual_mode) in &receipt_classes { + for (expected_action, expected_mode) in expected_classes { + let result = validate_recovery_observation_receipt( + receipt.clone(), + &receipt.actor_sha256, + &receipt.observation.control_id, + expected_action, + expected_mode, + receipt.issued_at_unix_nanos, + ); + if (*actual_action, *actual_mode) == (expected_action, expected_mode) { + assert!(result.is_ok(), "the matching receipt class must validate"); + } else { + assert_eq!( + result.expect_err("receipts must not cross action or mode boundaries").code(), + &S3ErrorCode::AccessDenied + ); + } + } + + let actor_mismatch = validate_recovery_observation_receipt( + receipt.clone(), + &hex_sha256(b"actor-b", ToOwned::to_owned), + &receipt.observation.control_id, + *actual_action, + *actual_mode, + receipt.issued_at_unix_nanos, ) - .is_ok() - ); + .expect_err("receipts must remain bound to the authenticated actor"); + assert_eq!(actor_mismatch.code(), &S3ErrorCode::AccessDenied); + + let expired = validate_recovery_observation_receipt( + receipt.clone(), + &receipt.actor_sha256, + &receipt.observation.control_id, + *actual_action, + *actual_mode, + receipt.expires_at_unix_nanos, + ) + .expect_err("expired receipts must fail closed"); + assert_eq!(expired.code(), &S3ErrorCode::AccessDenied); + } let assert_denied = |receipt: IlmRecoveryObservationReceipt, actor: &str, control: &str, now: i64| { let err = validate_recovery_observation_receipt( receipt, @@ -1835,19 +2158,7 @@ mod tests { .expect_err("invalid observation receipt must be denied"); assert_eq!(err.code(), &S3ErrorCode::AccessDenied); }; - assert_denied( - payload.clone(), - &hex_sha256(b"actor-b", ToOwned::to_owned), - &payload.observation.control_id, - payload.issued_at_unix_nanos, - ); assert_denied(payload.clone(), &payload.actor_sha256, &"cd".repeat(32), payload.issued_at_unix_nanos); - assert_denied( - payload.clone(), - &payload.actor_sha256, - &payload.observation.control_id, - payload.expires_at_unix_nanos, - ); let mut invalid = payload.clone(); invalid.schema = "rustfs-ilm-recovery-observation-receipt-v2".to_string(); @@ -1989,6 +2300,48 @@ mod tests { }) if observed_export_id == export_id && observed_export_sha256 == export_sha256 )); + let source_generation_sha256 = "ef".repeat(32); + let retry_dry_run_json = format!( + r#"{{"action":"retry_transition_recovery","mode":"dry_run","expected_control_revision":7,"expected_source_generation_sha256":"{source_generation_sha256}"}}"# + ); + let retry_dry_run = parse_recovery_record_mutation_request(retry_dry_run_json.as_bytes()).unwrap(); + assert!(matches!( + validate_recovery_record_mutation_request(&retry_dry_run), + Ok(ValidatedIlmRecoveryRecordMutation::RetryTransitionDryRun { + expected_control_revision: 7, + expected_source_generation_sha256: observed, + }) if observed == source_generation_sha256 + )); + let retry_execute_json = retry_dry_run_json.replace(r#""mode":"dry_run""#, r#""mode":"execute","confirm":true"#); + let retry_execute = parse_recovery_record_mutation_request(retry_execute_json.as_bytes()).unwrap(); + assert!(matches!( + validate_recovery_record_mutation_request(&retry_execute), + Ok(ValidatedIlmRecoveryRecordMutation::RetryTransitionExecute { + expected_control_revision: 7, + expected_source_generation_sha256: observed, + }) if observed == source_generation_sha256 + )); + assert!( + parse_recovery_record_mutation_request( + retry_dry_run_json + .replace(r#""mode":"dry_run""#, r#""mode":"dry_run","confirm":false"#) + .as_bytes() + ) + .is_err() + ); + let retry_without_confirmation = retry_execute_json.replace(r#""confirm":true,"#, ""); + if let Ok(request) = parse_recovery_record_mutation_request(retry_without_confirmation.as_bytes()) { + assert!(validate_recovery_record_mutation_request(&request).is_err()); + } + for invalid in [ + retry_execute_json.replace(r#""expected_control_revision":7"#, r#""expected_control_revision":0"#), + retry_execute_json.replace(source_generation_sha256.as_str(), "EF".repeat(32).as_str()), + retry_execute_json.replace(source_generation_sha256.as_str(), "too-short"), + ] { + let request = parse_recovery_record_mutation_request(invalid.as_bytes()).unwrap(); + assert!(validate_recovery_record_mutation_request(&request).is_err()); + } + let dry_run_with_confirmation = dry_run_json.replace(r#""mode":"dry_run""#, r#""mode":"dry_run","confirm":false"#); assert!(parse_recovery_record_mutation_request(dry_run_with_confirmation.as_bytes()).is_err()); assert!(parse_recovery_record_mutation_request(dry_run_json.replace('}', r#","confirm":null}"#).as_bytes()).is_err()); @@ -1997,10 +2350,15 @@ mod tests { for invalid in [ execute_json.replace(r#""confirm":true,"#, ""), execute_json.replace(r#""confirm":true"#, r#""confirm":false"#), + execute_json.replace(r#""confirm":true"#, r#""confirm":null"#), execute_json.replace( r#""acknowledge_remote_cleanup_abandoned":true"#, r#""acknowledge_remote_cleanup_abandoned":false"#, ), + execute_json.replace( + r#""acknowledge_remote_cleanup_abandoned":true"#, + r#""acknowledge_remote_cleanup_abandoned":null"#, + ), execute_json.replace(export_id.as_str(), uppercase_export_id.as_str()), execute_json.replace(export_sha256.as_str(), "too-short"), execute_json.replace("opaque-execute", ""), @@ -2013,8 +2371,19 @@ mod tests { } } + assert!(parse_recovery_record_mutation_request(execute_json.replace(r#""mode":"execute","#, "").as_bytes()).is_err()); + assert!( + parse_recovery_record_mutation_request( + dry_run_json + .replace("legacy_remote_cleanup_abandoned", "operator_override") + .as_bytes() + ) + .is_err() + ); + for invalid in [ br#"{"action":"export","observation_receipt":"opaque","extra":true}"#.as_slice(), + br#"{"action":"abandon_remote_cleanup","mode":null}"#.as_slice(), br#"{"action":"abandon_remote_cleanup","mode":"preview"}"#.as_slice(), br#"{"action":"unknown","observation_receipt":"opaque"}"#.as_slice(), ] { @@ -2054,11 +2423,24 @@ mod tests { observation_receipt_expires_at_unix_nanos: 900_000_000_001, }; let dry_run_json = serde_json::to_value(&dry_run).unwrap(); - assert_eq!(dry_run_json["action"], "abandon_remote_cleanup"); - assert_eq!(dry_run_json["mode"], "dry_run"); - assert_eq!(dry_run_json["status"], "ready"); assert_eq!( - serde_json::from_value::(dry_run_json).unwrap(), + dry_run_json, + serde_json::json!({ + "action": "abandon_remote_cleanup", + "mode": "dry_run", + "status": "ready", + "disposition_id": "ab".repeat(32), + "export_id": "cd".repeat(32), + "export_sha256": "ef".repeat(32), + "source_generation_sha256": "12".repeat(32), + "copy_set_sha256": "34".repeat(32), + "source_copy_count": 2, + "observation_receipt": "opaque-execute", + "observation_receipt_expires_at_unix_nanos": 900_000_000_001_i64, + }) + ); + assert_eq!( + serde_json::from_value::(dry_run_json.clone()).unwrap(), dry_run ); @@ -2066,22 +2448,68 @@ mod tests { action: IlmRecoveryReceiptAction::AbandonRemoteCleanup, mode: IlmRecoveryReceiptMode::Execute, disposition_id: "ab".repeat(32), - state: IlmRecoveryDispositionState::Applying, - outcome: IlmRecoveryDispositionOutcome::AcceptedForRecovery, + state: IlmRecoveryDispositionResponseState::Applying, + outcome: IlmRecoveryDispositionExecutionOutcome::AcceptedForRecovery, confirmed_absent_copy_count: 1, source_copy_count: 2, }; let execute_json = serde_json::to_value(&execute).unwrap(); - assert_eq!(execute_json["state"], "applying"); - assert_eq!(execute_json["outcome"], "accepted_for_recovery"); assert_eq!( - serde_json::from_value::(execute_json).unwrap(), + execute_json, + serde_json::json!({ + "action": "abandon_remote_cleanup", + "mode": "execute", + "disposition_id": "ab".repeat(32), + "state": "applying", + "outcome": "accepted_for_recovery", + "confirmed_absent_copy_count": 1, + "source_copy_count": 2, + }) + ); + assert_eq!( + serde_json::from_value::(execute_json.clone()).unwrap(), execute ); - let mut unknown = serde_json::to_value(&dry_run).unwrap(); - unknown["unexpected"] = serde_json::json!(true); - assert!(serde_json::from_value::(unknown).is_err()); + let mut unknown_dry_run = dry_run_json; + unknown_dry_run["unexpected"] = serde_json::json!(true); + assert!(serde_json::from_value::(unknown_dry_run).is_err()); + + let mut unknown_execute = execute_json; + unknown_execute["unexpected"] = serde_json::json!(true); + assert!(serde_json::from_value::(unknown_execute).is_err()); + } + + #[test] + fn prepared_recovery_disposition_is_operation_aborted_and_not_a_wire_state() { + let err = recovery_disposition_response_state(IlmRecoveryDispositionState::Prepared) + .expect_err("Prepared must not escape through the closed execute response"); + assert_eq!(err.code(), &S3ErrorCode::OperationAborted); + assert_eq!(err.message(), Some("ILM recovery disposition is accepted but not yet applying")); + assert!(serde_json::from_str::(r#""prepared""#).is_err()); + assert_eq!( + serde_json::to_string(&IlmRecoveryDispositionResponseState::Applying).unwrap(), + r#""applying""# + ); + assert_eq!( + serde_json::to_string(&IlmRecoveryDispositionResponseState::Completed).unwrap(), + r#""completed""# + ); + } + + #[test] + fn recovery_disposition_error_mapping_is_stable_and_fail_closed() { + let not_found = map_recovery_disposition_error(StorageError::ConfigNotFound); + assert_eq!(not_found.code(), &S3ErrorCode::NoSuchKey); + assert_eq!(not_found.message(), Some("ILM recovery export or disposition not found")); + + let overloaded = map_recovery_disposition_error(StorageError::SlowDown); + assert_eq!(overloaded.code(), &S3ErrorCode::SlowDown); + assert_eq!(overloaded.message(), Some("ILM recovery disposition admission capacity is exhausted")); + + let stale = map_recovery_disposition_error(StorageError::PreconditionFailed); + assert_eq!(stale.code(), &S3ErrorCode::OperationAborted); + assert_eq!(stale.message(), Some("ILM recovery disposition request cannot proceed")); } #[test] @@ -2097,7 +2525,7 @@ mod tests { ); } - fn manual_transition_job_request(method: Method, path: &'static str) -> S3Request { + fn credential_less_admin_request(method: Method, path: &'static str) -> S3Request { S3Request { input: Body::empty(), method, @@ -2473,7 +2901,7 @@ mod tests { #[tokio::test] async fn transition_admin_gate_keeps_its_missing_credentials_response() { let err = authorize_transition_admin_request( - &manual_transition_job_request(Method::GET, "/rustfs/admin/v3/ilm/transition/jobs/job-123"), + &credential_less_admin_request(Method::GET, "/rustfs/admin/v3/ilm/transition/jobs/job-123"), AdminAction::ListTierAction, ) .await @@ -2486,8 +2914,8 @@ mod tests { #[tokio::test] async fn recovery_admin_gate_keeps_its_missing_credentials_response() { let err = authorize_recovery_admin_request( - &manual_transition_job_request(Method::GET, "/rustfs/admin/v3/ilm/recovery/controls/control-123"), - AdminAction::ListTierAction, + &credential_less_admin_request(Method::POST, "/rustfs/admin/v3/ilm/recovery/records/control-id"), + AdminAction::SetTierAction, ) .await .expect_err("a recovery admin request without credentials must fail"); @@ -2619,7 +3047,7 @@ mod tests { async fn manual_transition_job_handlers_reject_missing_credentials_before_status_contract() { let status_err = ManualTransitionJobStatusHandler {} .call( - manual_transition_job_request(Method::GET, "/rustfs/admin/v3/ilm/transition/jobs/job-123"), + credential_less_admin_request(Method::GET, "/rustfs/admin/v3/ilm/transition/jobs/job-123"), Params::new(), ) .await @@ -2629,7 +3057,7 @@ mod tests { let cancel_err = ManualTransitionJobCancelHandler {} .call( - manual_transition_job_request(Method::DELETE, "/rustfs/admin/v3/ilm/transition/jobs/job-123"), + credential_less_admin_request(Method::DELETE, "/rustfs/admin/v3/ilm/transition/jobs/job-123"), Params::new(), ) .await diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index f41918068..6b6b4179d 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -236,12 +236,18 @@ pub(crate) mod lifecycle { pub(crate) use super::ecstore_bucket::lifecycle::recovery_control::{ IlmRecoveryClassification, IlmRecoveryControlView, IlmRecoveryProtocol, inspect_recovery_control, list_recovery_controls, }; + pub(crate) use super::ecstore_bucket::lifecycle::recovery_disposition::{ + IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionReasonCode, IlmRecoveryDispositionState, + dry_run_recovery_disposition, execute_recovery_disposition, + }; pub(crate) use super::ecstore_bucket::lifecycle::recovery_export::{ IlmRecoveryExportObservation, create_recovery_export, inspect_recovery_export_observation, load_recovery_export, }; pub(crate) use super::ecstore_bucket::lifecycle::transition_transaction::{ - TransitionOperatorDeleteResult, TransitionOperatorError, delete_transition_candidate_for_operator, - finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator, + TransitionOperatorDeleteResult, TransitionOperatorError, TransitionRecoveryRetryResult, TransitionRecoveryRetryStatus, + delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator, + inspect_transition_recovery_retry_for_operator, inspect_transition_transaction_for_operator, + retry_transition_recovery_for_operator, }; pub(crate) async fn enqueue_transition_for_existing_objects_scoped(