diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index e0cc06d26..6e4fc604d 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -76,6 +76,22 @@ pub mod bucket { }; } + pub mod recovery_disposition { + pub use crate::bucket::lifecycle::recovery_disposition::{ + CreatedIlmRecoveryDisposition, IlmRecoveryDisposition, IlmRecoveryDispositionAction, IlmRecoveryDispositionError, + IlmRecoveryDispositionIdentity, IlmRecoveryDispositionOwnerLease, IlmRecoveryDispositionReasonCode, + IlmRecoveryDispositionState, ObservedIlmRecoveryDisposition, create_recovery_disposition_if_absent, + load_recovery_disposition, recovery_disposition_id, save_recovery_disposition_if_current, + }; + } + + pub mod recovery_export { + pub use crate::bucket::lifecycle::recovery_export::{ + IlmRecoveryExportCreated, IlmRecoveryExportObservation, create_recovery_export, + inspect_recovery_export_observation, load_recovery_export, + }; + } + pub mod transition_transaction { pub use crate::bucket::lifecycle::transition_transaction::{ TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus, @@ -446,9 +462,12 @@ pub mod notification { #[cfg(any(test, feature = "test-util"))] pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test; pub use crate::services::notification_sys::{ - ClusterTierDailyStats, CrossPoolFenceFleetProofToken, LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr, - NotificationSys, ScannerPublicationLeaseGrant, acquire_cross_pool_fence_fleet_proof, + ClusterTierDailyStats, CrossPoolFenceFleetProofToken, IlmRecoveryExportFleetProofToken, + LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant, + acquire_cross_pool_fence_fleet_proof, acquire_ilm_recovery_export_fleet_proof, acquire_legacy_transition_state_reconcile_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys, + ilm_recovery_export_fleet_proof_matches, ilm_recovery_export_local_process_epoch, + ilm_recovery_export_member_epochs_sha256, ilm_recovery_export_topology_generation, legacy_transition_state_reconcile_fleet_proof_matches, new_global_notification_sys, scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe, }; diff --git a/crates/ecstore/src/bucket/lifecycle/config_boundary.rs b/crates/ecstore/src/bucket/lifecycle/config_boundary.rs index dc0fa8a83..12b2e2ba5 100644 --- a/crates/ecstore/src/bucket/lifecycle/config_boundary.rs +++ b/crates/ecstore/src/bucket/lifecycle/config_boundary.rs @@ -41,6 +41,21 @@ where com::read_config(api, file).await } +pub(crate) async fn read_config_limited_preserve_empty(api: Arc, file: &str, max_bytes: usize) -> Result> +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + >, +{ + com::read_config_limited_preserve_empty(api, file, max_bytes).await +} + pub(crate) async fn read_config_with_metadata(api: Arc, file: &str, opts: &ObjectOptions) -> Result<(Vec, ObjectInfo)> where S: ObjectIO< @@ -56,6 +71,26 @@ where com::read_config_with_metadata(api, file, opts).await } +pub(crate) async fn read_config_limited_preserve_empty_with_metadata( + api: Arc, + file: &str, + opts: &ObjectOptions, + max_bytes: usize, +) -> Result<(Vec, ObjectInfo)> +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + >, +{ + com::read_config_limited_preserve_empty_with_metadata_opts(api, file, opts, max_bytes).await +} + pub(crate) async fn save_config(api: Arc, file: &str, data: Vec) -> Result<()> where S: ObjectIO< diff --git a/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs index 58e7ae607..67c980d5d 100644 --- a/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs +++ b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs @@ -22,7 +22,7 @@ use super::{ bucket_lifecycle_ops::{ ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token, }, - manual_transition_job, recovery_control, tier_delete_journal, transition_transaction, + manual_transition_job, recovery_control, recovery_disposition, recovery_export, tier_delete_journal, transition_transaction, }; use crate::error::{Error, Result}; use crate::services::tier::tier_probe_intent; @@ -42,6 +42,8 @@ pub(crate) enum DurableIlmRecordKind { ManualTransitionTask, ManualTransitionWorkerResult, RecoveryControl, + RecoveryExport, + RecoveryDisposition, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -112,8 +114,20 @@ pub(crate) const RECOVERY_CONTROL_NAMESPACE: DurableIlmNamespace = DurableIlmNam max_record_size: recovery_control::MAX_ILM_RECOVERY_CONTROL_SIZE, kind: DurableIlmRecordKind::RecoveryControl, }; +pub(crate) const RECOVERY_EXPORT_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "recovery-export", + prefix: recovery_export::ILM_RECOVERY_EXPORT_PREFIX, + max_record_size: recovery_export::MAX_ILM_RECOVERY_EXPORT_SIZE, + kind: DurableIlmRecordKind::RecoveryExport, +}; +pub(crate) const RECOVERY_DISPOSITION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "recovery-disposition", + prefix: recovery_disposition::ILM_RECOVERY_DISPOSITION_PREFIX, + max_record_size: recovery_disposition::MAX_ILM_RECOVERY_DISPOSITION_SIZE, + kind: DurableIlmRecordKind::RecoveryDisposition, +}; -pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [ +pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 12] = [ TIER_DELETE_JOURNAL_NAMESPACE, TIER_DELETE_JOURNAL_V6_NAMESPACE, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE, @@ -124,6 +138,8 @@ pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [ MANUAL_TRANSITION_TASK_NAMESPACE, MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE, RECOVERY_CONTROL_NAMESPACE, + RECOVERY_EXPORT_NAMESPACE, + RECOVERY_DISPOSITION_NAMESPACE, ]; #[derive(Debug, Clone, PartialEq, Eq)] @@ -261,6 +277,28 @@ pub(crate) enum DurableIlmRecordCheckpoint { #[serde(default, skip_serializing_if = "Option::is_none")] owner_fence_sha256: Option, }, + RecoveryExport { + content_sha256: String, + source_generation_sha256: String, + topology_generation: String, + member_epochs_sha256: String, + creator_sha256: String, + retain_until_unix_nanos: i64, + }, + RecoveryDisposition { + content_sha256: String, + identity_sha256: String, + copy_manifest_sha256: String, + copy_manifest_count: usize, + created_at_unix_nanos: i64, + revision: u64, + state: recovery_disposition::IlmRecoveryDispositionState, + owner_fence_sha256: Option, + owner_lease_acquired_at_unix_nanos: Option, + owner_lease_expires_at_unix_nanos: Option, + confirmed_absent_sha256: Vec, + retain_until_unix_nanos: i64, + }, } impl DurableIlmRecordCheckpoint { @@ -275,7 +313,9 @@ impl DurableIlmRecordCheckpoint { | Self::ManualTransitionScope { content_sha256, .. } | Self::ManualTransitionTask { content_sha256 } | Self::ManualTransitionWorkerResult { content_sha256 } - | Self::RecoveryControl { content_sha256, .. } => content_sha256, + | Self::RecoveryControl { content_sha256, .. } + | Self::RecoveryExport { content_sha256, .. } + | Self::RecoveryDisposition { content_sha256, .. } => content_sha256, } } @@ -316,6 +356,9 @@ impl DurableIlmRecordCheckpoint { { return Err(Error::other("durable ILM tier delete journal checkpoint is invalid")); } + if !recovery_disposition_checkpoint_is_valid(checkpoint) { + return Err(Error::other("durable ILM recovery disposition checkpoint is invalid")); + } } if self == next { if let Self::ManualTransitionJob { @@ -594,6 +637,83 @@ impl DurableIlmRecordCheckpoint { && previous_attempts == next_attempts; adjacent && (claim || source_refresh || completion) } + ( + Self::RecoveryDisposition { + identity_sha256: previous_identity, + copy_manifest_sha256: previous_manifest, + copy_manifest_count: previous_manifest_count, + created_at_unix_nanos: previous_created_at, + revision: previous_revision, + state: previous_state, + owner_fence_sha256: previous_owner, + owner_lease_acquired_at_unix_nanos: previous_owner_acquired, + owner_lease_expires_at_unix_nanos: previous_owner_expires, + confirmed_absent_sha256: previous_confirmed, + retain_until_unix_nanos: previous_retain_until, + .. + }, + Self::RecoveryDisposition { + identity_sha256: next_identity, + copy_manifest_sha256: next_manifest, + copy_manifest_count: next_manifest_count, + created_at_unix_nanos: next_created_at, + revision: next_revision, + state: next_state, + owner_fence_sha256: next_owner, + owner_lease_acquired_at_unix_nanos: next_owner_acquired, + owner_lease_expires_at_unix_nanos: next_owner_expires, + confirmed_absent_sha256: next_confirmed, + retain_until_unix_nanos: next_retain_until, + .. + }, + ) => { + use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared}; + + let immutable_identity_matches = previous_identity == next_identity + && previous_manifest == next_manifest + && previous_manifest_count == next_manifest_count + && previous_created_at == next_created_at + && previous_retain_until == next_retain_until; + let adjacent = previous_revision.checked_add(1) == Some(*next_revision); + let progress_is_monotonic = sorted_sha256_set_is_subset(previous_confirmed, next_confirmed); + let legal_edge = match (previous_state, next_state) { + (Prepared, Prepared) => { + let claim = previous_owner.is_none() && next_owner.is_some(); + let takeover = previous_owner.is_some() + && previous_owner != next_owner + && previous_owner_expires + .zip(*next_owner_acquired) + .is_some_and(|(expires, acquired)| acquired >= expires); + previous_confirmed == next_confirmed && (claim || takeover) + } + (Prepared, Applying) => { + previous_confirmed == next_confirmed + && previous_owner.is_some() + && previous_owner == next_owner + && previous_owner_acquired == next_owner_acquired + && previous_owner_expires == next_owner_expires + } + (Applying, Applying) => { + let progress = previous_owner == next_owner + && previous_owner_acquired == next_owner_acquired + && previous_owner_expires == next_owner_expires + && previous_confirmed.len().checked_add(1) == Some(next_confirmed.len()); + let takeover = previous_owner.is_some() + && previous_owner != next_owner + && previous_confirmed == next_confirmed + && previous_owner_expires + .zip(*next_owner_acquired) + .is_some_and(|(expires, acquired)| acquired >= expires); + progress || takeover + } + (Applying, Completed) => { + previous_owner.is_some() && next_owner.is_none() && previous_confirmed == next_confirmed + } + _ => false, + }; + + immutable_identity_matches && adjacent && progress_is_monotonic && legal_edge + } _ => false, }; @@ -611,6 +731,11 @@ impl DurableIlmRecordCheckpoint { /// after the exact terminal ETag and terminal receipt were committed, to /// purge older object versions exposed by that deletion. pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool { + for checkpoint in [self, terminal] { + if !recovery_disposition_checkpoint_is_valid(checkpoint) { + return false; + } + } if let Self::TierProbeIntent { state, .. } = terminal && !matches!( state, @@ -627,6 +752,11 @@ impl DurableIlmRecordCheckpoint { { return false; } + if let Self::RecoveryDisposition { state, .. } = terminal + && state != &recovery_disposition::IlmRecoveryDispositionState::Completed + { + return false; + } if self == terminal || self.validate_successor(terminal).is_ok() { return true; } @@ -752,11 +882,121 @@ impl DurableIlmRecordCheckpoint { && terminal_revision > previous_revision && terminal_attempts >= previous_attempts } + ( + Self::RecoveryDisposition { + identity_sha256: previous_identity, + copy_manifest_sha256: previous_manifest, + copy_manifest_count: previous_manifest_count, + created_at_unix_nanos: previous_created_at, + revision: previous_revision, + state: previous_state, + owner_fence_sha256: previous_owner, + confirmed_absent_sha256: previous_confirmed, + retain_until_unix_nanos: previous_retain_until, + .. + }, + Self::RecoveryDisposition { + identity_sha256: terminal_identity, + copy_manifest_sha256: terminal_manifest, + copy_manifest_count: terminal_manifest_count, + created_at_unix_nanos: terminal_created_at, + revision: terminal_revision, + state: recovery_disposition::IlmRecoveryDispositionState::Completed, + confirmed_absent_sha256: terminal_confirmed, + retain_until_unix_nanos: terminal_retain_until, + .. + }, + ) => { + matches!( + previous_state, + recovery_disposition::IlmRecoveryDispositionState::Prepared + | recovery_disposition::IlmRecoveryDispositionState::Applying + ) && previous_identity == terminal_identity + && previous_manifest == terminal_manifest + && previous_manifest_count == terminal_manifest_count + && previous_created_at == terminal_created_at + && previous_retain_until == terminal_retain_until + && terminal_revision.checked_sub(*previous_revision).is_some_and(|distance| { + let minimum_distance = match previous_state { + recovery_disposition::IlmRecoveryDispositionState::Prepared if previous_owner.is_some() => 3, + recovery_disposition::IlmRecoveryDispositionState::Prepared => 4, + recovery_disposition::IlmRecoveryDispositionState::Applying + if previous_confirmed.len() == *previous_manifest_count => + { + 1 + } + recovery_disposition::IlmRecoveryDispositionState::Applying => 2, + recovery_disposition::IlmRecoveryDispositionState::Completed => u64::MAX, + }; + distance >= minimum_distance + }) + && sorted_sha256_set_is_subset(previous_confirmed, terminal_confirmed) + } _ => false, } } } +fn recovery_disposition_checkpoint_is_valid(checkpoint: &DurableIlmRecordCheckpoint) -> bool { + use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared}; + + let DurableIlmRecordCheckpoint::RecoveryDisposition { + content_sha256, + identity_sha256, + copy_manifest_sha256, + copy_manifest_count, + created_at_unix_nanos, + revision, + state, + owner_fence_sha256, + owner_lease_acquired_at_unix_nanos, + owner_lease_expires_at_unix_nanos, + confirmed_absent_sha256, + retain_until_unix_nanos, + } = checkpoint + else { + return true; + }; + let owner_fence_sha256 = owner_fence_sha256.as_deref(); + + is_canonical_sha256(content_sha256) + && is_canonical_sha256(identity_sha256) + && is_canonical_sha256(copy_manifest_sha256) + && *copy_manifest_count > 0 + && *created_at_unix_nanos > 0 + && *revision > 0 + && *retain_until_unix_nanos > 0 + && owner_fence_sha256.is_none_or(is_canonical_sha256) + && match ( + owner_fence_sha256, + *owner_lease_acquired_at_unix_nanos, + *owner_lease_expires_at_unix_nanos, + ) { + (None, None, None) => true, + (Some(_), Some(acquired), Some(expires)) => acquired > 0 && expires > acquired, + _ => false, + } + && confirmed_absent_sha256.len() <= *copy_manifest_count + && confirmed_absent_sha256.iter().all(|digest| is_canonical_sha256(digest)) + && confirmed_absent_sha256.windows(2).all(|pair| pair[0] < pair[1]) + && match *state { + Prepared => confirmed_absent_sha256.is_empty(), + Applying => owner_fence_sha256.is_some(), + Completed => owner_fence_sha256.is_none() && confirmed_absent_sha256.len() == *copy_manifest_count, + } +} + +fn is_canonical_sha256(value: &str) -> bool { + is_sha256_checksum(value) + && !value + .bytes() + .any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase()) +} + +fn sorted_sha256_set_is_subset(subset: &[String], superset: &[String]) -> bool { + subset.iter().all(|candidate| superset.binary_search(candidate).is_ok()) +} + fn tier_delete_dispatch_parent_progress_delta( previous_sequence: u64, previous_completed_journals: u64, @@ -1348,6 +1588,55 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result { + let (protocol, export_id) = recovery_export::recovery_export_id_from_record_object_name(path)?; + let export = recovery_export::IlmRecoveryExport::decode(&export_id, data)?; + let canonical = recovery_export::recovery_export_record_object_name(protocol, &export_id)?; + if canonical != path || export.protocol != protocol { + return Err(Error::other("ILM recovery export path is not canonical")); + } + let source_generation_sha256 = checkpoint_hash(&export.source_generation)?; + ( + "export_id", + export_id, + DurableIlmRecordCheckpoint::RecoveryExport { + content_sha256, + source_generation_sha256, + topology_generation: export.topology_generation, + member_epochs_sha256: export.member_epochs_sha256, + creator_sha256: export.creator_sha256, + retain_until_unix_nanos: export.retain_until_unix_nanos, + }, + ) + } + DurableIlmRecordKind::RecoveryDisposition => { + // The disposition module owns strict schema, checksum, canonical + // path, immutable-manifest, and state-specific validation. Keep + // this boundary limited to decommission identity/checkpoint + // projection so the two readers cannot accept different records. + let disposition = recovery_disposition::decode_recovery_disposition_checkpoint(path, data)?; + if disposition.content_sha256 != content_sha256 { + return Err(Error::other("ILM recovery disposition checkpoint content digest is invalid")); + } + ( + "disposition_id", + disposition.disposition_id, + DurableIlmRecordCheckpoint::RecoveryDisposition { + content_sha256: disposition.content_sha256, + identity_sha256: disposition.identity_sha256, + copy_manifest_sha256: disposition.copy_manifest_sha256, + copy_manifest_count: disposition.copy_manifest_count, + created_at_unix_nanos: disposition.created_at_unix_nanos, + revision: disposition.revision, + state: disposition.state, + owner_fence_sha256: disposition.owner_fence_sha256, + owner_lease_acquired_at_unix_nanos: disposition.owner_lease_acquired_at_unix_nanos, + owner_lease_expires_at_unix_nanos: disposition.owner_lease_expires_at_unix_nanos, + confirmed_absent_sha256: disposition.confirmed_absent_sha256, + retain_until_unix_nanos: disposition.retain_until_unix_nanos, + }, + ) + } DurableIlmRecordKind::ManualTransitionJob => { let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path) .map_err(|err| Error::other(err.to_string()))?; @@ -1503,6 +1792,203 @@ mod tests { } } + fn recovery_disposition_checkpoint( + revision: u64, + state: recovery_disposition::IlmRecoveryDispositionState, + owner_fence: Option<&str>, + confirmed_absent_sha256: Vec, + ) -> DurableIlmRecordCheckpoint { + let (owner_lease_acquired_at_unix_nanos, owner_lease_expires_at_unix_nanos) = match owner_fence { + Some("f") => (Some(10), Some(20)), + Some(_) => (Some(1), Some(10)), + None => (None, None), + }; + DurableIlmRecordCheckpoint::RecoveryDisposition { + content_sha256: format!("{revision:064x}"), + identity_sha256: "a".repeat(64), + copy_manifest_sha256: "d".repeat(64), + copy_manifest_count: 2, + created_at_unix_nanos: 1_700_000_000_000_000_000, + revision, + state, + owner_fence_sha256: owner_fence.map(|digest| digest.repeat(64)), + owner_lease_acquired_at_unix_nanos, + owner_lease_expires_at_unix_nanos, + confirmed_absent_sha256, + retain_until_unix_nanos: 1_820_000_000_000_000_000, + } + } + + #[test] + fn recovery_disposition_namespace_is_registered_without_shadowing_its_root() { + let disposition_id = "a".repeat(64); + let path = format!( + "{}/tier_delete_journal/{}/{}/{}.json", + recovery_disposition::ILM_RECOVERY_DISPOSITION_PREFIX, + &disposition_id[..2], + &disposition_id[2..4], + disposition_id + ); + let namespace = classify_durable_ilm_record(&path) + .expect("recovery disposition path should classify") + .expect("recovery disposition should be durable"); + + assert_eq!(namespace, &RECOVERY_DISPOSITION_NAMESPACE); + assert!(classify_durable_ilm_record(recovery_disposition::ILM_RECOVERY_DISPOSITION_PREFIX).is_err()); + } + + #[test] + fn recovery_disposition_checkpoint_accepts_only_monotonic_progress() { + use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared}; + + let first_copy = "b".repeat(64); + let second_copy = "c".repeat(64); + let prepared = recovery_disposition_checkpoint(1, Prepared, None, Vec::new()); + let claimed = recovery_disposition_checkpoint(2, Prepared, Some("e"), Vec::new()); + let applying = recovery_disposition_checkpoint(3, Applying, Some("e"), Vec::new()); + let first_absent = recovery_disposition_checkpoint(4, Applying, Some("e"), vec![first_copy.clone()]); + let taken_over = recovery_disposition_checkpoint(5, Applying, Some("f"), vec![first_copy.clone()]); + let all_absent = recovery_disposition_checkpoint(6, Applying, Some("f"), vec![first_copy.clone(), second_copy.clone()]); + let completed = recovery_disposition_checkpoint(7, Completed, None, vec![first_copy.clone(), second_copy.clone()]); + + prepared + .validate_successor(&claimed) + .expect("Prepared should record an owner claim without absence progress"); + claimed + .validate_successor(&applying) + .expect("Prepared should advance to Applying without folding in deletion progress"); + applying + .validate_successor(&first_absent) + .expect("Applying should append newly confirmed absent copies"); + first_absent + .validate_successor(&taken_over) + .expect("Applying should record a fenced owner takeover without losing progress"); + taken_over + .validate_successor(&all_absent) + .expect("Applying should preserve every earlier confirmation while making progress"); + all_absent + .validate_successor(&completed) + .expect("a fully confirmed manifest should advance to Completed"); + + assert!( + prepared.validate_successor(&completed).is_err(), + "adjacent receipt updates must not skip Applying" + ); + assert!( + first_absent + .validate_successor(&recovery_disposition_checkpoint(5, Applying, Some("e"), Vec::new())) + .is_err(), + "confirmed-absent progress must not move backwards" + ); + assert!( + applying + .validate_successor(&recovery_disposition_checkpoint(4, Completed, None, vec![first_copy.clone()])) + .is_err(), + "Completed must cover the complete immutable copy manifest" + ); + assert!( + completed + .validate_successor(&recovery_disposition_checkpoint(7, Applying, Some("e"), vec![second_copy])) + .is_err(), + "Completed is terminal" + ); + assert!( + first_absent + .validate_successor(&recovery_disposition_checkpoint(5, Applying, Some("e"), vec![first_copy.clone()])) + .is_err(), + "a same-state revision bump must change the owner fence or absence progress" + ); + assert!( + applying + .validate_successor(&recovery_disposition_checkpoint(4, Applying, None, vec![first_copy])) + .is_err(), + "Applying must retain a fenced owner" + ); + + let mut noncanonical_identity = claimed.clone(); + if let DurableIlmRecordCheckpoint::RecoveryDisposition { identity_sha256, .. } = &mut noncanonical_identity { + *identity_sha256 = "A".repeat(64); + } + assert!(prepared.validate_successor(&noncanonical_identity).is_err()); + let mut changed_created_at = claimed; + if let DurableIlmRecordCheckpoint::RecoveryDisposition { + created_at_unix_nanos, .. + } = &mut changed_created_at + { + *created_at_unix_nanos += 1; + } + assert!(prepared.validate_successor(&changed_created_at).is_err()); + + let mut early_takeover = taken_over; + if let DurableIlmRecordCheckpoint::RecoveryDisposition { + owner_lease_acquired_at_unix_nanos, + .. + } = &mut early_takeover + { + *owner_lease_acquired_at_unix_nanos = Some(9); + } + assert!(first_absent.validate_successor(&early_takeover).is_err()); + assert!( + applying + .validate_successor(&recovery_disposition_checkpoint( + 4, + Applying, + Some("e"), + vec!["c".repeat(64), "b".repeat(64)], + )) + .is_err(), + "confirmed-absent entries must be a canonical sorted set" + ); + } + + #[test] + fn recovery_disposition_terminal_predecessor_requires_exact_identity_and_full_manifest() { + use recovery_disposition::IlmRecoveryDispositionState::{Applying, Completed, Prepared}; + + let first_copy = "b".repeat(64); + let second_copy = "c".repeat(64); + let prepared = recovery_disposition_checkpoint(1, Prepared, None, Vec::new()); + let applying = recovery_disposition_checkpoint(3, Applying, Some("e"), vec![first_copy.clone()]); + let completed = recovery_disposition_checkpoint(5, Completed, None, vec![first_copy.clone(), second_copy]); + + assert!(prepared.is_predecessor_of_terminal(&completed)); + assert!(applying.is_predecessor_of_terminal(&completed)); + assert!( + !prepared.is_predecessor_of_terminal(&recovery_disposition_checkpoint(2, Applying, Some("e"), Vec::new())), + "a nonterminal disposition must not authorize terminal cleanup" + ); + assert!( + !prepared.is_predecessor_of_terminal(&recovery_disposition_checkpoint( + 4, + Completed, + None, + vec![first_copy.clone(), "c".repeat(64)], + )), + "terminal proof must leave enough revisions for claim, apply, progress, and completion" + ); + assert!( + !applying.is_predecessor_of_terminal(&recovery_disposition_checkpoint( + 4, + Completed, + None, + vec![first_copy.clone(), "c".repeat(64)], + )), + "an incomplete Applying checkpoint cannot complete without a progress generation" + ); + + let mut other_identity = completed; + if let DurableIlmRecordCheckpoint::RecoveryDisposition { identity_sha256, .. } = &mut other_identity { + *identity_sha256 = "e".repeat(64); + } + assert!(!prepared.is_predecessor_of_terminal(&other_identity)); + + let incomplete_terminal = recovery_disposition_checkpoint(4, Completed, None, vec![first_copy]); + assert!( + !prepared.is_predecessor_of_terminal(&incomplete_terminal), + "a partial confirmed-absent set must not become terminal proof" + ); + } + fn tier_probe_intent_fixture() -> tier_probe_intent::TierProbeIntent { let probe_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse"); tier_probe_intent::TierProbeIntent { diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index 3eb93cca4..7197ad832 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -25,6 +25,8 @@ mod object_handlers_common; mod object_lock_boundary; pub use self::core as lifecycle; pub mod recovery_control; +pub mod recovery_disposition; +pub mod recovery_export; mod replication_sink; pub mod rule; mod runtime_boundary; diff --git a/crates/ecstore/src/bucket/lifecycle/recovery_control.rs b/crates/ecstore/src/bucket/lifecycle/recovery_control.rs index 599849ca9..de6224e82 100644 --- a/crates/ecstore/src/bucket/lifecycle/recovery_control.rs +++ b/crates/ecstore/src/bucket/lifecycle/recovery_control.rs @@ -168,7 +168,7 @@ impl IlmRecoverySourceGeneration { Ok(generation) } - fn validate(&self) -> Result<()> { + pub(crate) fn validate(&self) -> Result<()> { if self.source_schema.trim().is_empty() { return Err(IlmRecoveryControlError::Corrupt("source schema is empty")); } diff --git a/crates/ecstore/src/bucket/lifecycle/recovery_disposition.rs b/crates/ecstore/src/bucket/lifecycle/recovery_disposition.rs new file mode 100644 index 000000000..5721971ea --- /dev/null +++ b/crates/ecstore/src/bucket/lifecycle/recovery_disposition.rs @@ -0,0 +1,1222 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::config_boundary; +use super::recovery_control::{ + IlmRecoveryClassification, IlmRecoveryProtocol, IlmRecoverySourceGeneration, load_recovery_control, observe_recovery_source, + recovery_control_record_object_name, +}; +use super::recovery_export::{IlmRecoveryExport, load_recovery_export, recovery_export_id}; +use super::tier_delete_journal::{ + TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, validate_legacy_tier_delete_recovery_path, +}; +use crate::disk::RUSTFS_META_BUCKET; +use crate::error::{Error, Result as EcstoreResult}; +use crate::object_api::{ObjectOptions, WriteCompletion}; +use crate::storage_api_contracts::{namespace::NamespaceLocking as _, object::HTTPPreconditions}; +use crate::store::ECStore; + +pub const ILM_RECOVERY_DISPOSITION_SCHEMA: &str = "rustfs-ilm-recovery-disposition-v1"; +pub const ILM_RECOVERY_DISPOSITION_PREFIX: &str = "ilm/recovery-dispositions"; +pub const MAX_ILM_RECOVERY_DISPOSITION_SIZE: usize = 16 * 1024; +const DISPOSITION_RETENTION_NANOS: i64 = 365 * 24 * 60 * 60 * 1_000_000_000; +const OWNER_FENCE_CHECKPOINT_DOMAIN: &[u8] = b"rustfs-ilm-recovery-disposition-owner-fence-v1"; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum IlmRecoveryDispositionError { + #[error("ILM recovery disposition is corrupt: {0}")] + Corrupt(&'static str), + #[error("ILM recovery disposition schema is unsupported: {0}")] + UnsupportedSchema(String), + #[error("ILM recovery disposition checksum mismatch")] + ChecksumMismatch, + #[error("ILM recovery disposition successor is invalid: {0}")] + InvalidSuccessor(&'static str), + #[error("ILM recovery disposition json error: {0}")] + Json(#[from] serde_json::Error), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IlmRecoveryDispositionAction { + AbandonRemoteCleanup, +} + +impl IlmRecoveryDispositionAction { + pub const fn as_str(self) -> &'static str { + match self { + Self::AbandonRemoteCleanup => "abandon_remote_cleanup", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IlmRecoveryDispositionReasonCode { + LegacyRemoteCleanupAbandoned, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IlmRecoveryDispositionState { + Prepared, + Applying, + Completed, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IlmRecoveryDispositionIdentity { + pub disposition_id: String, + pub protocol: IlmRecoveryProtocol, + pub action: IlmRecoveryDispositionAction, + pub export_id: String, + pub export_content_sha256: String, + pub control_id: String, + pub control_etag: String, + pub control_revision: u64, + pub canonical_source_path: String, + pub source_generation: IlmRecoverySourceGeneration, + pub admitted_topology_generation: String, + pub admitted_member_epochs_sha256: String, + pub actor_sha256: String, + pub reason_code: IlmRecoveryDispositionReasonCode, + pub confirmed_at_unix_nanos: i64, +} + +impl IlmRecoveryDispositionIdentity { + fn validate(&self) -> Result<()> { + validate_sha256(&self.disposition_id, "disposition ID is invalid")?; + validate_sha256(&self.export_id, "export ID is invalid")?; + validate_sha256(&self.export_content_sha256, "export content checksum is invalid")?; + validate_sha256(&self.control_id, "control ID is invalid")?; + validate_sha256(&self.admitted_topology_generation, "admitted topology generation is invalid")?; + validate_sha256(&self.admitted_member_epochs_sha256, "admitted member epoch digest is invalid")?; + validate_sha256(&self.actor_sha256, "actor digest is invalid")?; + if self.protocol != IlmRecoveryProtocol::TierDeleteJournal { + return Err(IlmRecoveryDispositionError::Corrupt( + "only legacy tier-delete journals support disposition", + )); + } + if self.control_etag.trim().is_empty() || self.control_revision == 0 { + return Err(IlmRecoveryDispositionError::Corrupt("control generation is invalid")); + } + validate_legacy_tier_delete_recovery_path(&self.canonical_source_path) + .map_err(|_| IlmRecoveryDispositionError::Corrupt("legacy source path is not canonical"))?; + self.source_generation + .validate() + .map_err(|_| IlmRecoveryDispositionError::Corrupt("source generation is invalid"))?; + if !matches!( + self.source_generation.source_schema.as_str(), + TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA | TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA + ) { + return Err(IlmRecoveryDispositionError::Corrupt( + "only legacy v1/v2 journal generations support disposition", + )); + } + if self.source_generation.copies.iter().any(|copy| { + copy.canonical_path != self.canonical_source_path + || copy.etag != self.source_generation.source_etag + || copy.content_sha256 != self.source_generation.content_sha256 + }) { + return Err(IlmRecoveryDispositionError::Corrupt( + "copy manifest does not describe one exact source generation", + )); + } + if recovery_disposition_id(&self.export_id, self.action)? != self.disposition_id { + return Err(IlmRecoveryDispositionError::Corrupt("disposition ID does not match export and action")); + } + if recovery_export_id(&self.control_id, &self.source_generation) + .map_err(|_| IlmRecoveryDispositionError::Corrupt("export identity is invalid"))? + != self.export_id + { + return Err(IlmRecoveryDispositionError::Corrupt( + "export ID does not match control and source generation", + )); + } + if self.confirmed_at_unix_nanos <= 0 { + return Err(IlmRecoveryDispositionError::Corrupt("confirmation timestamp is not positive")); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IlmRecoveryDispositionOwnerLease { + pub owner_id: String, + pub owner_epoch: Uuid, + pub lease_acquired_at_unix_nanos: i64, + pub lease_expires_at_unix_nanos: i64, + pub topology_generation: String, + pub member_epochs_sha256: String, +} + +impl IlmRecoveryDispositionOwnerLease { + fn validate(&self) -> Result<()> { + if self.owner_id.trim().is_empty() || self.owner_epoch.is_nil() { + return Err(IlmRecoveryDispositionError::Corrupt("owner fence is invalid")); + } + if self.lease_acquired_at_unix_nanos <= 0 || self.lease_expires_at_unix_nanos <= self.lease_acquired_at_unix_nanos { + return Err(IlmRecoveryDispositionError::Corrupt("owner lease interval is invalid")); + } + validate_sha256(&self.topology_generation, "owner topology generation is invalid")?; + validate_sha256(&self.member_epochs_sha256, "owner member epoch digest is invalid") + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IlmRecoveryDisposition { + pub identity: IlmRecoveryDispositionIdentity, + pub created_at_unix_nanos: i64, + pub retain_until_unix_nanos: i64, + pub revision: u64, + pub state: IlmRecoveryDispositionState, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner: Option, + pub confirmed_absent: Vec, +} + +impl IlmRecoveryDisposition { + pub fn new(identity: IlmRecoveryDispositionIdentity, created_at_unix_nanos: i64) -> Result { + let retain_until_unix_nanos = created_at_unix_nanos + .checked_add(DISPOSITION_RETENTION_NANOS) + .ok_or(IlmRecoveryDispositionError::Corrupt("retention timestamp overflow"))?; + let disposition = Self { + identity, + created_at_unix_nanos, + retain_until_unix_nanos, + revision: 1, + state: IlmRecoveryDispositionState::Prepared, + owner: None, + confirmed_absent: Vec::new(), + }; + disposition.validate()?; + Ok(disposition) + } + + pub fn validate(&self) -> Result<()> { + self.identity.validate()?; + if self.created_at_unix_nanos <= 0 + || self.created_at_unix_nanos < self.identity.confirmed_at_unix_nanos + || self.retain_until_unix_nanos < self.created_at_unix_nanos.saturating_add(DISPOSITION_RETENTION_NANOS) + { + return Err(IlmRecoveryDispositionError::Corrupt("disposition retention is invalid")); + } + if self.revision == 0 { + return Err(IlmRecoveryDispositionError::Corrupt("revision is zero")); + } + if let Some(owner) = &self.owner { + owner.validate()?; + } + if !is_strictly_sorted_unique(&self.confirmed_absent) { + return Err(IlmRecoveryDispositionError::Corrupt( + "confirmed-absent entries are not in unique canonical order", + )); + } + let manifest_authorities: Vec<&str> = self + .identity + .source_generation + .copies + .iter() + .map(|copy| copy.authority.as_str()) + .collect(); + if self + .confirmed_absent + .iter() + .any(|authority| manifest_authorities.binary_search(&authority.as_str()).is_err()) + { + return Err(IlmRecoveryDispositionError::Corrupt( + "confirmed-absent entry is not in the immutable copy manifest", + )); + } + match self.state { + IlmRecoveryDispositionState::Prepared if !self.confirmed_absent.is_empty() => { + Err(IlmRecoveryDispositionError::Corrupt("prepared disposition carries absence progress")) + } + IlmRecoveryDispositionState::Applying if self.owner.is_none() => { + Err(IlmRecoveryDispositionError::Corrupt("applying disposition has no owner lease")) + } + IlmRecoveryDispositionState::Completed + if self.owner.is_some() || self.confirmed_absent.len() != manifest_authorities.len() => + { + Err(IlmRecoveryDispositionError::Corrupt( + "completed disposition does not cover its entire manifest", + )) + } + _ => Ok(()), + } + } + + pub fn claim(&mut self, owner: IlmRecoveryDispositionOwnerLease) -> Result<()> { + if self.state == IlmRecoveryDispositionState::Completed || self.owner.is_some() { + return Err(IlmRecoveryDispositionError::InvalidSuccessor( + "only an unowned nonterminal disposition can be claimed", + )); + } + owner.validate()?; + self.bump_revision()?; + self.owner = Some(owner); + self.validate() + } + + pub fn take_over(&mut self, owner: IlmRecoveryDispositionOwnerLease) -> Result<()> { + let current = self + .owner + .as_ref() + .ok_or(IlmRecoveryDispositionError::InvalidSuccessor("takeover requires an existing owner"))?; + if self.state == IlmRecoveryDispositionState::Completed + || owner.owner_epoch == current.owner_epoch + || owner.lease_acquired_at_unix_nanos < current.lease_expires_at_unix_nanos + { + return Err(IlmRecoveryDispositionError::InvalidSuccessor("takeover did not fence an expired owner")); + } + owner.validate()?; + self.bump_revision()?; + self.owner = Some(owner); + self.validate() + } + + pub fn begin_applying(&mut self) -> Result<()> { + if self.state != IlmRecoveryDispositionState::Prepared || self.owner.is_none() { + return Err(IlmRecoveryDispositionError::InvalidSuccessor( + "applying requires an owned prepared disposition", + )); + } + self.bump_revision()?; + self.state = IlmRecoveryDispositionState::Applying; + self.validate() + } + + pub fn confirm_absent(&mut self, authority: impl Into) -> Result<()> { + if self.state != IlmRecoveryDispositionState::Applying || self.owner.is_none() { + return Err(IlmRecoveryDispositionError::InvalidSuccessor( + "absence progress requires an owned applying disposition", + )); + } + let authority = authority.into(); + if self + .identity + .source_generation + .copies + .binary_search_by(|copy| copy.authority.cmp(&authority)) + .is_err() + { + return Err(IlmRecoveryDispositionError::InvalidSuccessor( + "absence progress does not name a manifest entry", + )); + } + match self.confirmed_absent.binary_search(&authority) { + Ok(_) => return Err(IlmRecoveryDispositionError::InvalidSuccessor("absence progress is already recorded")), + Err(index) => self.confirmed_absent.insert(index, authority), + } + self.bump_revision()?; + self.validate() + } + + pub fn complete(&mut self) -> Result<()> { + if self.state != IlmRecoveryDispositionState::Applying + || self.confirmed_absent.len() != self.identity.source_generation.copies.len() + { + return Err(IlmRecoveryDispositionError::InvalidSuccessor( + "completion requires absence proof for the entire manifest", + )); + } + self.bump_revision()?; + self.state = IlmRecoveryDispositionState::Completed; + self.owner = None; + self.validate() + } + + pub fn validate_successor(&self, next: &Self) -> Result<()> { + self.validate()?; + next.validate()?; + if self.identity != next.identity + || self.created_at_unix_nanos != next.created_at_unix_nanos + || self.retain_until_unix_nanos != next.retain_until_unix_nanos + { + return Err(IlmRecoveryDispositionError::InvalidSuccessor("immutable identity changed")); + } + if self.revision.checked_add(1) != Some(next.revision) { + return Err(IlmRecoveryDispositionError::InvalidSuccessor("revision did not advance by one")); + } + if !is_sorted_subset(&self.confirmed_absent, &next.confirmed_absent) { + return Err(IlmRecoveryDispositionError::InvalidSuccessor("confirmed-absent progress regressed")); + } + + match (self.state, next.state) { + (IlmRecoveryDispositionState::Prepared, IlmRecoveryDispositionState::Prepared) + | (IlmRecoveryDispositionState::Applying, IlmRecoveryDispositionState::Applying) => { + self.validate_same_state_successor(next) + } + (IlmRecoveryDispositionState::Prepared, IlmRecoveryDispositionState::Applying) + if self.owner.is_some() && self.owner == next.owner && self.confirmed_absent == next.confirmed_absent => + { + Ok(()) + } + (IlmRecoveryDispositionState::Applying, IlmRecoveryDispositionState::Completed) + if self.owner.is_some() + && next.owner.is_none() + && self.confirmed_absent == next.confirmed_absent + && self.confirmed_absent.len() == self.identity.source_generation.copies.len() => + { + Ok(()) + } + _ => Err(IlmRecoveryDispositionError::InvalidSuccessor("state transition is not allowed")), + } + } + + fn is_same_or_later_generation_of(&self, previous: &Self) -> bool { + if self.identity != previous.identity + || self.created_at_unix_nanos != previous.created_at_unix_nanos + || self.retain_until_unix_nanos != previous.retain_until_unix_nanos + || self.revision < previous.revision + || !is_sorted_subset(&previous.confirmed_absent, &self.confirmed_absent) + { + return false; + } + if self.revision == previous.revision { + return self == previous; + } + let Some(revision_distance) = self.revision.checked_sub(previous.revision) else { + return false; + }; + let previous_progress = previous.confirmed_absent.len(); + let current_progress = self.confirmed_absent.len(); + let remaining = self.identity.source_generation.copies.len().saturating_sub(previous_progress); + let owner_change_is_fenced = || match (&previous.owner, &self.owner) { + (Some(previous_owner), Some(current_owner)) if previous_owner != current_owner => { + current_owner.lease_acquired_at_unix_nanos >= previous_owner.lease_expires_at_unix_nanos + } + _ => true, + }; + let minimum_distance = match (previous.state, self.state) { + (IlmRecoveryDispositionState::Prepared, IlmRecoveryDispositionState::Prepared) => { + if self.owner.is_some() + && (previous.owner.is_none() || (previous.owner != self.owner && owner_change_is_fenced())) + { + 1 + } else { + return false; + } + } + (IlmRecoveryDispositionState::Prepared, IlmRecoveryDispositionState::Applying) => { + let owner_steps = if previous.owner.is_none() { + 2 + } else if previous.owner == self.owner { + 1 + } else if self.owner.is_some() && owner_change_is_fenced() { + 2 + } else { + return false; + }; + owner_steps + current_progress as u64 + } + (IlmRecoveryDispositionState::Prepared, IlmRecoveryDispositionState::Completed) => { + u64::from(previous.owner.is_none()) + 2 + self.identity.source_generation.copies.len() as u64 + } + (IlmRecoveryDispositionState::Applying, IlmRecoveryDispositionState::Applying) => { + let progress_steps = current_progress.saturating_sub(previous_progress) as u64; + if previous.owner == self.owner && progress_steps > 0 { + progress_steps + } else if previous.owner != self.owner && self.owner.is_some() && owner_change_is_fenced() { + 1 + progress_steps + } else { + return false; + } + } + (IlmRecoveryDispositionState::Applying, IlmRecoveryDispositionState::Completed) => remaining as u64 + 1, + _ => return false, + }; + revision_distance >= minimum_distance + } + + fn validate_same_state_successor(&self, next: &Self) -> Result<()> { + match (&self.owner, &next.owner) { + (None, Some(_)) if self.confirmed_absent == next.confirmed_absent => Ok(()), + (Some(current), Some(candidate)) if current == candidate => { + if self.state == IlmRecoveryDispositionState::Prepared && self.confirmed_absent != next.confirmed_absent { + Err(IlmRecoveryDispositionError::InvalidSuccessor( + "prepared disposition advanced absence progress", + )) + } else if self.state == IlmRecoveryDispositionState::Applying + && self.confirmed_absent.len().checked_add(1) == Some(next.confirmed_absent.len()) + { + Ok(()) + } else { + Err(IlmRecoveryDispositionError::InvalidSuccessor( + "absence progress did not advance by one copy", + )) + } + } + (Some(current), Some(candidate)) + if self.confirmed_absent == next.confirmed_absent + && candidate.owner_epoch != current.owner_epoch + && candidate.lease_acquired_at_unix_nanos >= current.lease_expires_at_unix_nanos => + { + Ok(()) + } + _ => Err(IlmRecoveryDispositionError::InvalidSuccessor( + "same-state successor changed owner and progress together", + )), + } + } + + pub fn encode(&self) -> Result> { + self.validate()?; + let disposition_bytes = serde_json::to_vec(self)?; + let persisted = PersistedIlmRecoveryDisposition { + schema: ILM_RECOVERY_DISPOSITION_SCHEMA.to_string(), + content_sha256: hex_sha256(&disposition_bytes, ToOwned::to_owned), + disposition: self.clone(), + }; + let encoded = serde_json::to_vec(&persisted)?; + if encoded.len() > MAX_ILM_RECOVERY_DISPOSITION_SIZE { + return Err(IlmRecoveryDispositionError::Corrupt("encoded disposition exceeds maximum size")); + } + Ok(encoded) + } + + pub fn decode(expected_disposition_id: &str, data: &[u8]) -> Result { + validate_sha256(expected_disposition_id, "disposition ID is invalid")?; + if data.len() > MAX_ILM_RECOVERY_DISPOSITION_SIZE { + return Err(IlmRecoveryDispositionError::Corrupt("encoded disposition exceeds maximum size")); + } + let persisted: PersistedIlmRecoveryDisposition = serde_json::from_slice(data)?; + if persisted.schema != ILM_RECOVERY_DISPOSITION_SCHEMA { + return Err(IlmRecoveryDispositionError::UnsupportedSchema(persisted.schema)); + } + validate_sha256(&persisted.content_sha256, "content checksum is invalid")?; + let disposition_bytes = serde_json::to_vec(&persisted.disposition)?; + if hex_sha256(&disposition_bytes, ToOwned::to_owned) != persisted.content_sha256 { + return Err(IlmRecoveryDispositionError::ChecksumMismatch); + } + persisted.disposition.validate()?; + if persisted.disposition.identity.disposition_id != expected_disposition_id { + return Err(IlmRecoveryDispositionError::Corrupt("disposition ID does not match record key")); + } + Ok(persisted.disposition) + } + + fn bump_revision(&mut self) -> Result<()> { + self.revision = self + .revision + .checked_add(1) + .ok_or(IlmRecoveryDispositionError::Corrupt("revision overflow"))?; + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedIlmRecoveryDisposition { + schema: String, + content_sha256: String, + disposition: IlmRecoveryDisposition, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObservedIlmRecoveryDisposition { + pub disposition: IlmRecoveryDisposition, + pub etag: String, + pub content_sha256: String, + pub encoded: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreatedIlmRecoveryDisposition { + pub observed: ObservedIlmRecoveryDisposition, + pub replayed: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DecodedIlmRecoveryDispositionCheckpoint { + pub(crate) disposition_id: String, + pub(crate) content_sha256: String, + pub(crate) identity_sha256: String, + pub(crate) copy_manifest_sha256: String, + pub(crate) copy_manifest_count: usize, + pub(crate) created_at_unix_nanos: i64, + pub(crate) revision: u64, + pub(crate) state: IlmRecoveryDispositionState, + pub(crate) owner_fence_sha256: Option, + pub(crate) owner_lease_acquired_at_unix_nanos: Option, + pub(crate) owner_lease_expires_at_unix_nanos: Option, + pub(crate) confirmed_absent_sha256: Vec, + pub(crate) retain_until_unix_nanos: i64, +} + +pub fn recovery_disposition_id(export_id: &str, action: IlmRecoveryDispositionAction) -> Result { + validate_sha256(export_id, "export ID is invalid")?; + Ok(length_delimited_digest(&[export_id.as_bytes(), action.as_str().as_bytes()])) +} + +pub fn recovery_disposition_record_object_name(protocol: IlmRecoveryProtocol, disposition_id: &str) -> Result { + validate_sha256(disposition_id, "disposition ID is invalid")?; + if protocol != IlmRecoveryProtocol::TierDeleteJournal { + return Err(IlmRecoveryDispositionError::Corrupt("disposition protocol is unsupported")); + } + Ok(format!( + "{}/{}/{}/{}/{}.json", + ILM_RECOVERY_DISPOSITION_PREFIX, + protocol.as_str(), + &disposition_id[..2], + &disposition_id[2..4], + disposition_id + )) +} + +pub fn recovery_disposition_id_from_record_object_name(object: &str) -> Result<(IlmRecoveryProtocol, String)> { + let suffix = object + .strip_prefix(ILM_RECOVERY_DISPOSITION_PREFIX) + .and_then(|suffix| suffix.strip_prefix('/')) + .ok_or(IlmRecoveryDispositionError::Corrupt("disposition record path has wrong prefix"))?; + let mut parts = suffix.split('/'); + let protocol = match parts.next() { + Some("tier_delete_journal") => IlmRecoveryProtocol::TierDeleteJournal, + _ => { + return Err(IlmRecoveryDispositionError::Corrupt("disposition record protocol is invalid")); + } + }; + let shard_a = parts + .next() + .ok_or(IlmRecoveryDispositionError::Corrupt("disposition record path is incomplete"))?; + let shard_b = parts + .next() + .ok_or(IlmRecoveryDispositionError::Corrupt("disposition record path is incomplete"))?; + let disposition_id = parts + .next() + .and_then(|name| name.strip_suffix(".json")) + .ok_or(IlmRecoveryDispositionError::Corrupt("disposition record suffix is invalid"))?; + if parts.next().is_some() { + return Err(IlmRecoveryDispositionError::Corrupt("disposition record path is not canonical")); + } + validate_sha256(disposition_id, "disposition ID is invalid")?; + if shard_a != &disposition_id[..2] || shard_b != &disposition_id[2..4] { + return Err(IlmRecoveryDispositionError::Corrupt( + "disposition record shard does not match disposition ID", + )); + } + Ok((protocol, disposition_id.to_string())) +} + +pub(crate) fn decode_recovery_disposition_checkpoint( + path: &str, + data: &[u8], +) -> EcstoreResult { + let (protocol, disposition_id) = recovery_disposition_id_from_record_object_name(path).map_err(disposition_store_error)?; + let disposition = IlmRecoveryDisposition::decode(&disposition_id, data).map_err(disposition_store_error)?; + let canonical = recovery_disposition_record_object_name(protocol, &disposition_id).map_err(disposition_store_error)?; + if canonical != path || disposition.identity.protocol != protocol { + return Err(Error::other("ILM recovery disposition path is not canonical")); + } + let identity_sha256 = checkpoint_hash(&disposition.identity)?; + let owner_fence_sha256 = disposition + .owner + .as_ref() + .map(|owner| checkpoint_domain_hash(OWNER_FENCE_CHECKPOINT_DOMAIN, owner)) + .transpose()?; + let mut confirmed_absent_sha256 = disposition + .confirmed_absent + .iter() + .map(|authority| hex_sha256(authority.as_bytes(), ToOwned::to_owned)) + .collect::>(); + confirmed_absent_sha256.sort_unstable(); + Ok(DecodedIlmRecoveryDispositionCheckpoint { + disposition_id, + content_sha256: hex_sha256(data, ToOwned::to_owned), + identity_sha256, + copy_manifest_sha256: disposition.identity.source_generation.copy_set_sha256.clone(), + copy_manifest_count: disposition.identity.source_generation.copies.len(), + created_at_unix_nanos: disposition.created_at_unix_nanos, + revision: disposition.revision, + state: disposition.state, + owner_fence_sha256, + owner_lease_acquired_at_unix_nanos: disposition.owner.as_ref().map(|owner| owner.lease_acquired_at_unix_nanos), + owner_lease_expires_at_unix_nanos: disposition.owner.as_ref().map(|owner| owner.lease_expires_at_unix_nanos), + confirmed_absent_sha256, + retain_until_unix_nanos: disposition.retain_until_unix_nanos, + }) +} + +pub async fn create_recovery_disposition_if_absent( + api: Arc, + disposition: &IlmRecoveryDisposition, +) -> EcstoreResult { + if disposition.revision != 1 + || disposition.state != IlmRecoveryDispositionState::Prepared + || disposition.owner.is_some() + || !disposition.confirmed_absent.is_empty() + { + return Err(Error::PreconditionFailed); + } + let object = recovery_disposition_record_object_name(disposition.identity.protocol, &disposition.identity.disposition_id) + .map_err(disposition_store_error)?; + match load_recovery_disposition(api.clone(), disposition.identity.protocol, &disposition.identity.disposition_id).await { + Ok(observed) if observed.disposition.is_same_or_later_generation_of(disposition) => { + record_disposition_decommission_checkpoint(api.as_ref(), &object, &observed).await?; + return Ok(CreatedIlmRecoveryDisposition { + observed, + replayed: true, + }); + } + Ok(_) => return Err(Error::PreconditionFailed), + Err(err) if disposition_is_missing(&err) => {} + Err(err) => return Err(err), + } + let control_object = recovery_control_record_object_name(disposition.identity.protocol, &disposition.identity.control_id) + .map_err(|err| Error::other(err.to_string()))?; + let control_lock = api.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?; + let control_guard = control_lock + .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let source_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &disposition.identity.canonical_source_path) + .await?; + let source_guard = source_lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?; + let locks_current = || !control_guard.is_lock_lost() && !source_guard.is_lock_lost(); + validate_disposition_sources(api.clone(), disposition).await?; + let current_source = observe_recovery_source( + api.clone(), + &disposition.identity.canonical_source_path, + &disposition.identity.source_generation.source_schema, + ) + .await?; + if current_source.canonical_data.is_none() + || current_source.generation != disposition.identity.source_generation + || !locks_current() + { + return Err(Error::PreconditionFailed); + } + let encoded = disposition.encode().map_err(disposition_store_error)?; + let mut write_options = ObjectOptions { + max_parity: true, + write_completion: WriteCompletion::TailDrained, + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }; + write_options.add_namespace_lock_guard(&control_guard); + write_options.add_namespace_lock_guard(&source_guard); + if !locks_current() { + return Err(Error::PreconditionFailed); + } + let write_result = config_boundary::save_config_with_opts(api.clone(), &object, encoded.clone(), &write_options).await; + match load_recovery_disposition(api.clone(), disposition.identity.protocol, &disposition.identity.disposition_id).await { + Ok(observed) if observed.disposition.is_same_or_later_generation_of(disposition) => { + record_disposition_decommission_checkpoint(api.as_ref(), &object, &observed).await?; + if !locks_current() { + return Err(Error::PreconditionFailed); + } + let replayed = write_result.is_err() || observed.encoded != encoded; + Ok(CreatedIlmRecoveryDisposition { observed, replayed }) + } + Ok(_) => Err(Error::PreconditionFailed), + Err(read_err) => Err(write_result.err().unwrap_or(read_err)), + } +} + +async fn validate_disposition_sources(api: Arc, disposition: &IlmRecoveryDisposition) -> EcstoreResult<()> { + let stored_export = load_recovery_export(api.clone(), &disposition.identity.export_id).await?; + if stored_export.content_sha256 != disposition.identity.export_content_sha256 { + return Err(Error::PreconditionFailed); + } + let export = IlmRecoveryExport::decode(&stored_export.export_id, &stored_export.encoded)?; + let observed_control = load_recovery_control(api, disposition.identity.protocol, &disposition.identity.control_id).await?; + if export.control_id != disposition.identity.control_id + || export.protocol != disposition.identity.protocol + || export.control_etag != disposition.identity.control_etag + || export.control_revision != disposition.identity.control_revision + || export.classification != IlmRecoveryClassification::RetainedAmbiguous + || export.topology_generation != disposition.identity.admitted_topology_generation + || export.member_epochs_sha256 != disposition.identity.admitted_member_epochs_sha256 + || export.canonical_source_path != disposition.identity.canonical_source_path + || export.source_generation != disposition.identity.source_generation + || observed_control.etag != disposition.identity.control_etag + || observed_control.control.revision != disposition.identity.control_revision + || observed_control.control.classification != IlmRecoveryClassification::RetainedAmbiguous + || observed_control.control.identity.canonical_source_path != disposition.identity.canonical_source_path + || observed_control.control.observed_source_generation != disposition.identity.source_generation + { + return Err(Error::PreconditionFailed); + } + Ok(()) +} + +pub async fn load_recovery_disposition( + api: Arc, + protocol: IlmRecoveryProtocol, + disposition_id: &str, +) -> EcstoreResult { + let object = recovery_disposition_record_object_name(protocol, disposition_id).map_err(disposition_store_error)?; + let (encoded, metadata) = config_boundary::read_config_limited_preserve_empty_with_metadata( + api, + &object, + &ObjectOptions::default(), + MAX_ILM_RECOVERY_DISPOSITION_SIZE, + ) + .await?; + let etag = metadata + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or_else(|| Error::other("ILM recovery disposition is missing an ETag"))?; + let disposition = IlmRecoveryDisposition::decode(disposition_id, &encoded).map_err(disposition_store_error)?; + if disposition.identity.protocol != protocol { + return Err(Error::other("ILM recovery disposition protocol does not match record path")); + } + Ok(ObservedIlmRecoveryDisposition { + disposition, + etag, + content_sha256: hex_sha256(&encoded, ToOwned::to_owned), + encoded, + }) +} + +pub async fn save_recovery_disposition_if_current( + api: Arc, + current: &ObservedIlmRecoveryDisposition, + next: &IlmRecoveryDisposition, +) -> EcstoreResult { + current + .disposition + .validate_successor(next) + .map_err(disposition_store_error)?; + let protocol = current.disposition.identity.protocol; + let disposition_id = ¤t.disposition.identity.disposition_id; + let object = recovery_disposition_record_object_name(protocol, disposition_id).map_err(disposition_store_error)?; + let authoritative = load_recovery_disposition(api.clone(), protocol, disposition_id).await?; + if authoritative != *current { + if authoritative.disposition.is_same_or_later_generation_of(next) { + record_disposition_decommission_checkpoint(api.as_ref(), &object, &authoritative).await?; + return Ok(authoritative); + } + return Err(Error::PreconditionFailed); + } + let encoded = next.encode().map_err(disposition_store_error)?; + let write_result = config_boundary::save_config_with_opts( + api.clone(), + &object, + encoded.clone(), + &ObjectOptions { + max_parity: true, + write_completion: WriteCompletion::TailDrained, + http_preconditions: Some(HTTPPreconditions { + if_match: Some(current.etag.clone()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await; + match load_recovery_disposition(api.clone(), protocol, disposition_id).await { + Ok(observed) if observed.disposition.is_same_or_later_generation_of(next) => { + record_disposition_decommission_checkpoint(api.as_ref(), &object, &observed).await?; + Ok(observed) + } + Ok(_) => Err(Error::PreconditionFailed), + Err(read_err) => Err(write_result.err().unwrap_or(read_err)), + } +} + +async fn record_disposition_decommission_checkpoint( + api: &ECStore, + object: &str, + observed: &ObservedIlmRecoveryDisposition, +) -> EcstoreResult<()> { + if observed.disposition.state == IlmRecoveryDispositionState::Completed { + api.record_durable_ilm_decommission_terminal(object, &observed.encoded).await + } else { + api.record_durable_ilm_decommission_progress(object, &observed.encoded).await + } +} + +fn validate_sha256(value: &str, message: &'static str) -> Result<()> { + if !is_sha256_checksum(value) + || value + .bytes() + .any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase()) + { + return Err(IlmRecoveryDispositionError::Corrupt(message)); + } + Ok(()) +} + +fn is_strictly_sorted_unique(values: &[String]) -> bool { + values.windows(2).all(|pair| pair[0] < pair[1]) +} + +fn is_sorted_subset(previous: &[String], next: &[String]) -> bool { + previous.iter().all(|value| next.binary_search(value).is_ok()) +} + +fn length_delimited_digest(parts: &[&[u8]]) -> String { + let mut encoded = Vec::new(); + for part in parts { + encoded.extend_from_slice(&(part.len() as u64).to_be_bytes()); + encoded.extend_from_slice(part); + } + hex_sha256(&encoded, ToOwned::to_owned) +} + +fn checkpoint_hash(value: &T) -> EcstoreResult { + let encoded = serde_json::to_vec(value).map_err(Error::other)?; + Ok(hex_sha256(&encoded, ToOwned::to_owned)) +} + +fn checkpoint_domain_hash(domain: &[u8], value: &T) -> EcstoreResult { + let encoded = serde_json::to_vec(value).map_err(Error::other)?; + Ok(length_delimited_digest(&[domain, &encoded])) +} + +fn disposition_store_error(err: IlmRecoveryDispositionError) -> Error { + Error::other(err) +} + +fn disposition_is_missing(err: &Error) -> bool { + matches!( + err, + Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::VersionNotFound(_, _, _) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bucket::lifecycle::recovery_control::IlmRecoverySourceCopy; + + fn digest(value: &[u8]) -> String { + hex_sha256(value, ToOwned::to_owned) + } + + fn sample_disposition() -> IlmRecoveryDisposition { + let source_path = format!("ilm/tier-delete-journal/{}.json", digest(b"journal-identity")); + let content_sha256 = digest(b"legacy-source"); + let source_generation = IlmRecoverySourceGeneration::new( + "rustfs-tier-delete-journal-v2", + "source-etag", + content_sha256.clone(), + vec![ + IlmRecoverySourceCopy { + authority: "pool-1/set-0".to_string(), + canonical_path: source_path.clone(), + etag: "source-etag".to_string(), + encoded_len: 13, + content_sha256: content_sha256.clone(), + }, + IlmRecoverySourceCopy { + authority: "pool-0/set-0".to_string(), + canonical_path: source_path.clone(), + etag: "source-etag".to_string(), + encoded_len: 13, + content_sha256, + }, + ], + ) + .expect("source generation should be valid"); + let control_id = digest(b"control"); + let export_id = recovery_export_id(&control_id, &source_generation).expect("export ID should be valid"); + let action = IlmRecoveryDispositionAction::AbandonRemoteCleanup; + let identity = IlmRecoveryDispositionIdentity { + disposition_id: recovery_disposition_id(&export_id, action).expect("ID should be valid"), + protocol: IlmRecoveryProtocol::TierDeleteJournal, + action, + export_id, + export_content_sha256: digest(b"export-envelope"), + control_id, + control_etag: "control-etag".to_string(), + control_revision: 1, + canonical_source_path: source_path, + source_generation, + admitted_topology_generation: digest(b"topology"), + admitted_member_epochs_sha256: digest(b"member-epochs"), + actor_sha256: digest(b"actor"), + reason_code: IlmRecoveryDispositionReasonCode::LegacyRemoteCleanupAbandoned, + confirmed_at_unix_nanos: 1_000_000_000, + }; + IlmRecoveryDisposition::new(identity, 1_000_000_001).expect("disposition should be valid") + } + + fn owner(epoch: u128, acquired: i64) -> IlmRecoveryDispositionOwnerLease { + IlmRecoveryDispositionOwnerLease { + owner_id: "node-a".to_string(), + owner_epoch: Uuid::from_u128(epoch), + lease_acquired_at_unix_nanos: acquired, + lease_expires_at_unix_nanos: acquired + 1_000, + topology_generation: digest(b"topology-now"), + member_epochs_sha256: digest(b"member-epochs-now"), + } + } + + #[test] + fn disposition_id_and_path_are_canonical_and_deterministic() { + let disposition = sample_disposition(); + let id = &disposition.identity.disposition_id; + assert_eq!( + recovery_disposition_id(&disposition.identity.export_id, IlmRecoveryDispositionAction::AbandonRemoteCleanup) + .expect("ID should be valid"), + *id + ); + let path = + recovery_disposition_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, id).expect("path should be valid"); + assert_eq!( + recovery_disposition_id_from_record_object_name(&path).expect("path should parse"), + (IlmRecoveryProtocol::TierDeleteJournal, id.clone()) + ); + let wrong_shard = path.replacen(&format!("/{}/", &id[..2]), "/ff/", 1); + assert!(recovery_disposition_id_from_record_object_name(&wrong_shard).is_err()); + } + + #[test] + fn checksum_envelope_round_trips_and_rejects_tampering() { + let disposition = sample_disposition(); + let encoded = disposition.encode().expect("disposition should encode"); + assert_eq!( + IlmRecoveryDisposition::decode(&disposition.identity.disposition_id, &encoded).expect("disposition should decode"), + disposition + ); + + let mut envelope: serde_json::Value = serde_json::from_slice(&encoded).expect("fixture should be json"); + envelope["disposition"]["identity"]["control_etag"] = serde_json::Value::String("tampered".to_string()); + let tampered = serde_json::to_vec(&envelope).expect("fixture should encode"); + assert!(matches!( + IlmRecoveryDisposition::decode(&disposition.identity.disposition_id, &tampered), + Err(IlmRecoveryDispositionError::ChecksumMismatch) + )); + + envelope["unknown"] = serde_json::Value::Bool(true); + let unknown = serde_json::to_vec(&envelope).expect("fixture should encode"); + assert!(matches!( + IlmRecoveryDisposition::decode(&disposition.identity.disposition_id, &unknown), + Err(IlmRecoveryDispositionError::Json(_)) + )); + + for source_path in [ + "ilm/tier-delete-journal/legacy.json", + "ilm/tier-delete-journal/../legacy.json", + ] { + let mut invalid = disposition.clone(); + invalid.identity.canonical_source_path = source_path.to_string(); + for copy in &mut invalid.identity.source_generation.copies { + copy.canonical_path = source_path.to_string(); + } + invalid.identity.source_generation = IlmRecoverySourceGeneration::new( + invalid.identity.source_generation.source_schema.clone(), + invalid.identity.source_generation.source_etag.clone(), + invalid.identity.source_generation.content_sha256.clone(), + invalid.identity.source_generation.copies.clone(), + ) + .expect("copy manifest should be internally consistent"); + assert!(invalid.validate().is_err(), "noncanonical source path must fail closed"); + } + + let mut rebound = disposition; + rebound.identity.control_id = digest(b"different-control"); + assert!(rebound.validate().is_err(), "export ID must bind the exact control and source generation"); + } + + #[test] + fn state_and_absence_progress_successors_are_strict() { + let prepared = sample_disposition(); + let mut claimed = prepared.clone(); + claimed.claim(owner(1, 2_000_000_000)).expect("claim should succeed"); + prepared.validate_successor(&claimed).expect("claim should be a successor"); + + let mut applying = claimed.clone(); + applying.begin_applying().expect("begin applying should succeed"); + claimed.validate_successor(&applying).expect("applying should follow a claim"); + + let mut one_absent = applying.clone(); + one_absent + .confirm_absent("pool-0/set-0") + .expect("manifest entry should be recorded"); + applying + .validate_successor(&one_absent) + .expect("absence should advance monotonically"); + + let mut batched_absence = applying.clone(); + batched_absence.revision += 1; + batched_absence.confirmed_absent = vec!["pool-0/set-0".to_string(), "pool-1/set-0".to_string()]; + assert!( + applying.validate_successor(&batched_absence).is_err(), + "each durable progress revision must confirm exactly one copy" + ); + + let mut skipped_progress = one_absent.clone(); + skipped_progress.revision += 1; + skipped_progress.state = IlmRecoveryDispositionState::Completed; + skipped_progress.owner = None; + skipped_progress.confirmed_absent.push("pool-1/set-0".to_string()); + assert!(one_absent.validate_successor(&skipped_progress).is_err()); + + let mut all_absent = one_absent.clone(); + all_absent + .confirm_absent("pool-1/set-0") + .expect("manifest entry should be recorded"); + let mut completed = all_absent.clone(); + completed.complete().expect("full absence should complete"); + all_absent + .validate_successor(&completed) + .expect("completed should follow full absence"); + assert!(completed.validate_successor(&completed).is_err()); + + let mut regressed = all_absent.clone(); + regressed.revision += 1; + regressed.confirmed_absent.remove(0); + assert!(one_absent.validate_successor(®ressed).is_err()); + } + + #[test] + fn advanced_replay_requires_a_reachable_generation() { + let prepared = sample_disposition(); + + let mut skipped_to_applying = prepared.clone(); + skipped_to_applying.revision = 2; + skipped_to_applying.state = IlmRecoveryDispositionState::Applying; + skipped_to_applying.owner = Some(owner(1, 2_000_000_000)); + skipped_to_applying.confirmed_absent.push("pool-0/set-0".to_string()); + assert!(skipped_to_applying.validate().is_ok()); + assert!(!skipped_to_applying.is_same_or_later_generation_of(&prepared)); + + let mut skipped_to_completed = prepared.clone(); + skipped_to_completed.revision = 2; + skipped_to_completed.state = IlmRecoveryDispositionState::Completed; + skipped_to_completed.confirmed_absent = vec!["pool-0/set-0".to_string(), "pool-1/set-0".to_string()]; + assert!(skipped_to_completed.validate().is_ok()); + assert!(!skipped_to_completed.is_same_or_later_generation_of(&prepared)); + + let mut claimed = prepared.clone(); + claimed.claim(owner(1, 2_000_000_000)).unwrap(); + let mut applying = claimed.clone(); + applying.begin_applying().unwrap(); + applying.confirm_absent("pool-0/set-0").unwrap(); + applying.confirm_absent("pool-1/set-0").unwrap(); + let mut completed = applying.clone(); + completed.complete().unwrap(); + assert!(completed.is_same_or_later_generation_of(&prepared)); + + let mut same_revision_conflict = prepared.clone(); + same_revision_conflict.identity.actor_sha256 = digest(b"different-actor"); + assert!(!same_revision_conflict.is_same_or_later_generation_of(&prepared)); + } + + #[test] + fn owner_takeover_requires_expiry_and_cannot_advance_progress() { + let mut applying = sample_disposition(); + applying.claim(owner(1, 2_000_000_000)).expect("claim should succeed"); + applying.begin_applying().expect("begin applying should succeed"); + + let mut early = applying.clone(); + assert!(early.take_over(owner(2, 2_000_000_999)).is_err()); + + let mut taken_over = applying.clone(); + taken_over + .take_over(owner(2, 2_000_001_000)) + .expect("expired owner should be replaceable"); + applying + .validate_successor(&taken_over) + .expect("takeover should be a valid successor"); + + let mut changed_both = taken_over.clone(); + changed_both.revision += 1; + changed_both.owner = Some(owner(3, 2_000_002_000)); + changed_both.confirmed_absent.push("pool-0/set-0".to_string()); + assert!(taken_over.validate_successor(&changed_both).is_err()); + } + + #[test] + fn manifest_and_confirmed_absent_must_remain_canonical() { + let mut disposition = sample_disposition(); + disposition.identity.source_generation.copies.swap(0, 1); + assert!(disposition.validate().is_err()); + + let mut disposition = sample_disposition(); + disposition.state = IlmRecoveryDispositionState::Applying; + disposition.owner = Some(owner(1, 2_000_000_000)); + disposition.confirmed_absent = vec!["pool-9/set-9".to_string()]; + assert!(disposition.validate().is_err()); + + let mut disposition = sample_disposition(); + disposition.state = IlmRecoveryDispositionState::Completed; + disposition.confirmed_absent = vec!["pool-0/set-0".to_string()]; + assert!(disposition.validate().is_err()); + } + + #[test] + fn encoded_disposition_is_bounded_to_sixteen_kibibytes() { + let mut disposition = sample_disposition(); + disposition.identity.control_etag = "x".repeat(MAX_ILM_RECOVERY_DISPOSITION_SIZE); + assert!(matches!( + disposition.encode(), + Err(IlmRecoveryDispositionError::Corrupt("encoded disposition exceeds maximum size")) + )); + assert!(matches!( + IlmRecoveryDisposition::decode( + &sample_disposition().identity.disposition_id, + &vec![b'x'; MAX_ILM_RECOVERY_DISPOSITION_SIZE + 1] + ), + Err(IlmRecoveryDispositionError::Corrupt("encoded disposition exceeds maximum size")) + )); + } + + #[test] + fn durable_checkpoint_hashes_identity_owner_manifest_and_absence_set() { + let prepared = sample_disposition(); + let path = + recovery_disposition_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, &prepared.identity.disposition_id) + .expect("path should be valid"); + let prepared_encoded = prepared.encode().expect("prepared disposition should encode"); + let prepared_checkpoint = + decode_recovery_disposition_checkpoint(&path, &prepared_encoded).expect("prepared checkpoint should decode"); + assert_eq!(prepared_checkpoint.state, IlmRecoveryDispositionState::Prepared); + assert_eq!(prepared_checkpoint.owner_fence_sha256, None); + + let mut disposition = prepared; + disposition.claim(owner(1, 2_000_000_000)).expect("claim should succeed"); + disposition.begin_applying().expect("begin applying should succeed"); + disposition + .confirm_absent("pool-0/set-0") + .expect("absence should be recorded"); + let encoded = disposition.encode().expect("disposition should encode"); + let checkpoint = decode_recovery_disposition_checkpoint(&path, &encoded).expect("checkpoint should decode"); + assert_eq!(checkpoint.disposition_id, disposition.identity.disposition_id); + assert_eq!(checkpoint.content_sha256, digest(&encoded)); + assert_eq!(checkpoint.copy_manifest_sha256, disposition.identity.source_generation.copy_set_sha256); + assert_eq!(checkpoint.copy_manifest_count, 2); + assert_eq!(checkpoint.revision, disposition.revision); + assert_eq!(checkpoint.state, IlmRecoveryDispositionState::Applying); + assert_eq!( + checkpoint.owner_fence_sha256, + Some( + checkpoint_domain_hash( + OWNER_FENCE_CHECKPOINT_DOMAIN, + disposition.owner.as_ref().expect("applying disposition should have an owner") + ) + .expect("owner fence should hash") + ) + ); + assert_eq!(checkpoint.confirmed_absent_sha256, vec![digest(b"pool-0/set-0")]); + assert_eq!(checkpoint.retain_until_unix_nanos, disposition.retain_until_unix_nanos); + } +} diff --git a/crates/ecstore/src/bucket/lifecycle/recovery_export.rs b/crates/ecstore/src/bucket/lifecycle/recovery_export.rs new file mode 100644 index 000000000..b408f5580 --- /dev/null +++ b/crates/ecstore/src/bucket/lifecycle/recovery_export.rs @@ -0,0 +1,840 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{collections::HashSet, sync::Arc}; + +use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum}; +use serde::{Deserialize, Serialize}; + +use super::config_boundary; +use super::recovery_control::{ + IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryProtocol, IlmRecoverySourceCopy, IlmRecoverySourceGeneration, + MAX_ILM_RECOVERY_CONTROL_SIZE, ObservedIlmRecoveryControl, ObservedIlmRecoverySource, recovery_control_record_object_name, +}; +use super::tier_delete_journal::{ + TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, validate_legacy_tier_delete_recovery_source, +}; +use crate::disk::RUSTFS_META_BUCKET; +use crate::error::{Error, Result}; +use crate::object_api::{ObjectOptions, WriteCompletion}; +use crate::services::notification_sys::{ + acquire_ilm_recovery_export_fleet_proof, ilm_recovery_export_fleet_proof_matches, ilm_recovery_export_member_epochs_sha256, + ilm_recovery_export_topology_generation, +}; +use crate::storage_api_contracts::{list::ListOperations as _, namespace::NamespaceLocking as _, object::HTTPPreconditions}; +use crate::store::ECStore; + +pub const ILM_RECOVERY_EXPORT_SCHEMA: &str = "rustfs-ilm-recovery-export-v1"; +pub const ILM_RECOVERY_EXPORT_PREFIX: &str = "ilm/recovery-exports"; +pub const MAX_ILM_RECOVERY_EXPORT_SIZE: usize = 128 * 1024; +const MAX_ILM_RECOVERY_EXPORTS: usize = 10_000; +const MAX_ILM_RECOVERY_EXPORT_BYTES: u64 = 1024 * 1024 * 1024; +const MAX_ACTOR_EXPORTS_PER_MINUTE: usize = 10; +const MAX_CLUSTER_EXPORTS_PER_MINUTE: usize = 100; +const EXPORT_RETENTION_NANOS: i64 = 90 * 24 * 60 * 60 * 1_000_000_000; +const EXPORT_ADMISSION_LOCK: &str = "ilm/recovery-admission/export.lock"; +const MAX_LEGACY_TIER_DELETE_SOURCE_SIZE: usize = 64 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IlmRecoveryExportObservation { + pub control_id: String, + pub protocol: IlmRecoveryProtocol, + pub control_etag: String, + pub control_revision: u64, + pub classification: IlmRecoveryClassification, + pub canonical_source_path: String, + pub source_generation: IlmRecoverySourceGeneration, + pub topology_generation: String, + pub member_epochs_sha256: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IlmRecoveryExport { + pub export_id: String, + pub control_id: String, + pub protocol: IlmRecoveryProtocol, + pub control_etag: String, + pub control_revision: u64, + pub classification: IlmRecoveryClassification, + pub canonical_source_path: String, + pub source_generation: IlmRecoverySourceGeneration, + pub topology_generation: String, + pub member_epochs_sha256: String, + pub creator_sha256: String, + pub created_at_unix_nanos: i64, + pub retain_until_unix_nanos: i64, + pub source_bytes_base64: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedIlmRecoveryExport { + schema: String, + content_sha256: String, + export: IlmRecoveryExport, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct IlmRecoveryExportCreated { + pub export_id: String, + pub content_sha256: String, + pub encoded: Vec, + pub replayed: bool, +} + +impl IlmRecoveryExport { + fn validate(&self) -> Result<()> { + self.source_generation.validate().map_err(Error::other)?; + validate_sha256(&self.export_id, "ILM recovery export ID is invalid")?; + validate_sha256(&self.control_id, "ILM recovery export control ID is invalid")?; + validate_sha256(&self.topology_generation, "ILM recovery export topology generation is invalid")?; + validate_sha256(&self.member_epochs_sha256, "ILM recovery export member epoch digest is invalid")?; + validate_sha256(&self.creator_sha256, "ILM recovery export creator digest is invalid")?; + if self.protocol != IlmRecoveryProtocol::TierDeleteJournal + || self.classification != IlmRecoveryClassification::RetainedAmbiguous + || !is_legacy_export_schema(&self.source_generation.source_schema) + { + return Err(Error::other("ILM recovery export source is not an exportable legacy journal")); + } + if self.control_etag.trim().is_empty() || self.control_revision == 0 { + return Err(Error::other("ILM recovery export control generation is invalid")); + } + if self.canonical_source_path.is_empty() + || self.canonical_source_path.starts_with('/') + || self.canonical_source_path.ends_with('/') + || self.canonical_source_path.split('/').any(str::is_empty) + { + return Err(Error::other("ILM recovery export source path is invalid")); + } + if self.created_at_unix_nanos <= 0 + || self.retain_until_unix_nanos < self.created_at_unix_nanos.saturating_add(EXPORT_RETENTION_NANOS) + { + return Err(Error::other("ILM recovery export retention is invalid")); + } + let source = base64_simd::STANDARD + .decode_to_vec(self.source_bytes_base64.as_bytes()) + .map_err(|_| Error::other("ILM recovery export source encoding is invalid"))?; + validate_legacy_tier_delete_recovery_source(&self.canonical_source_path, &self.source_generation.source_schema, &source)?; + let encoded_len = u64::try_from(source.len()).map_err(|_| Error::other("ILM recovery export source length overflow"))?; + if source.is_empty() + || source.len() > MAX_LEGACY_TIER_DELETE_SOURCE_SIZE + || hex_sha256(&source, ToOwned::to_owned) != self.source_generation.content_sha256 + || self.source_generation.copies.iter().any(|copy| { + copy.canonical_path != self.canonical_source_path + || copy.etag != self.source_generation.source_etag + || copy.content_sha256 != self.source_generation.content_sha256 + || copy.encoded_len != encoded_len + }) + { + return Err(Error::other("ILM recovery export source bytes do not match the observed generation")); + } + if recovery_export_id(&self.control_id, &self.source_generation)? != self.export_id { + return Err(Error::other("ILM recovery export ID does not match its source generation")); + } + Ok(()) + } + + pub fn encode(&self) -> Result> { + self.validate()?; + let export_bytes = serde_json::to_vec(self).map_err(Error::other)?; + let persisted = PersistedIlmRecoveryExport { + schema: ILM_RECOVERY_EXPORT_SCHEMA.to_string(), + content_sha256: hex_sha256(&export_bytes, ToOwned::to_owned), + export: self.clone(), + }; + let encoded = serde_json::to_vec(&persisted).map_err(Error::other)?; + if encoded.len() > MAX_ILM_RECOVERY_EXPORT_SIZE { + return Err(Error::other("encoded ILM recovery export exceeds maximum size")); + } + Ok(encoded) + } + + pub fn decode(expected_export_id: &str, data: &[u8]) -> Result { + validate_sha256(expected_export_id, "ILM recovery export ID is invalid")?; + if data.len() > MAX_ILM_RECOVERY_EXPORT_SIZE { + return Err(Error::other("encoded ILM recovery export exceeds maximum size")); + } + let persisted: PersistedIlmRecoveryExport = serde_json::from_slice(data).map_err(Error::other)?; + if persisted.schema != ILM_RECOVERY_EXPORT_SCHEMA { + return Err(Error::other("ILM recovery export schema is unsupported")); + } + validate_sha256(&persisted.content_sha256, "ILM recovery export checksum is invalid")?; + let export_bytes = serde_json::to_vec(&persisted.export).map_err(Error::other)?; + if hex_sha256(&export_bytes, ToOwned::to_owned) != persisted.content_sha256 { + return Err(Error::other("ILM recovery export checksum mismatch")); + } + persisted.export.validate()?; + if persisted.export.export_id != expected_export_id { + return Err(Error::other("ILM recovery export ID does not match record key")); + } + Ok(persisted.export) + } +} + +pub fn recovery_export_record_object_name(protocol: IlmRecoveryProtocol, export_id: &str) -> Result { + validate_sha256(export_id, "ILM recovery export ID is invalid")?; + Ok(format!( + "{}/{}/{}/{}/{}.json", + ILM_RECOVERY_EXPORT_PREFIX, + protocol.as_str(), + &export_id[..2], + &export_id[2..4], + export_id + )) +} + +pub fn recovery_export_id_from_record_object_name(object: &str) -> Result<(IlmRecoveryProtocol, String)> { + let suffix = object + .strip_prefix(ILM_RECOVERY_EXPORT_PREFIX) + .and_then(|suffix| suffix.strip_prefix('/')) + .ok_or_else(|| Error::other("ILM recovery export path has wrong prefix"))?; + let mut parts = suffix.split('/'); + let protocol = match parts.next() { + Some("tier_delete_journal") => IlmRecoveryProtocol::TierDeleteJournal, + _ => return Err(Error::other("ILM recovery export protocol is invalid")), + }; + let shard_a = parts + .next() + .ok_or_else(|| Error::other("ILM recovery export path is incomplete"))?; + let shard_b = parts + .next() + .ok_or_else(|| Error::other("ILM recovery export path is incomplete"))?; + let export_id = parts + .next() + .and_then(|name| name.strip_suffix(".json")) + .ok_or_else(|| Error::other("ILM recovery export suffix is invalid"))?; + if parts.next().is_some() { + return Err(Error::other("ILM recovery export path is not canonical")); + } + validate_sha256(export_id, "ILM recovery export ID is invalid")?; + if shard_a != &export_id[..2] || shard_b != &export_id[2..4] { + return Err(Error::other("ILM recovery export shard does not match export ID")); + } + Ok((protocol, export_id.to_string())) +} + +pub async fn inspect_recovery_export_observation(api: Arc, control_id: &str) -> Result { + let proof = acquire_ilm_recovery_export_fleet_proof() + .await + .ok_or_else(|| Error::other("ILM recovery export fleet proof is unavailable"))?; + let observed_control = load_exportable_control(api.clone(), control_id).await?; + let observed_source = observe_export_source( + api, + &observed_control.control.identity.canonical_source_path, + &observed_control.control.observed_source_generation.source_schema, + ) + .await?; + if !observed_source.is_consistent() + || observed_source.generation != observed_control.control.observed_source_generation + || !ilm_recovery_export_fleet_proof_matches(&proof).await + { + return Err(Error::other("ILM recovery export observation changed or is incomplete")); + } + Ok(IlmRecoveryExportObservation { + control_id: control_id.to_string(), + protocol: observed_control.control.identity.protocol, + control_etag: observed_control.etag, + control_revision: observed_control.control.revision, + classification: observed_control.control.classification, + canonical_source_path: observed_control.control.identity.canonical_source_path, + source_generation: observed_source.generation, + topology_generation: ilm_recovery_export_topology_generation(&proof), + member_epochs_sha256: ilm_recovery_export_member_epochs_sha256(&proof), + }) +} + +pub async fn create_recovery_export( + api: Arc, + observation: &IlmRecoveryExportObservation, + creator_sha256: &str, +) -> Result { + validate_sha256(creator_sha256, "ILM recovery export creator digest is invalid")?; + let lock = api.new_ns_lock(RUSTFS_META_BUCKET, EXPORT_ADMISSION_LOCK).await?; + let admission_guard = lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?; + + let proof = acquire_ilm_recovery_export_fleet_proof() + .await + .ok_or_else(|| Error::other("ILM recovery export fleet proof is unavailable"))?; + if ilm_recovery_export_topology_generation(&proof) != observation.topology_generation + || ilm_recovery_export_member_epochs_sha256(&proof) != observation.member_epochs_sha256 + { + return Err(Error::PreconditionFailed); + } + let control_object = recovery_control_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, &observation.control_id) + .map_err(Error::other)?; + let control_lock = api.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?; + let control_guard = control_lock + .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let source_lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &observation.canonical_source_path) + .await?; + let source_guard = source_lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?; + let locks_current = || !admission_guard.is_lock_lost() && !control_guard.is_lock_lost() && !source_guard.is_lock_lost(); + let (current, current_source_bytes) = current_observation_under_proof_no_lock(api.clone(), observation, &proof).await?; + if ¤t != observation || !locks_current() { + return Err(Error::PreconditionFailed); + } + let current_source_base64 = base64_simd::STANDARD.encode_to_string(current_source_bytes); + let candidate_export_id = recovery_export_id(¤t.control_id, ¤t.source_generation)?; + let object = recovery_export_record_object_name(current.protocol, &candidate_export_id)?; + match load_recovery_export_decoded(api.clone(), &candidate_export_id).await { + Ok((existing, export)) if export_matches_observation(&export, observation) => { + if !locks_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await { + return Err(Error::PreconditionFailed); + } + api.record_durable_ilm_decommission_progress(&object, &existing.encoded) + .await?; + if !locks_current() { + return Err(Error::PreconditionFailed); + } + return Ok(existing.with_replayed()); + } + Ok(_) => return Err(Error::PreconditionFailed), + Err(Error::ConfigNotFound) => {} + Err(err) => return Err(err), + } + let inventory = collect_export_inventory(api.clone()).await?; + if !locks_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await { + return Err(Error::PreconditionFailed); + } + let created_at_unix_nanos = now_unix_nanos()?; + let export = build_export_from_source(¤t, creator_sha256, created_at_unix_nanos, ¤t_source_base64)?; + let encoded = export.encode()?; + inventory.check(creator_sha256, encoded.len(), created_at_unix_nanos)?; + + let mut write_options = ObjectOptions { + max_parity: true, + write_completion: WriteCompletion::TailDrained, + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }; + write_options.add_namespace_lock_guard(&admission_guard); + write_options.add_namespace_lock_guard(&control_guard); + write_options.add_namespace_lock_guard(&source_guard); + if !locks_current() { + return Err(Error::PreconditionFailed); + } + let write_result = config_boundary::save_config_with_opts(api.clone(), &object, encoded.clone(), &write_options).await; + let stored = match load_recovery_export(api.clone(), &export.export_id).await { + Ok(stored) if stored.encoded == encoded => stored, + Ok(_) => return Err(Error::PreconditionFailed), + Err(read_err) => return Err(write_result.err().unwrap_or(read_err)), + }; + if !locks_current() || !ilm_recovery_export_fleet_proof_matches(&proof).await { + return Err(Error::PreconditionFailed); + } + api.record_durable_ilm_decommission_progress(&object, &encoded).await?; + if !locks_current() { + return Err(Error::PreconditionFailed); + } + Ok(stored) +} + +pub async fn load_recovery_export(api: Arc, export_id: &str) -> Result { + let (created, _) = load_recovery_export_decoded(api, export_id).await?; + Ok(created) +} + +async fn load_recovery_export_decoded( + api: Arc, + export_id: &str, +) -> Result<(IlmRecoveryExportCreated, IlmRecoveryExport)> { + let object = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, export_id)?; + let encoded = config_boundary::read_config_limited_preserve_empty(api, &object, MAX_ILM_RECOVERY_EXPORT_SIZE).await?; + let export = IlmRecoveryExport::decode(export_id, &encoded)?; + let content_sha256 = hex_sha256(&encoded, ToOwned::to_owned); + Ok(( + IlmRecoveryExportCreated { + export_id: export.export_id.clone(), + content_sha256, + encoded, + replayed: false, + }, + export, + )) +} + +impl IlmRecoveryExportCreated { + fn with_replayed(mut self) -> Self { + self.replayed = true; + self + } +} + +async fn load_exportable_control(api: Arc, control_id: &str) -> Result { + load_exportable_control_with_options(api, control_id, &ObjectOptions::default()).await +} + +async fn load_exportable_control_no_lock(api: Arc, control_id: &str) -> Result { + load_exportable_control_with_options( + api, + control_id, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await +} + +async fn load_exportable_control_with_options( + api: Arc, + control_id: &str, + options: &ObjectOptions, +) -> Result { + let object = recovery_control_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, control_id).map_err(Error::other)?; + let (data, metadata) = + config_boundary::read_config_limited_preserve_empty_with_metadata(api, &object, options, MAX_ILM_RECOVERY_CONTROL_SIZE) + .await?; + let etag = metadata + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or_else(|| Error::other("ILM recovery control is missing an ETag"))?; + let control = IlmRecoveryControl::decode(control_id, &data).map_err(Error::other)?; + if control.identity.protocol != IlmRecoveryProtocol::TierDeleteJournal + || control.classification != IlmRecoveryClassification::RetainedAmbiguous + || !is_legacy_export_schema(&control.observed_source_generation.source_schema) + { + return Err(Error::other("ILM recovery control is not exportable")); + } + Ok(ObservedIlmRecoveryControl { control, etag }) +} + +async fn current_observation_under_proof_no_lock( + api: Arc, + expected: &IlmRecoveryExportObservation, + proof: &crate::services::notification_sys::IlmRecoveryExportFleetProofToken, +) -> Result<(IlmRecoveryExportObservation, Vec)> { + let observed_control = load_exportable_control_no_lock(api.clone(), &expected.control_id).await?; + let observed_source = observe_export_source_no_lock( + api, + &observed_control.control.identity.canonical_source_path, + &observed_control.control.observed_source_generation.source_schema, + ) + .await?; + let source_bytes = observed_source + .canonical_data + .clone() + .ok_or_else(|| Error::other("ILM recovery export source copies diverge"))?; + if !observed_source.is_consistent() + || observed_source.generation != observed_control.control.observed_source_generation + || !ilm_recovery_export_fleet_proof_matches(proof).await + { + return Err(Error::PreconditionFailed); + } + Ok(( + IlmRecoveryExportObservation { + control_id: expected.control_id.clone(), + protocol: observed_control.control.identity.protocol, + control_etag: observed_control.etag, + control_revision: observed_control.control.revision, + classification: observed_control.control.classification, + canonical_source_path: observed_control.control.identity.canonical_source_path, + source_generation: observed_source.generation, + topology_generation: ilm_recovery_export_topology_generation(proof), + member_epochs_sha256: ilm_recovery_export_member_epochs_sha256(proof), + }, + source_bytes, + )) +} + +async fn observe_export_source( + api: Arc, + canonical_path: &str, + source_schema: &str, +) -> Result { + if canonical_path.is_empty() + || canonical_path.starts_with('/') + || canonical_path.ends_with('/') + || canonical_path.split('/').any(str::is_empty) + || !is_legacy_export_schema(source_schema) + { + return Err(Error::other("ILM recovery export source identity is invalid")); + } + let lock = api.new_ns_lock(RUSTFS_META_BUCKET, canonical_path).await?; + let _guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?; + observe_export_source_no_lock(api, canonical_path, source_schema).await +} + +async fn observe_export_source_no_lock( + api: Arc, + canonical_path: &str, + source_schema: &str, +) -> Result { + let mut copies = Vec::new(); + let mut canonical: Option<(String, String, Vec)> = None; + let mut consistent = true; + for set in api.all_set_disks() { + let authority = format!("pool-{}/set-{}", set.pool_index, set.set_index); + let result = config_boundary::read_config_limited_preserve_empty_with_metadata( + set, + canonical_path, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + MAX_LEGACY_TIER_DELETE_SOURCE_SIZE, + ) + .await; + match result { + Ok((data, metadata)) => { + if data.is_empty() || data.len() > MAX_LEGACY_TIER_DELETE_SOURCE_SIZE { + return Err(Error::other("ILM recovery export source exceeds its protocol size limit")); + } + validate_legacy_tier_delete_recovery_source(canonical_path, source_schema, &data)?; + let etag = metadata + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or_else(|| Error::other("ILM recovery export source copy is missing an ETag"))?; + let content_sha256 = hex_sha256(&data, ToOwned::to_owned); + let encoded_len = + u64::try_from(data.len()).map_err(|_| Error::other("ILM recovery export source length does not fit u64"))?; + copies.push(IlmRecoverySourceCopy { + authority, + canonical_path: canonical_path.to_string(), + etag: etag.clone(), + encoded_len, + content_sha256: content_sha256.clone(), + }); + match canonical.as_ref() { + Some((first_etag, first_digest, first_data)) => { + consistent &= first_etag == &etag && first_digest == &content_sha256 && first_data == &data; + } + None => canonical = Some((etag, content_sha256, data)), + } + } + Err(err) if export_source_is_missing(&err) => {} + Err(err) => return Err(err), + } + } + let Some((source_etag, content_sha256, source_bytes)) = canonical else { + return Err(Error::ConfigNotFound); + }; + let generation = + IlmRecoverySourceGeneration::new(source_schema, source_etag, content_sha256, copies).map_err(Error::other)?; + Ok(ObservedIlmRecoverySource { + generation, + canonical_data: consistent.then_some(source_bytes), + }) +} + +fn export_source_is_missing(err: &Error) -> bool { + matches!( + err, + Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::VersionNotFound(_, _, _) + ) +} + +fn build_export_from_source( + observation: &IlmRecoveryExportObservation, + creator_sha256: &str, + created_at_unix_nanos: i64, + source_bytes_base64: &str, +) -> Result { + let retain_until_unix_nanos = created_at_unix_nanos + .checked_add(EXPORT_RETENTION_NANOS) + .ok_or_else(|| Error::other("ILM recovery export retention timestamp overflow"))?; + let export = IlmRecoveryExport { + export_id: recovery_export_id(&observation.control_id, &observation.source_generation)?, + control_id: observation.control_id.clone(), + protocol: observation.protocol, + control_etag: observation.control_etag.clone(), + control_revision: observation.control_revision, + classification: observation.classification, + canonical_source_path: observation.canonical_source_path.clone(), + source_generation: observation.source_generation.clone(), + topology_generation: observation.topology_generation.clone(), + member_epochs_sha256: observation.member_epochs_sha256.clone(), + creator_sha256: creator_sha256.to_string(), + created_at_unix_nanos, + retain_until_unix_nanos, + source_bytes_base64: source_bytes_base64.to_string(), + }; + export.validate()?; + Ok(export) +} + +pub(crate) fn recovery_export_id(control_id: &str, generation: &IlmRecoverySourceGeneration) -> Result { + validate_sha256(control_id, "ILM recovery export control ID is invalid")?; + validate_sha256(&generation.content_sha256, "ILM recovery export source checksum is invalid")?; + validate_sha256(&generation.copy_set_sha256, "ILM recovery export copy-set checksum is invalid")?; + let mut data = Vec::new(); + for part in [control_id, &generation.content_sha256, &generation.copy_set_sha256] { + data.extend_from_slice(&(part.len() as u64).to_be_bytes()); + data.extend_from_slice(part.as_bytes()); + } + Ok(hex_sha256(&data, ToOwned::to_owned)) +} + +fn export_matches_observation(export: &IlmRecoveryExport, observation: &IlmRecoveryExportObservation) -> bool { + export.control_id == observation.control_id + && export.protocol == observation.protocol + && export.classification == observation.classification + && export.canonical_source_path == observation.canonical_source_path + && export.source_generation == observation.source_generation +} + +#[derive(Debug, Default)] +struct IlmRecoveryExportInventory { + count: usize, + bytes: u64, + creations: Vec<(i64, String)>, +} + +impl IlmRecoveryExportInventory { + fn check(&self, creator_sha256: &str, candidate_len: usize, now: i64) -> Result<()> { + let recent_after = now.saturating_sub(60 * 1_000_000_000); + let cluster_recent = self + .creations + .iter() + .filter(|(created_at, _)| *created_at > recent_after) + .count(); + let actor_recent = self + .creations + .iter() + .filter(|(created_at, creator)| *created_at > recent_after && creator == creator_sha256) + .count(); + check_export_admission(self.count, self.bytes, actor_recent, cluster_recent, candidate_len) + } +} + +async fn collect_export_inventory(api: Arc) -> Result { + let mut marker = None; + let mut seen_markers = HashSet::new(); + let mut inventory = IlmRecoveryExportInventory::default(); + loop { + let page = api + .clone() + .list_objects_v2( + RUSTFS_META_BUCKET, + &format!("{ILM_RECOVERY_EXPORT_PREFIX}/"), + marker.clone(), + None, + 1_000, + false, + None, + false, + ) + .await?; + for object in page.objects { + let (_, export_id) = recovery_export_id_from_record_object_name(&object.name)?; + let (stored, export) = load_recovery_export_decoded(api.clone(), &export_id).await?; + inventory.count = inventory + .count + .checked_add(1) + .ok_or_else(|| Error::other("ILM recovery export count overflow"))?; + inventory.bytes = inventory + .bytes + .checked_add(u64::try_from(stored.encoded.len()).map_err(|_| Error::other("ILM recovery export size overflow"))?) + .ok_or_else(|| Error::other("ILM recovery export byte total overflow"))?; + inventory + .creations + .push((export.created_at_unix_nanos, export.creator_sha256)); + } + if !page.is_truncated { + break; + } + let next = page + .next_continuation_token + .ok_or_else(|| Error::other("ILM recovery export inventory omitted its continuation marker"))?; + marker = Some(record_export_inventory_marker(&mut seen_markers, next)?); + } + Ok(inventory) +} + +fn record_export_inventory_marker(seen_markers: &mut HashSet, next: String) -> Result { + if !seen_markers.insert(next.clone()) { + return Err(Error::other("ILM recovery export inventory repeated its continuation marker")); + } + Ok(next) +} + +fn check_export_admission( + count: usize, + bytes: u64, + actor_recent: usize, + cluster_recent: usize, + candidate_len: usize, +) -> Result<()> { + let candidate_len = u64::try_from(candidate_len).map_err(|_| Error::other("ILM recovery export size does not fit u64"))?; + if count >= MAX_ILM_RECOVERY_EXPORTS + || bytes + .checked_add(candidate_len) + .is_none_or(|total| total > MAX_ILM_RECOVERY_EXPORT_BYTES) + || actor_recent >= MAX_ACTOR_EXPORTS_PER_MINUTE + || cluster_recent >= MAX_CLUSTER_EXPORTS_PER_MINUTE + { + return Err(Error::SlowDown); + } + Ok(()) +} + +fn is_legacy_export_schema(schema: &str) -> bool { + matches!(schema, TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA | TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA) +} + +fn validate_sha256(value: &str, message: &'static str) -> Result<()> { + if !is_sha256_checksum(value) + || value + .bytes() + .any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase()) + { + return Err(Error::other(message)); + } + Ok(()) +} + +fn now_unix_nanos() -> Result { + i64::try_from(time::OffsetDateTime::now_utc().unix_timestamp_nanos()) + .map_err(|_| Error::other("ILM recovery export timestamp does not fit i64")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bucket::lifecycle::recovery_control::IlmRecoverySourceCopy; + + const PINNED_V1_EXPORT: &[u8] = br#"{"schema":"rustfs-ilm-recovery-export-v1","content_sha256":"3dfb3ec3892256e909de1211c1a963ca7008963ff32b3a869f7161a7b9b44028","export":{"export_id":"2b78e7a825bfc2edbf7f773d0b6ed3bf93e360ff1702d73a449109c11bfaa105","control_id":"0fcd568a5cb9bdb4677b69354b11ee415af8f784519cff3da49a26f84eaee7f2","protocol":"tier_delete_journal","control_etag":"control-etag","control_revision":1,"classification":"retained_ambiguous","canonical_source_path":"ilm/tier-delete-journal/872072554f66ab326f10ce7adbae11422b7a4b0663aa7112d6061a8f6ed41b94.json","source_generation":{"source_schema":"rustfs-tier-delete-journal-v1","source_etag":"etag-a","content_sha256":"0e0b010ebdeeb7b41473fe8575e989d6bb1303c0ca551dd984e9400f0ae306bd","copy_set_sha256":"5a7406115b6c3923ffe79dcd1f43ccae7beed786e557163f019dd10ec409a653","copies":[{"authority":"pool-0/set-0","canonical_path":"ilm/tier-delete-journal/872072554f66ab326f10ce7adbae11422b7a4b0663aa7112d6061a8f6ed41b94.json","etag":"etag-a","encoded_len":81,"content_sha256":"0e0b010ebdeeb7b41473fe8575e989d6bb1303c0ca551dd984e9400f0ae306bd"}]},"topology_generation":"e6e2b826e31fca5c36125c48f130dcb6f961e698ff8a8776a1f290cf0892e8e6","member_epochs_sha256":"612dd8a861161819a4ad8f6f3e2a0567602877c043a2353ca933a13c78dc0ed4","creator_sha256":"50c9c4aeb40b5b206b6d98f516f8b8c0efd29ce2e56a76b345fb9240c225a1b7","created_at_unix_nanos":1000000000,"retain_until_unix_nanos":7776001000000000,"source_bytes_base64":"eyJ2ZXJzaW9uIjoxLCJvYmpfbmFtZSI6ImxlZ2FjeS9yZW1vdGUiLCJ2ZXJzaW9uX2lkIjoib3BhcXVlIiwidGllcl9uYW1lIjoiV0FSTSJ9"}}"#; + + fn legacy_source() -> Vec { + br#"{"version":1,"obj_name":"legacy/remote","version_id":"opaque","tier_name":"WARM"}"#.to_vec() + } + + fn observation() -> IlmRecoveryExportObservation { + let source = legacy_source(); + let source_path = super::super::tier_delete_journal::tier_delete_journal_object_name( + &super::super::tier_delete_journal::decode_tier_delete_journal_entry(&source).expect("legacy fixture should decode"), + ); + let source_sha256 = hex_sha256(&source, ToOwned::to_owned); + let generation = IlmRecoverySourceGeneration::new( + TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, + "etag-a", + source_sha256.clone(), + vec![IlmRecoverySourceCopy { + authority: "pool-0/set-0".to_string(), + canonical_path: source_path.clone(), + etag: "etag-a".to_string(), + encoded_len: source.len() as u64, + content_sha256: source_sha256, + }], + ) + .expect("generation should be valid"); + IlmRecoveryExportObservation { + control_id: hex_sha256(b"control", ToOwned::to_owned), + protocol: IlmRecoveryProtocol::TierDeleteJournal, + control_etag: "control-etag".to_string(), + control_revision: 1, + classification: IlmRecoveryClassification::RetainedAmbiguous, + canonical_source_path: source_path, + source_generation: generation, + topology_generation: hex_sha256(b"topology", ToOwned::to_owned), + member_epochs_sha256: hex_sha256(b"epochs", ToOwned::to_owned), + } + } + + #[test] + fn recovery_export_round_trip_is_strict_and_deterministic() { + let observed = observation(); + let creator = hex_sha256(b"actor", ToOwned::to_owned); + let export = build_export_from_source( + &observed, + &creator, + 1_000_000_000, + &base64_simd::STANDARD.encode_to_string(legacy_source()), + ) + .expect("export should be valid"); + assert_eq!( + export.export_id, + recovery_export_id(&observed.control_id, &observed.source_generation).unwrap() + ); + let encoded = export.encode().expect("export should encode"); + assert_eq!(encoded, PINNED_V1_EXPORT, "v1 export wire format must remain pinned"); + assert_eq!(IlmRecoveryExport::decode(&export.export_id, &encoded).unwrap(), export); + assert_eq!( + IlmRecoveryExport::decode("2b78e7a825bfc2edbf7f773d0b6ed3bf93e360ff1702d73a449109c11bfaa105", PINNED_V1_EXPORT) + .unwrap(), + export, + ); + + let path = recovery_export_record_object_name(export.protocol, &export.export_id).unwrap(); + let durable = super::super::durable_namespace::validate_durable_ilm_record(&path, &encoded) + .expect("export should be registered as a durable ILM record"); + assert_eq!(durable.namespace, "recovery-export"); + assert_eq!(durable.id_kind, "export_id"); + assert_eq!(durable.id, export.export_id); + + let mut wrong_source = export.clone(); + wrong_source.source_bytes_base64 = base64_simd::STANDARD.encode_to_string(b"changed"); + assert!(wrong_source.encode().is_err()); + + let mut persisted: serde_json::Value = serde_json::from_slice(&encoded).unwrap(); + persisted["unknown"] = serde_json::json!(true); + assert!(IlmRecoveryExport::decode(&export.export_id, &serde_json::to_vec(&persisted).unwrap()).is_err()); + } + + #[test] + fn export_inventory_rejects_non_adjacent_continuation_cycles() { + let mut seen = HashSet::new(); + assert_eq!(record_export_inventory_marker(&mut seen, "a".to_string()).unwrap(), "a"); + assert_eq!(record_export_inventory_marker(&mut seen, "b".to_string()).unwrap(), "b"); + record_export_inventory_marker(&mut seen, "a".to_string()) + .expect_err("a non-adjacent continuation marker cycle must fail closed"); + } + + #[test] + fn recovery_export_path_rejects_noncanonical_shards() { + let id = hex_sha256(b"export", ToOwned::to_owned); + let path = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, &id).unwrap(); + assert_eq!(recovery_export_id_from_record_object_name(&path).unwrap().1, id); + let wrong_shard = path.replacen(&format!("/{}/", &id[..2]), "/zz/", 1); + assert!(recovery_export_id_from_record_object_name(&wrong_shard).is_err()); + } + + #[test] + fn canonical_replay_survives_fleet_rotation_but_not_source_change() { + let observed = observation(); + let creator = hex_sha256(b"actor", ToOwned::to_owned); + let export = build_export_from_source( + &observed, + &creator, + 1_000_000_000, + &base64_simd::STANDARD.encode_to_string(legacy_source()), + ) + .unwrap(); + let mut rotated = observed; + rotated.control_etag = "new-control-etag".to_string(); + rotated.control_revision += 1; + rotated.topology_generation = hex_sha256(b"new-topology", ToOwned::to_owned); + rotated.member_epochs_sha256 = hex_sha256(b"new-members", ToOwned::to_owned); + assert!(export_matches_observation(&export, &rotated)); + + rotated.source_generation.content_sha256 = hex_sha256(b"changed", ToOwned::to_owned); + assert!(!export_matches_observation(&export, &rotated)); + } + + #[test] + fn export_admission_enforces_exact_count_byte_and_rate_boundaries() { + assert!(check_export_admission(9_999, MAX_ILM_RECOVERY_EXPORT_BYTES - 1, 9, 99, 1).is_ok()); + assert!(check_export_admission(10_000, 0, 0, 0, 1).is_err()); + assert!(check_export_admission(0, MAX_ILM_RECOVERY_EXPORT_BYTES, 0, 0, 1).is_err()); + assert!(check_export_admission(0, 0, 10, 0, 1).is_err()); + assert!(check_export_admission(0, 0, 0, 100, 1).is_err()); + } +} diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index 03b04a8a2..b8b529e43 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -82,8 +82,8 @@ const TIER_DELETE_DISPATCH_MEMBER_DELETE_CONCURRENCY: usize = 32; const TIER_DELETE_DISPATCH_PREPARE_CONCURRENCY: usize = 16; const TIER_DELETE_DISPATCH_CAS_CONCURRENCY: usize = 32; const TIER_DELETE_JOURNAL_VERSION: u8 = 2; -const TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v1"; -const TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v2"; +pub(crate) const TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v1"; +pub(crate) const TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-v2"; const TIER_DELETE_JOURNAL_UNKNOWN_RECOVERY_SCHEMA: &str = "rustfs-tier-delete-journal-unknown"; const TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS: &str = "tier_delete_journal_v1"; const TIER_DELETE_JOURNAL_V2_RECOVERY_CLASS: &str = "tier_delete_journal_v2"; @@ -884,6 +884,23 @@ struct PersistedTierDeleteJournalEntry { } impl PersistedTierDeleteJournalEntry { + fn validate_legacy_recovery_shape(&self) -> Result<()> { + let has_later_version_fields = self.version_id_exact.is_some() + || self.version_state.is_some() + || self.state.is_some() + || self.source.is_some() + || self.dispatch.is_some(); + match self.version { + 1 if self.backend_identity.is_none() && !has_later_version_fields => Ok(()), + TIER_DELETE_JOURNAL_VERSION if self.backend_identity.is_some() && !has_later_version_fields => Ok(()), + 1 => Err(Error::other("tier delete journal v1 entry contains fields from a later version")), + TIER_DELETE_JOURNAL_VERSION => Err(Error::other( + "tier delete journal v2 entry is missing its identity or contains fields from a later version", + )), + _ => Err(Error::other("tier delete journal is not an exportable legacy version")), + } + } + fn from_jentry(je: &Jentry) -> Result { validate_version_state(je.version_state, &je.version_id, je.version_id_exact)?; let legacy_unknown = je.version_state == rustfs_filemeta::TransitionVersionState::Unknown; @@ -5531,6 +5548,12 @@ fn canonical_legacy_tier_delete_journal_identity(object_name: &str) -> Option<&s .then_some(identity) } +pub(crate) fn validate_legacy_tier_delete_recovery_path(object_name: &str) -> Result<()> { + canonical_legacy_tier_delete_journal_identity(object_name) + .map(|_| ()) + .ok_or_else(|| Error::other("legacy tier delete journal path is not canonical")) +} + fn legacy_tier_delete_recovery_descriptor(entry: &Jentry) -> Option<(&'static str, &'static str)> { match entry.persisted_version { 1 => Some((TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V1_RECOVERY_CLASS)), @@ -5539,6 +5562,21 @@ fn legacy_tier_delete_recovery_descriptor(entry: &Jentry) -> Option<(&'static st } } +pub(crate) fn validate_legacy_tier_delete_recovery_source(object_name: &str, source_schema: &str, data: &[u8]) -> Result<()> { + validate_legacy_tier_delete_recovery_path(object_name)?; + let persisted: PersistedTierDeleteJournalEntry = + serde_json::from_slice(data).map_err(|err| Error::other_with_context("decode tier delete journal failed", err))?; + persisted.validate_legacy_recovery_shape()?; + let entry = persisted.into_jentry()?; + let Some((decoded_schema, _)) = legacy_tier_delete_recovery_descriptor(&entry) else { + return Err(Error::other("tier delete journal is not an exportable legacy version")); + }; + if decoded_schema != source_schema || tier_delete_journal_object_name(&entry) != object_name { + return Err(Error::other("legacy tier delete journal identity does not match its recovery source")); + } + Ok(()) +} + fn legacy_tier_delete_control_matches( control: &IlmRecoveryControl, identity: &IlmRecoveryControlIdentity, @@ -6140,17 +6178,18 @@ where #[cfg(test)] mod tests { use super::{ - TIER_DELETE_DISPATCH_MANIFEST_VERSION, TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE, TIER_DELETE_DISPATCH_PARENT_VERSION, - TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_LEGACY_PREFIX, TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION, - TIER_DELETE_JOURNAL_STATE_VERSION, TIER_DELETE_JOURNAL_TRANSACTION_VERSION, TIER_DELETE_JOURNAL_V6_PREFIX, - TierDeleteDispatchChunkBinding, TierDeleteDispatchManifest, TierDeleteDispatchManifestState, TierDeleteDispatchParent, - TierDeleteDispatchParentState, TierDeleteDispatchRecord, await_tier_delete_journal_recovery, + PersistedTierDeleteJournalEntry, TIER_DELETE_DISPATCH_MANIFEST_VERSION, TIER_DELETE_DISPATCH_PARENT_RECORD_TYPE, + TIER_DELETE_DISPATCH_PARENT_VERSION, TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_LEGACY_PREFIX, + TIER_DELETE_JOURNAL_SOLE_OWNER_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, TIER_DELETE_JOURNAL_TRANSACTION_VERSION, + TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA, TIER_DELETE_JOURNAL_V6_PREFIX, + TIER_DELETE_JOURNAL_VERSION, TierDeleteDispatchChunkBinding, TierDeleteDispatchManifest, TierDeleteDispatchManifestState, + TierDeleteDispatchParent, TierDeleteDispatchParentState, TierDeleteDispatchRecord, await_tier_delete_journal_recovery, decode_tier_delete_dispatch_record, decode_tier_delete_journal_entry, encode_tier_delete_dispatch_manifest, encode_tier_delete_dispatch_parent, encode_tier_delete_journal_entry, object_info_references_tier_delete, record_tier_delete_journal_backend_identity, same_tier_delete_authorization_identity, same_tier_delete_journal_identity, tier_delete_dispatch_child_matches_parent, tier_delete_dispatch_chunk_manifest_object_name, tier_delete_dispatch_journal_set_digest, tier_delete_dispatch_manifest_object_name, tier_delete_journal_object_name, - tier_delete_source_matches_dispatch_scope, + tier_delete_source_matches_dispatch_scope, validate_legacy_tier_delete_recovery_source, }; use crate::bucket::lifecycle::tier_sweeper::{ Jentry, TierDeleteDispatchBinding, TierDeleteJournalState, TierDeleteSourceIdentity, @@ -6609,6 +6648,72 @@ mod tests { } } + #[test] + fn legacy_recovery_export_rejects_fields_from_later_journal_versions() { + let later = bound_v6_journal_entry(TierDeleteJournalState::Prepared); + let v1 = PersistedTierDeleteJournalEntry { + version: 1, + obj_name: "remote/object".to_string(), + version_id: "opaque".to_string(), + tier_name: "WARM".to_string(), + backend_identity: None, + version_id_exact: None, + version_state: None, + state: None, + source: None, + dispatch: None, + }; + let mut v2 = v1.clone(); + v2.version = TIER_DELETE_JOURNAL_VERSION; + v2.backend_identity = Some([7; 32]); + + let assert_rejected = |persisted: PersistedTierDeleteJournalEntry, schema: &str| { + let normalized = persisted + .clone() + .into_jentry() + .expect("the generic compatibility decoder should demonstrate the discarded field"); + let object_name = tier_delete_journal_object_name(&normalized); + let encoded = serde_json::to_vec(&persisted).expect("mixed-version journal fixture should encode"); + let err = validate_legacy_tier_delete_recovery_source(&object_name, schema, &encoded) + .expect_err("legacy recovery export must reject fields from later versions"); + assert!(err.to_string().contains("later version")); + }; + + let mut invalid_v1 = Vec::new(); + let mut with_backend = v1.clone(); + with_backend.backend_identity = Some([7; 32]); + invalid_v1.push(with_backend); + for persisted in [&v1, &v2] { + let schema = if persisted.version == 1 { + TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA + } else { + TIER_DELETE_JOURNAL_V2_RECOVERY_SCHEMA + }; + let mut invalid = Vec::new(); + let mut with_exact = persisted.clone(); + with_exact.version_id_exact = Some(false); + invalid.push(with_exact); + let mut with_version_state = persisted.clone(); + with_version_state.version_state = Some(rustfs_filemeta::TransitionVersionState::Unknown); + invalid.push(with_version_state); + let mut with_state = persisted.clone(); + with_state.state = Some(TierDeleteJournalState::Committed); + invalid.push(with_state); + let mut with_source = persisted.clone(); + with_source.source = later.source.clone(); + invalid.push(with_source); + let mut with_dispatch = persisted.clone(); + with_dispatch.dispatch = later.dispatch.clone(); + invalid.push(with_dispatch); + for record in invalid { + assert_rejected(record, schema); + } + } + for record in invalid_v1 { + assert_rejected(record, TIER_DELETE_JOURNAL_V1_RECOVERY_SCHEMA); + } + } + #[test] fn tier_delete_journal_path_is_stable_and_sanitized() { let je = journal_entry(); diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index c6a15a6b9..f6f5f58e8 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -1688,6 +1688,15 @@ impl PeerRestClient { Ok((self.topology_member.clone(), supported_version, epoch)) } + pub async fn probe_ilm_recovery_export(&self, topology_fingerprint: String) -> Result<(String, Uuid)> { + let probe = rustfs_protos::ilm_recovery_export_capability_probe(Uuid::new_v4().as_bytes()); + let result = self + .heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe) + .await?; + let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?; + Ok((self.topology_member.clone(), epoch)) + } + pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> { let result = tokio::time::timeout(BUCKET_METADATA_RELOAD_TIMEOUT, async { let result = self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await; diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 8c5132487..4081a3e3a 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -449,14 +449,6 @@ where Ok(data) } -pub(crate) async fn read_config_limited(api: Arc, file: &str, max_bytes: usize) -> Result> -where - S: EcstoreObjectIO, -{ - let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), false, Some(max_bytes)).await?; - Ok(data) -} - pub(crate) async fn read_config_limited_preserve_empty(api: Arc, file: &str, max_bytes: usize) -> Result> where S: EcstoreObjectIO, @@ -465,6 +457,14 @@ where Ok(data) } +pub(crate) async fn read_config_limited(api: Arc, file: &str, max_bytes: usize) -> Result> +where + S: EcstoreObjectIO, +{ + let (data, _obj) = read_config_with_metadata_inner(api, file, &ObjectOptions::default(), false, Some(max_bytes)).await?; + Ok(data) +} + pub(crate) async fn read_config_limited_preserve_empty_with_metadata( api: Arc, file: &str, @@ -476,6 +476,18 @@ where read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await } +pub(crate) async fn read_config_limited_preserve_empty_with_metadata_opts( + api: Arc, + file: &str, + opts: &ObjectOptions, + max_bytes: usize, +) -> Result<(Vec, ObjectInfo)> +where + S: EcstoreObjectIO, +{ + read_config_with_metadata_inner(api, file, opts, true, Some(max_bytes)).await +} + /// Read an existing config object without treating an empty payload as absent. /// Callers that validate their own payload format need to distinguish corruption /// from `ConfigNotFound`. diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index d1955a397..7769ea1b1 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -33,11 +33,11 @@ use rustfs_madmin::net::NetInfo; use rustfs_madmin::{ItemState, ServerProperties, StorageInfo}; use rustfs_utils::XHost; use sha2::{Digest, Sha256}; -use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher}; +use std::collections::{BTreeMap, BTreeSet, HashMap, hash_map::DefaultHasher}; use std::future::Future; use std::hash::{Hash, Hasher}; use std::sync::{ - Arc, Mutex, OnceLock, + Arc, LazyLock, Mutex, OnceLock, atomic::{AtomicBool, AtomicUsize, Ordering}, }; use std::time::{Duration, Instant, SystemTime}; @@ -311,12 +311,20 @@ pub struct LegacyTransitionStateReconcileFleetProofToken { _permit: FleetCapabilityProofPermit, } +/// Effect-window authority for one immutable ILM recovery export. +pub struct IlmRecoveryExportFleetProofToken { + token: FleetCapabilityProofToken, + _permit: FleetCapabilityProofPermit, +} + static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock> = OnceLock::new(); static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock> = OnceLock::new(); static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock> = OnceLock::new(); static DECOMMISSION_TARGET_FENCE_FLEET_PROOF: OnceLock> = OnceLock::new(); static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock> = OnceLock::new(); +static ILM_RECOVERY_EXPORT_FLEET_PROOF: OnceLock> = OnceLock::new(); static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock = OnceLock::new(); +static ILM_RECOVERY_EXPORT_LOCAL_PROCESS_EPOCH: LazyLock = LazyLock::new(Uuid::new_v4); fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock { CROSS_POOL_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) @@ -338,6 +346,10 @@ fn legacy_transition_state_reconcile_fleet_proof_slot() -> &'static std::sync::R LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) } +fn ilm_recovery_export_fleet_proof_slot() -> &'static std::sync::RwLock { + ILM_RECOVERY_EXPORT_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default())) +} + fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) { if let Some(proof) = state.proof.take() { proof.generation.revoke(); @@ -573,6 +585,117 @@ pub async fn legacy_transition_state_reconcile_fleet_proof_matches( .await } +pub async fn acquire_ilm_recovery_export_fleet_proof() -> Option { + let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?; + let proof = { + let state = ilm_recovery_export_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + acquire_ilm_recovery_export_fleet_proof_from(&state, expected_topology, Instant::now())? + }; + let observed = observe_ilm_recovery_export_fleet(expected_topology).await?; + let state = ilm_recovery_export_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + ilm_recovery_export_fleet_proof_matches_observation_at(&state, &proof, expected_topology, &observed, Instant::now()) + .then_some(proof) +} + +fn acquire_ilm_recovery_export_fleet_proof_from( + state: &FleetCapabilityProofState, + expected_topology: &str, + now: Instant, +) -> Option { + let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?; + let permit = state.proof.as_ref()?.generation.try_acquire()?; + Some(IlmRecoveryExportFleetProofToken { token, _permit: permit }) +} + +pub async fn ilm_recovery_export_fleet_proof_matches(proof: &IlmRecoveryExportFleetProofToken) -> bool { + let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else { + return false; + }; + { + let state = ilm_recovery_export_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !ilm_recovery_export_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now()) { + return false; + } + } + let Some(observed) = observe_ilm_recovery_export_fleet(expected_topology).await else { + return false; + }; + let state = ilm_recovery_export_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + ilm_recovery_export_fleet_proof_matches_observation_at(&state, proof, expected_topology, &observed, Instant::now()) +} + +pub fn ilm_recovery_export_topology_generation(proof: &IlmRecoveryExportFleetProofToken) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"rustfs-ilm-recovery-export-topology-v1\0"); + hasher.update(proof.token.topology_fingerprint.as_bytes()); + rustfs_utils::crypto::hex(hasher.finalize().as_slice()) +} + +pub fn ilm_recovery_export_member_epochs_sha256(proof: &IlmRecoveryExportFleetProofToken) -> String { + let encoded = serde_json::to_vec(proof.token.peer_epochs.as_ref()).expect("member epoch map is JSON encodable"); + let mut hasher = Sha256::new(); + hasher.update(b"rustfs-ilm-recovery-export-members-v1\0"); + hasher.update(encoded); + rustfs_utils::crypto::hex(hasher.finalize().as_slice()) +} + +pub fn ilm_recovery_export_local_process_epoch() -> Uuid { + *ILM_RECOVERY_EXPORT_LOCAL_PROCESS_EPOCH +} + +fn ilm_recovery_export_fleet_proof_matches_at( + state: &FleetCapabilityProofState, + proof: &IlmRecoveryExportFleetProofToken, + expected_topology: &str, + now: Instant, +) -> bool { + proof._permit.generation.is_accepting() + && fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now) + && state + .proof + .as_ref() + .is_some_and(|current| Arc::ptr_eq(¤t.generation, &proof._permit.generation)) +} + +fn ilm_recovery_export_fleet_proof_matches_observation_at( + state: &FleetCapabilityProofState, + proof: &IlmRecoveryExportFleetProofToken, + expected_topology: &str, + observed: &BTreeMap, + now: Instant, +) -> bool { + ilm_recovery_export_fleet_proof_matches_at(state, proof, expected_topology, now) + && proof.token.peer_epochs.as_ref() == observed +} + +async fn observe_ilm_recovery_export_fleet(expected_topology: &str) -> Option> { + #[cfg(test)] + { + let state = ilm_recovery_export_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if fleet_capability_proof_valid_at(state.proof.as_ref(), expected_topology, Instant::now()) { + return state.proof.as_ref().map(|proof| proof.peer_epochs.as_ref().clone()); + } + } + let notification_sys = get_global_notification_sys()?; + timeout( + REMOTE_VERSION_STATE_PROBE_TIMEOUT, + notification_sys.probe_ilm_recovery_export_fleet(expected_topology), + ) + .await + .ok()? + .ok() +} + async fn legacy_transition_state_reconcile_fleet_proof_matches_with_observer( slot: &std::sync::RwLock, proof: &LegacyTransitionStateReconcileFleetProofToken, @@ -661,7 +784,7 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() { state.proof.clone() } else { Some(FleetCapabilityProof::new( - topology, + topology.clone(), Arc::new(BTreeMap::new()), now + Duration::from_secs(60 * 60), )) @@ -694,6 +817,21 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() { decommission_state.topology_conflict = false; decommission_state.draining_generation = None; decommission_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation); + drop(decommission_state); + let mut export_state = ilm_recovery_export_fleet_proof_slot() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !fleet_capability_proof_valid_at(export_state.proof.as_ref(), &topology, now) { + debug_assert!( + export_state + .proof + .as_ref() + .is_none_or(|current| current.generation.is_drained()) + ); + export_state.topology_conflict = false; + export_state.draining_generation = None; + export_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation); + } } #[cfg(test)] @@ -950,6 +1088,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { tier_delete_journal_fleet_proof_slot(), decommission_target_fence_fleet_proof_slot(), legacy_transition_state_reconcile_fleet_proof_slot(), + ilm_recovery_export_fleet_proof_slot(), ] { mark_fleet_capability_topology_conflict(slot); } @@ -959,29 +1098,42 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { tokio::spawn(async move { loop { - let result = match get_global_notification_sys() { - Some(notification_sys) => { - match timeout( + let notification_sys = get_global_notification_sys(); + let remote_version_state_probe = async { + match notification_sys.as_ref() { + Some(notification_sys) => timeout( REMOTE_VERSION_STATE_PROBE_TIMEOUT, notification_sys.probe_remote_version_state_fleet(&topology_fingerprint), ) .await - { - Ok(result) => result, - Err(_) => Err(Error::other("remote version state fleet capability probe timed out")), - } + .unwrap_or_else(|_| Err(Error::other("remote version state fleet capability probe timed out"))), + None => Err(Error::other("remote version state fleet capability notification system is unavailable")), } - None => Err(Error::other("remote version state fleet capability notification system is unavailable")), }; - let fence_probe = match get_global_notification_sys() { - Some(notification_sys) => timeout( - REMOTE_VERSION_STATE_PROBE_TIMEOUT, - notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint), - ) - .await - .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))), - None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")), + let cross_pool_fence_probe = async { + match notification_sys.as_ref() { + Some(notification_sys) => timeout( + REMOTE_VERSION_STATE_PROBE_TIMEOUT, + notification_sys.probe_cross_pool_fence_fleet(&topology_fingerprint), + ) + .await + .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))), + None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")), + } }; + let recovery_export_probe = async { + match notification_sys.as_ref() { + Some(notification_sys) => timeout( + REMOTE_VERSION_STATE_PROBE_TIMEOUT, + notification_sys.probe_ilm_recovery_export_fleet(&topology_fingerprint), + ) + .await + .unwrap_or_else(|_| Err(Error::other("ILM recovery export fleet capability probe timed out"))), + None => Err(Error::other("ILM recovery export fleet capability notification system is unavailable")), + } + }; + let (result, fence_probe, recovery_export_result) = + tokio::join!(remote_version_state_probe, cross_pool_fence_probe, recovery_export_probe); let (fence_result, journal_result, decommission_target_fence_result, reconcile_result) = match fence_probe { Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version), Err(err) => { @@ -1004,6 +1156,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot()); revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot()); revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot()); + revoke_fleet_capability_proof(ilm_recovery_export_fleet_proof_slot()); } else if let Some(err) = publish_fleet_capability_probe_result( remote_version_state_fleet_proof_slot(), &topology_fingerprint, @@ -1030,6 +1183,24 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) { "notification capability probe" ); } + if !topology_conflict + && let Some(err) = publish_fleet_capability_probe_result( + ilm_recovery_export_fleet_proof_slot(), + &topology_fingerprint, + recovery_export_result, + Instant::now(), + ) + { + debug!( + event = EVENT_NOTIFICATION_CAPABILITY_PROBE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_NOTIFICATION, + capability = "ilm_recovery_export_v1", + state = "failed_closed", + error = %err, + "notification capability probe" + ); + } if !topology_conflict && let Some(err) = publish_fleet_capability_probe_result( tier_delete_journal_fleet_proof_slot(), @@ -1174,6 +1345,46 @@ impl NotificationSys { } Ok((peer_epochs, minimum_version)) } + + async fn probe_ilm_recovery_export_fleet(&self, topology_fingerprint: &str) -> Result> { + if self.peer_clients.len() != self.peer_topology_hosts.len() { + return Err(Error::other("ILM recovery export capability fleet membership is incomplete")); + } + let local_member = runtime_sources::local_node_name().await; + if local_member.trim().is_empty() { + return Err(Error::other("ILM recovery export local member identity is unavailable")); + } + let mut peer_epochs = BTreeMap::new(); + insert_remote_version_state_peer(&mut peer_epochs, local_member.clone(), ilm_recovery_export_local_process_epoch())?; + let probes = self.peer_clients.iter().map(|client| async { + let client = client + .as_ref() + .ok_or_else(|| Error::other("ILM recovery export capability peer is unreachable"))?; + client.probe_ilm_recovery_export(topology_fingerprint.to_string()).await + }); + for result in join_all(probes).await { + let (peer, epoch) = result?; + insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?; + } + validate_ilm_recovery_export_members(&self.peer_topology_hosts, &local_member, &peer_epochs)?; + Ok(peer_epochs) + } +} + +fn validate_ilm_recovery_export_members( + expected_remote_members: &[String], + local_member: &str, + observed: &BTreeMap, +) -> Result<()> { + let expected = expected_remote_members + .iter() + .cloned() + .chain(std::iter::once(local_member.to_string())) + .collect::>(); + if expected.len() != expected_remote_members.len().saturating_add(1) || observed.keys().ne(expected.iter()) { + return Err(Error::other("ILM recovery export capability fleet membership does not match topology")); + } + Ok(()) } /// Rolling tier activity summed over every cluster member that answered, with @@ -3507,6 +3718,81 @@ mod tests { assert!(captured != restarted.token()); } + #[test] + fn ilm_recovery_export_member_digest_is_order_independent_and_epoch_bound() { + let now = Instant::now(); + let local_epoch = ilm_recovery_export_local_process_epoch(); + assert!(!local_epoch.is_nil()); + assert_eq!(local_epoch, ilm_recovery_export_local_process_epoch()); + let remote_epoch = Uuid::new_v4(); + let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); + let peers = BTreeMap::from([("node-b".to_string(), remote_epoch), ("node-a".to_string(), local_epoch)]); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none()); + let proof = { + let state = slot.read().expect("export proof slot should not poison"); + acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).expect("complete fleet should admit export") + }; + let digest = ilm_recovery_export_member_epochs_sha256(&proof); + + let changed_slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); + let changed = BTreeMap::from([("node-a".to_string(), local_epoch), ("node-b".to_string(), Uuid::new_v4())]); + assert!(publish_fleet_capability_probe_result(&changed_slot, "topology-a", Ok(changed), now).is_none()); + let changed_proof = { + let state = changed_slot.read().expect("export proof slot should not poison"); + acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).expect("complete fleet should admit export") + }; + assert_ne!(digest, ilm_recovery_export_member_epochs_sha256(&changed_proof)); + } + + #[test] + fn ilm_recovery_export_members_must_match_the_exact_topology() { + let expected_remote = vec!["node-b".to_string()]; + let local = "node-a"; + let complete = BTreeMap::from([ + (local.to_string(), Uuid::new_v4()), + (expected_remote[0].clone(), Uuid::new_v4()), + ]); + assert!(validate_ilm_recovery_export_members(&expected_remote, local, &complete).is_ok()); + + let unexpected = BTreeMap::from([(local.to_string(), Uuid::new_v4()), ("node-c".to_string(), Uuid::new_v4())]); + assert!(validate_ilm_recovery_export_members(&expected_remote, local, &unexpected).is_err()); + assert!( + validate_ilm_recovery_export_members(&[local.to_string()], local, &complete).is_err(), + "the configured remote set cannot repeat the local member" + ); + } + + #[test] + fn ilm_recovery_export_restart_revokes_authority_until_permit_drains() { + let slot = std::sync::RwLock::new(FleetCapabilityProofState::default()); + let now = Instant::now(); + let original = BTreeMap::from([("node-a".to_string(), Uuid::new_v4())]); + assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original), now).is_none()); + let admitted = { + let state = slot.read().expect("export proof slot should not poison"); + acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).expect("fresh fleet should admit export") + }; + + let restarted = BTreeMap::from([("node-a".to_string(), Uuid::new_v4())]); + let draining = publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted.clone()), now) + .expect("restart must wait for the admitted export effect window"); + assert!(draining.to_string().contains("previous generation to drain")); + { + let state = slot.read().expect("export proof slot should not poison"); + assert!(!ilm_recovery_export_fleet_proof_matches_at(&state, &admitted, "topology-a", now)); + assert!( + acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).is_none(), + "successor authority must wait for the old effect window to drain" + ); + } + drop(admitted); + assert!( + publish_fleet_capability_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(1)).is_none() + ); + let state = slot.read().expect("export proof slot should not poison"); + assert!(acquire_ilm_recovery_export_fleet_proof_from(&state, "topology-a", now).is_some()); + } + #[test] fn tier_delete_journal_generation_is_stable_across_members_and_process_restarts() { let topology = "topology-a"; diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 86662f182..6ec5c3c00 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -830,6 +830,10 @@ mod tests { IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, list_recovery_controls, load_recovery_control, observe_recovery_source, save_recovery_control_if_absent, }, + recovery_export::{ + create_recovery_export, inspect_recovery_export_observation, load_recovery_export, + recovery_export_record_object_name, + }, tier_delete_journal::{ DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX, TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard, @@ -16870,6 +16874,43 @@ mod tests { } } + let creator_sha256 = rustfs_utils::crypto::hex_sha256(b"legacy-export-actor", ToOwned::to_owned); + let mut created_exports = Vec::new(); + for exportable in first_controls + .iter() + .filter(|control| control.classification == IlmRecoveryClassification::RetainedAmbiguous) + { + let observation = inspect_recovery_export_observation(store.clone(), &exportable.control_id) + .await + .expect("fresh legacy recovery observation should be exportable"); + let created = create_recovery_export(store.clone(), &observation, &creator_sha256) + .await + .expect("legacy recovery export should be created exactly once"); + assert!(!created.replayed); + let loaded = load_recovery_export(store.clone(), &created.export_id) + .await + .expect("created legacy recovery export should load"); + assert_eq!(loaded.encoded, created.encoded, "export readback must preserve the exact committed bytes"); + let replayed = create_recovery_export(store.clone(), &observation, &creator_sha256) + .await + .expect("the same observed generation should replay its immutable export"); + assert!(replayed.replayed); + assert_eq!(replayed.encoded, created.encoded); + created_exports.push(created); + } + assert_eq!(created_exports.len(), 2, "both v1 and v2 legacy journals must have an export path"); + + let corrupt_export_id = &created_exports[0].export_id; + let export_path = recovery_export_record_object_name(IlmRecoveryProtocol::TierDeleteJournal, corrupt_export_id) + .expect("export path should build"); + com::save_config(store.clone(), &export_path, Vec::new()) + .await + .expect("zero-byte corruption fixture should persist"); + let corrupt_export = load_recovery_export(store.clone(), corrupt_export_id) + .await + .expect_err("an existing zero-byte export must fail closed"); + assert!(!matches!(corrupt_export, Error::ConfigNotFound)); + com::save_config( store.clone(), &journal_paths[0], diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 81ead7d65..47b7811e1 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -175,6 +175,7 @@ pub const BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION: u32 = 2; pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v3\0"; pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0"; pub const CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-cross-pool-fence-capability-v1\0"; +pub const ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-ilm-recovery-export-capability-v1\0"; pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024; pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024; pub const TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE; @@ -219,6 +220,18 @@ pub fn is_cross_pool_fence_capability_probe(command: &[u8]) -> bool { && command.starts_with(CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX) } +pub fn ilm_recovery_export_capability_probe(nonce: &[u8; 16]) -> Vec { + let mut probe = Vec::with_capacity(ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX.len() + nonce.len()); + probe.extend_from_slice(ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX); + probe.extend_from_slice(nonce); + probe +} + +pub fn is_ilm_recovery_export_capability_probe(command: &[u8]) -> bool { + command.len() == ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX.len() + 16 + && command.starts_with(ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX) +} + pub fn encode_remote_version_state_capability( topology_member: &str, process_epoch: &[u8; 16], @@ -2127,12 +2140,13 @@ mod scanner_activity_tests { mod heal_control_tests { use super::{ CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, - REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX, canonical_heal_control_capability_ack, canonical_heal_control_request_body, - canonical_heal_control_response_body, decode_remote_version_state_capability, encode_cross_pool_fence_capability, - encode_remote_version_state_capability, heal_control_capability_probe, heal_control_coordinator_epoch, - heal_control_execution_timeout, heal_control_execution_timeout_for, internode_rpc_timeout, - is_cross_pool_fence_capability_probe, is_heal_control_capability_probe, is_remote_version_state_capability_probe, - normalize_internode_rpc_timeout, remote_version_state_capability_probe, + ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX, REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX, + canonical_heal_control_capability_ack, canonical_heal_control_request_body, canonical_heal_control_response_body, + decode_remote_version_state_capability, encode_cross_pool_fence_capability, encode_remote_version_state_capability, + heal_control_capability_probe, heal_control_coordinator_epoch, heal_control_execution_timeout, + heal_control_execution_timeout_for, ilm_recovery_export_capability_probe, internode_rpc_timeout, + is_cross_pool_fence_capability_probe, is_heal_control_capability_probe, is_ilm_recovery_export_capability_probe, + is_remote_version_state_capability_probe, normalize_internode_rpc_timeout, remote_version_state_capability_probe, }; use crate::heal_control; use std::time::Duration; @@ -2196,6 +2210,19 @@ mod heal_control_tests { assert!(!is_remote_version_state_capability_probe(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX)); } + #[test] + fn ilm_recovery_export_capability_probe_requires_exact_prefix_and_nonce() { + let probe = ilm_recovery_export_capability_probe(&[7; 16]); + assert!(is_ilm_recovery_export_capability_probe(&probe)); + assert!(!is_ilm_recovery_export_capability_probe(ILM_RECOVERY_EXPORT_CAPABILITY_PROBE_PREFIX)); + let mut wrong_prefix = probe.clone(); + wrong_prefix[0] ^= 1; + assert!(!is_ilm_recovery_export_capability_probe(&wrong_prefix)); + let mut extra = probe; + extra.push(0); + assert!(!is_ilm_recovery_export_capability_probe(&extra)); + } + #[test] fn remote_version_state_capability_binds_member_and_process_epoch() { let encoded = diff --git a/rustfs/src/admin/handlers/ilm_transition.rs b/rustfs/src/admin/handlers/ilm_transition.rs index 74788d0d9..d511470d0 100644 --- a/rustfs/src/admin/handlers/ilm_transition.rs +++ b/rustfs/src/admin/handlers/ilm_transition.rs @@ -14,18 +14,19 @@ use crate::admin::auth::authorize_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; -use crate::admin::runtime_sources::object_store_from_extensions; +use crate::admin::runtime_sources::{current_action_credentials, object_store_from_extensions}; use crate::admin::storage_api::bucket::is_reserved_or_invalid_bucket; use crate::admin::storage_api::error::StorageError; use crate::admin::storage_api::lifecycle::{ - IlmRecoveryClassification, IlmRecoveryProtocol, ManualTransitionCancelCheck, ManualTransitionJobRecord, - ManualTransitionJobState, ManualTransitionProgressSink, ManualTransitionQueueSnapshot, ManualTransitionRunOptions, - ManualTransitionRunReport, ManualTransitionScopeAdmission, ManualTransitionScopeAdmissionClaim, - TransitionOperatorDeleteResult, TransitionOperatorError, claim_manual_transition_scope_admission, - delete_manual_transition_scope_admission_if_current, delete_transition_candidate_for_operator, - enqueue_transition_for_existing_objects_scoped, finalize_missing_transition_transaction_for_operator, - inspect_recovery_control, inspect_transition_transaction_for_operator, list_recovery_controls, - load_manual_transition_job_record, load_manual_transition_scope_admission, manual_transition_job_lease_expired, + IlmRecoveryClassification, IlmRecoveryControlView, IlmRecoveryExportObservation, IlmRecoveryProtocol, + ManualTransitionCancelCheck, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink, + ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission, + ManualTransitionScopeAdmissionClaim, TransitionOperatorDeleteResult, TransitionOperatorError, + claim_manual_transition_scope_admission, create_recovery_export, delete_manual_transition_scope_admission_if_current, + delete_transition_candidate_for_operator, enqueue_transition_for_existing_objects_scoped, + finalize_missing_transition_transaction_for_operator, inspect_recovery_control, inspect_recovery_export_observation, + inspect_transition_transaction_for_operator, list_recovery_controls, load_manual_transition_job_record, + load_manual_transition_scope_admission, load_recovery_export, manual_transition_job_lease_expired, manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired, persist_manual_transition_job_progress_if_owned, renew_manual_transition_job_lease_if_owned, request_manual_transition_job_cancel, save_manual_transition_job_record, update_manual_transition_job_record, @@ -34,21 +35,30 @@ use crate::admin::storage_api::runtime::ECStore; use crate::admin::storage_api::s3::{S3ErrorCode as AdminS3ErrorCode, error as admin_s3_error}; use crate::admin::utils::json_response; use crate::server::{ADMIN_PREFIX, RemoteAddr}; -use http::HeaderMap; +use aes_gcm::{ + Aes256Gcm, Key, Nonce, + aead::{Aead, KeyInit}, +}; +use http::{HeaderMap, HeaderValue, header}; use hyper::{Method, StatusCode}; use matchit::Params; +use rand::RngExt; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; +use rustfs_credentials::Credentials; use rustfs_policy::policy::action::{Action, AdminAction}; use rustfs_utils::{ - MaskedAccessKey, + MaskedAccessKey, base64_decode_url_safe_no_pad, base64_encode_url_safe_no_pad, + crypto::hex_sha256, http::{AMZ_REQUEST_ID, REQUEST_ID_HEADER}, }; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::pin::Pin; use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; use std::time::Duration as StdDuration; +use time::{Duration, OffsetDateTime}; use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; use uuid::Uuid; @@ -60,6 +70,8 @@ const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_SUBSYSTEM_ILM_TRANSITION: &str = "ilm_transition"; const EVENT_ADMIN_ILM_TRANSITION_STATE: &str = "admin_ilm_transition_state"; const EVENT_ADMIN_ILM_TRANSITION_RECONCILE: &str = "admin_ilm_transition_reconcile"; +const ILM_RECOVERY_OBSERVATION_RECEIPT_TTL: Duration = Duration::minutes(15); +const MAX_ILM_RECOVERY_RECEIPT_SIZE: usize = 32 * 1024; static ACTIVE_MANUAL_TRANSITION_SCOPES: OnceLock>> = OnceLock::new(); #[cfg(feature = "e2e-test-hooks")] @@ -242,6 +254,16 @@ pub fn register_ilm_transition_route(r: &mut S3Router) -> std::i format!("{ADMIN_PREFIX}/v3/ilm/recovery/records/{{control_id}}").as_str(), AdminOperation(&IlmRecoveryControlInspectHandler {}), )?; + r.insert( + Method::POST, + format!("{ADMIN_PREFIX}/v3/ilm/recovery/records/{{control_id}}").as_str(), + AdminOperation(&IlmRecoveryExportCreateHandler {}), + )?; + r.insert( + Method::GET, + format!("{ADMIN_PREFIX}/v3/ilm/recovery/exports/{{export_id}}").as_str(), + AdminOperation(&IlmRecoveryExportDownloadHandler {}), + )?; Ok(()) } @@ -459,6 +481,14 @@ async fn authorize_transition_admin_request(req: &S3Request, action: Admin Ok(actor) } +async fn authorize_recovery_admin_request(req: &S3Request, action: AdminAction) -> S3Result { + if req.credentials.is_none() { + return Err(admin_s3_error(AdminS3ErrorCode::InvalidRequest, "authentication required")); + } + let credentials = authorize_admin_request(req, vec![Action::AdminAction(action)]).await?; + Ok(recovery_actor_sha256(&credentials)) +} + fn transition_transaction_id_from_params(params: &Params<'_, '_>) -> S3Result { Uuid::parse_str(params.get("transaction_id").unwrap_or("")) .map_err(|_| s3_error!(InvalidArgument, "invalid transition transaction id")) @@ -466,14 +496,19 @@ fn transition_transaction_id_from_params(params: &Params<'_, '_>) -> S3Result) -> S3Result { let control_id = params.get("control_id").unwrap_or(""); - if control_id.len() != 64 - || !control_id + validate_recovery_sha256(control_id, "invalid ILM recovery control id")?; + Ok(control_id.to_string()) +} + +fn validate_recovery_sha256(value: &str, message: &'static str) -> S3Result<()> { + if value.len() != 64 + || !value .bytes() .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) { - return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid ILM recovery control id")); + return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, message)); } - Ok(control_id.to_string()) + Ok(()) } fn map_recovery_control_error(err: StorageError) -> S3Error { @@ -484,6 +519,394 @@ fn map_recovery_control_error(err: StorageError) -> S3Error { } } +fn recovery_export_id_from_params(params: &Params<'_, '_>) -> S3Result { + let export_id = params.get("export_id").unwrap_or(""); + validate_recovery_sha256(export_id, "invalid ILM recovery export id")?; + Ok(export_id.to_string()) +} + +fn map_recovery_export_error(err: StorageError) -> S3Error { + if err == StorageError::ConfigNotFound { + admin_s3_error(AdminS3ErrorCode::NoSuchKey, "ILM recovery export not found") + } else if err == StorageError::SlowDown { + admin_s3_error(AdminS3ErrorCode::SlowDown, "ILM recovery export admission capacity is exhausted") + } else if err == StorageError::PreconditionFailed { + admin_s3_error(AdminS3ErrorCode::OperationAborted, "ILM recovery export observation is stale") + } else { + admin_s3_error(AdminS3ErrorCode::OperationAborted, "ILM recovery export request cannot proceed") + } +} + +fn recovery_export_download_headers(export_id: &str, encoded_len: usize) -> S3Result { + let mut headers = HeaderMap::new(); + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + headers.insert( + header::CONTENT_DISPOSITION, + HeaderValue::from_str(&format!("attachment; filename=\"ilm-recovery-export-{export_id}.json\"")) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InternalError, "invalid ILM recovery export filename"))?, + ); + headers.insert( + header::CONTENT_LENGTH, + HeaderValue::from_str(&encoded_len.to_string()) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InternalError, "invalid ILM recovery export length"))?, + ); + Ok(headers) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum IlmRecoveryReceiptAction { + Export, + AbandonRemoteCleanup, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum IlmRecoveryReceiptMode { + DryRun, + Execute, +} + +const fn default_recovery_receipt_mode() -> IlmRecoveryReceiptMode { + IlmRecoveryReceiptMode::Execute +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct IlmRecoveryObservationReceipt { + schema: String, + action: IlmRecoveryReceiptAction, + // Receipts issued before action modes were introduced represented the + // existing export execution path, so decode them as Execute until their + // fixed 15-minute lifetime elapses. + #[serde(default = "default_recovery_receipt_mode")] + mode: IlmRecoveryReceiptMode, + actor_sha256: String, + issued_at_unix_nanos: i64, + expires_at_unix_nanos: i64, + nonce: Uuid, + observation: IlmRecoveryExportObservation, +} + +#[derive(Debug, Serialize)] +struct IlmRecoveryControlInspectResponse { + #[serde(flatten)] + control: IlmRecoveryControlView, + export_ready: bool, + #[serde(skip_serializing_if = "Option::is_none")] + export_not_ready_reason: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + observation_receipt: Option, + #[serde(skip_serializing_if = "Option::is_none")] + observation_receipt_expires_at_unix_nanos: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum IlmRecoveryDispositionReasonCode { + LegacyRemoteCleanupAbandoned, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)] +enum IlmRecoveryRecordMutationRequest { + Export { + observation_receipt: String, + }, + AbandonRemoteCleanup { + mode: IlmRecoveryReceiptMode, + observation_receipt: String, + export_id: String, + export_sha256: String, + reason_code: IlmRecoveryDispositionReasonCode, + #[serde(default)] + confirm: Option, + #[serde(default)] + acknowledge_remote_cleanup_abandoned: Option, + }, +} + +fn parse_recovery_record_mutation_request(body: &[u8]) -> S3Result { + let value: serde_json::Value = serde_json::from_slice(body) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid ILM recovery request"))?; + let request: IlmRecoveryRecordMutationRequest = serde_json::from_value(value.clone()) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid ILM recovery request"))?; + if matches!( + &request, + IlmRecoveryRecordMutationRequest::AbandonRemoteCleanup { + mode: IlmRecoveryReceiptMode::DryRun, + .. + } + ) && value + .as_object() + .is_some_and(|object| object.contains_key("confirm") || object.contains_key("acknowledge_remote_cleanup_abandoned")) + { + return Err(admin_s3_error( + AdminS3ErrorCode::InvalidArgument, + "ILM recovery dry-run must not include terminal confirmation fields", + )); + } + Ok(request) +} + +#[allow(dead_code)] +enum ValidatedIlmRecoveryRecordMutation<'a> { + Export { + observation_receipt: &'a str, + }, + AbandonDryRun { + observation_receipt: &'a str, + export_id: &'a str, + export_sha256: &'a str, + reason_code: IlmRecoveryDispositionReasonCode, + }, + AbandonExecute { + observation_receipt: &'a str, + export_id: &'a str, + export_sha256: &'a str, + reason_code: IlmRecoveryDispositionReasonCode, + }, +} + +fn validate_recovery_record_mutation_request( + request: &IlmRecoveryRecordMutationRequest, +) -> S3Result> { + match request { + IlmRecoveryRecordMutationRequest::Export { observation_receipt } => { + Ok(ValidatedIlmRecoveryRecordMutation::Export { observation_receipt }) + } + IlmRecoveryRecordMutationRequest::AbandonRemoteCleanup { + mode, + observation_receipt, + export_id, + export_sha256, + reason_code, + confirm, + acknowledge_remote_cleanup_abandoned, + } => { + validate_recovery_sha256(export_id, "invalid ILM recovery export id")?; + validate_recovery_sha256(export_sha256, "invalid ILM recovery export checksum")?; + if observation_receipt.is_empty() { + return Err(admin_s3_error( + AdminS3ErrorCode::InvalidArgument, + "ILM recovery observation receipt must not be empty", + )); + } + match mode { + IlmRecoveryReceiptMode::DryRun if confirm.is_none() && acknowledge_remote_cleanup_abandoned.is_none() => { + Ok(ValidatedIlmRecoveryRecordMutation::AbandonDryRun { + observation_receipt, + export_id, + export_sha256, + reason_code: *reason_code, + }) + } + IlmRecoveryReceiptMode::DryRun => Err(admin_s3_error( + AdminS3ErrorCode::InvalidArgument, + "ILM recovery dry-run must not include terminal confirmation fields", + )), + IlmRecoveryReceiptMode::Execute + if *confirm == Some(true) && *acknowledge_remote_cleanup_abandoned == Some(true) => + { + Ok(ValidatedIlmRecoveryRecordMutation::AbandonExecute { + observation_receipt, + export_id, + export_sha256, + reason_code: *reason_code, + }) + } + IlmRecoveryReceiptMode::Execute => Err(admin_s3_error( + AdminS3ErrorCode::InvalidRequest, + "ILM recovery disposition requires confirm=true and acknowledge_remote_cleanup_abandoned=true", + )), + } + } + } +} + +#[derive(Debug, Serialize)] +struct IlmRecoveryExportCreateResponse { + export_id: String, + export_sha256: String, + download_url: String, + outcome: &'static str, +} + +// These response envelopes pin the future disposition wire contract before +// its storage state machine is connected to this handler. +#[allow(dead_code)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum IlmRecoveryDispositionDryRunStatus { + Ready, +} + +#[allow(dead_code)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum IlmRecoveryDispositionState { + Applying, + Completed, +} + +#[allow(dead_code)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum IlmRecoveryDispositionOutcome { + AcceptedForRecovery, + Completed, + Replayed, +} + +#[allow(dead_code)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct IlmRecoveryDispositionDryRunResponse { + action: IlmRecoveryReceiptAction, + mode: IlmRecoveryReceiptMode, + status: IlmRecoveryDispositionDryRunStatus, + disposition_id: String, + export_id: String, + export_sha256: String, + source_generation_sha256: String, + copy_set_sha256: String, + source_copy_count: usize, + observation_receipt: String, + observation_receipt_expires_at_unix_nanos: i64, +} + +#[allow(dead_code)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct IlmRecoveryDispositionExecuteResponse { + action: IlmRecoveryReceiptAction, + mode: IlmRecoveryReceiptMode, + disposition_id: String, + state: IlmRecoveryDispositionState, + outcome: IlmRecoveryDispositionOutcome, + confirmed_absent_copy_count: usize, + source_copy_count: usize, +} + +fn recovery_actor_sha256(credentials: &Credentials) -> String { + let access_key = &credentials.access_key; + let mut bound = Vec::with_capacity(access_key.len() + 40); + bound.extend_from_slice(b"rustfs-ilm-recovery-actor-v1\0"); + bound.extend_from_slice(access_key.as_bytes()); + hex_sha256(&bound, ToOwned::to_owned) +} + +fn recovery_receipt_credentials() -> S3Result { + current_action_credentials() + .filter(|credentials| !credentials.secret_key.is_empty()) + .ok_or_else(|| admin_s3_error(AdminS3ErrorCode::InternalError, "ILM recovery receipt key is unavailable")) +} + +fn recovery_receipt_key(credentials: &Credentials) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(b"rustfs-ilm-recovery-observation-receipt-v1\0"); + hasher.update(credentials.secret_key.as_bytes()); + hasher.finalize().into() +} + +fn encode_recovery_receipt(payload: &IlmRecoveryObservationReceipt, credentials: &Credentials) -> S3Result { + let payload = serde_json::to_vec(payload) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InternalError, "failed to encode ILM recovery receipt"))?; + if payload.len() > MAX_ILM_RECOVERY_RECEIPT_SIZE { + return Err(admin_s3_error( + AdminS3ErrorCode::InternalError, + "ILM recovery receipt exceeds maximum size", + )); + } + let cipher = Aes256Gcm::new(&Key::::from(recovery_receipt_key(credentials))); + let mut nonce_bytes = [0_u8; 12]; + rand::rng().fill(&mut nonce_bytes); + let ciphertext = cipher + .encrypt(&Nonce::from(nonce_bytes), payload.as_slice()) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InternalError, "failed to seal ILM recovery receipt"))?; + Ok(format!( + "{}.{}", + base64_encode_url_safe_no_pad(&nonce_bytes), + base64_encode_url_safe_no_pad(&ciphertext) + )) +} + +fn decode_recovery_receipt(token: &str, credentials: &Credentials) -> S3Result { + if token.len() > MAX_ILM_RECOVERY_RECEIPT_SIZE * 2 { + return Err(admin_s3_error(AdminS3ErrorCode::AccessDenied, "invalid or expired ILM recovery receipt")); + } + let Some((nonce, ciphertext)) = token.split_once('.') else { + return Err(admin_s3_error(AdminS3ErrorCode::AccessDenied, "invalid or expired ILM recovery receipt")); + }; + if ciphertext.contains('.') { + return Err(admin_s3_error(AdminS3ErrorCode::AccessDenied, "invalid or expired ILM recovery receipt")); + } + let nonce = base64_decode_url_safe_no_pad(nonce.as_bytes()) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::AccessDenied, "invalid or expired ILM recovery receipt"))?; + let nonce: [u8; 12] = nonce + .try_into() + .map_err(|_| admin_s3_error(AdminS3ErrorCode::AccessDenied, "invalid or expired ILM recovery receipt"))?; + let ciphertext = base64_decode_url_safe_no_pad(ciphertext.as_bytes()) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::AccessDenied, "invalid or expired ILM recovery receipt"))?; + let cipher = Aes256Gcm::new(&Key::::from(recovery_receipt_key(credentials))); + let plaintext = cipher + .decrypt(&Nonce::from(nonce), ciphertext.as_slice()) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::AccessDenied, "invalid or expired ILM recovery receipt"))?; + serde_json::from_slice(&plaintext) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::AccessDenied, "invalid or expired ILM recovery receipt")) +} + +fn issue_recovery_observation_receipt( + observation: IlmRecoveryExportObservation, + actor_sha256: String, + action: IlmRecoveryReceiptAction, + mode: IlmRecoveryReceiptMode, + now: OffsetDateTime, +) -> S3Result<(String, i64)> { + let expires_at = now + ILM_RECOVERY_OBSERVATION_RECEIPT_TTL; + let receipt = IlmRecoveryObservationReceipt { + schema: "rustfs-ilm-recovery-observation-receipt-v1".to_string(), + action, + mode, + actor_sha256, + issued_at_unix_nanos: i64::try_from(now.unix_timestamp_nanos()) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InternalError, "ILM recovery receipt timestamp is invalid"))?, + expires_at_unix_nanos: i64::try_from(expires_at.unix_timestamp_nanos()) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InternalError, "ILM recovery receipt timestamp is invalid"))?, + nonce: Uuid::new_v4(), + observation, + }; + let token = encode_recovery_receipt(&receipt, &recovery_receipt_credentials()?)?; + Ok((token, receipt.expires_at_unix_nanos)) +} + +fn validate_recovery_observation_receipt( + receipt: IlmRecoveryObservationReceipt, + actor_sha256: &str, + control_id: &str, + expected_action: IlmRecoveryReceiptAction, + expected_mode: IlmRecoveryReceiptMode, + now_unix_nanos: i64, +) -> S3Result { + let ttl_nanos = i64::try_from(ILM_RECOVERY_OBSERVATION_RECEIPT_TTL.whole_nanoseconds()) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InternalError, "ILM recovery receipt TTL is invalid"))?; + if receipt.schema != "rustfs-ilm-recovery-observation-receipt-v1" + || receipt.action != expected_action + || receipt.mode != expected_mode + || receipt.actor_sha256 != actor_sha256 + || receipt.observation.control_id != control_id + || receipt.nonce.is_nil() + || receipt.issued_at_unix_nanos <= 0 + || receipt.issued_at_unix_nanos > now_unix_nanos + || receipt.expires_at_unix_nanos <= now_unix_nanos + || receipt.expires_at_unix_nanos.checked_sub(receipt.issued_at_unix_nanos) != Some(ttl_nanos) + { + return Err(admin_s3_error(AdminS3ErrorCode::AccessDenied, "invalid or expired ILM recovery receipt")); + } + Ok(receipt.observation) +} + fn map_transition_operator_error(err: TransitionOperatorError) -> S3Error { match err { TransitionOperatorError::NotFound => s3_error!(NoSuchKey, "transition transaction not found"), @@ -1114,15 +1537,100 @@ pub struct IlmRecoveryControlInspectHandler {} #[async_trait::async_trait] impl Operation for IlmRecoveryControlInspectHandler { async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { - authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?; + let actor_sha256 = authorize_recovery_admin_request(&req, AdminAction::ListTierAction).await?; let control_id = recovery_control_id_from_params(¶ms)?; let Some(store) = object_store_from_extensions(&req.extensions) else { return Err(admin_s3_error(AdminS3ErrorCode::InternalError, "object store is not initialized")); }; - let control = inspect_recovery_control(store, &control_id) + let control = inspect_recovery_control(store.clone(), &control_id) .await .map_err(map_recovery_control_error)?; - json_response(StatusCode::OK, &control) + let now = OffsetDateTime::now_utc(); + let (export_ready, export_not_ready_reason, observation_receipt, expires_at) = + match inspect_recovery_export_observation(store, &control_id).await { + Ok(observation) => match issue_recovery_observation_receipt( + observation, + actor_sha256, + IlmRecoveryReceiptAction::Export, + IlmRecoveryReceiptMode::Execute, + now, + ) { + Ok((token, expires_at)) => (true, None, Some(token), Some(expires_at)), + Err(_) => (false, Some("receipt_key_unavailable"), None, None), + }, + Err(_) => (false, Some("fleet_or_source_not_ready"), None, None), + }; + json_response( + StatusCode::OK, + &IlmRecoveryControlInspectResponse { + control, + export_ready, + export_not_ready_reason, + observation_receipt, + observation_receipt_expires_at_unix_nanos: expires_at, + }, + ) + } +} + +pub struct IlmRecoveryExportCreateHandler {} + +#[async_trait::async_trait] +impl Operation for IlmRecoveryExportCreateHandler { + async fn call(&self, mut req: S3Request, params: Params<'_, '_>) -> S3Result> { + let actor_sha256 = authorize_recovery_admin_request(&req, AdminAction::SetTierAction).await?; + let control_id = recovery_control_id_from_params(¶ms)?; + let Some(store) = object_store_from_extensions(&req.extensions) else { + return Err(admin_s3_error(AdminS3ErrorCode::InternalError, "object store is not initialized")); + }; + let body = req.input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await.map_err(|_| { + admin_s3_error(AdminS3ErrorCode::InvalidRequest, "ILM recovery export body is too large or unreadable") + })?; + let request = parse_recovery_record_mutation_request(&body)?; + let ValidatedIlmRecoveryRecordMutation::Export { observation_receipt } = + validate_recovery_record_mutation_request(&request)? + else { + return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "unsupported ILM recovery action")); + }; + let receipt = decode_recovery_receipt(observation_receipt, &recovery_receipt_credentials()?)?; + let now = i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos()) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InternalError, "ILM recovery receipt timestamp is invalid"))?; + let observation = validate_recovery_observation_receipt( + receipt, + &actor_sha256, + &control_id, + IlmRecoveryReceiptAction::Export, + IlmRecoveryReceiptMode::Execute, + now, + )?; + let created = create_recovery_export(store, &observation, &actor_sha256) + .await + .map_err(map_recovery_export_error)?; + let response = IlmRecoveryExportCreateResponse { + download_url: format!("{ADMIN_PREFIX}/v3/ilm/recovery/exports/{}", created.export_id), + outcome: if created.replayed { "replayed" } else { "created" }, + export_id: created.export_id, + export_sha256: created.content_sha256, + }; + json_response(StatusCode::OK, &response) + } +} + +pub struct IlmRecoveryExportDownloadHandler {} + +#[async_trait::async_trait] +impl Operation for IlmRecoveryExportDownloadHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + authorize_transition_admin_request(&req, AdminAction::SetTierAction).await?; + let export_id = recovery_export_id_from_params(¶ms)?; + let Some(store) = object_store_from_extensions(&req.extensions) else { + return Err(admin_s3_error(AdminS3ErrorCode::InternalError, "object store is not initialized")); + }; + let export = load_recovery_export(store, &export_id) + .await + .map_err(map_recovery_export_error)?; + let headers = recovery_export_download_headers(&export_id, export.encoded.len())?; + Ok(S3Response::with_headers((StatusCode::OK, Body::from(export.encoded)), headers)) } } @@ -1242,6 +1750,350 @@ mod tests { }); } + #[test] + fn recovery_observation_receipt_is_opaque_actor_bound_and_tamper_evident() { + assert_eq!(ILM_RECOVERY_OBSERVATION_RECEIPT_TTL.whole_seconds(), 900); + let control_id = "ab".repeat(32); + let content_sha256 = hex_sha256(b"legacy", ToOwned::to_owned); + let copy_set_sha256 = hex_sha256( + &serde_json::to_vec(&serde_json::json!([{ + "authority": "pool-0/set-0", + "canonical_path": "ilm/tier-delete-journal/legacy.json", + "etag": "etag-a", + "encoded_len": 6, + "content_sha256": content_sha256, + }])) + .unwrap(), + ToOwned::to_owned, + ); + let observation: IlmRecoveryExportObservation = serde_json::from_value(serde_json::json!({ + "control_id": control_id, + "protocol": "tier_delete_journal", + "control_etag": "control-etag", + "control_revision": 1, + "classification": "retained_ambiguous", + "canonical_source_path": "ilm/tier-delete-journal/legacy.json", + "source_generation": { + "source_schema": "rustfs-tier-delete-journal-v1", + "source_etag": "etag-a", + "content_sha256": content_sha256, + "copy_set_sha256": copy_set_sha256, + "copies": [{ + "authority": "pool-0/set-0", + "canonical_path": "ilm/tier-delete-journal/legacy.json", + "etag": "etag-a", + "encoded_len": 6, + "content_sha256": content_sha256, + }] + }, + "topology_generation": hex_sha256(b"topology", ToOwned::to_owned), + "member_epochs_sha256": hex_sha256(b"members", ToOwned::to_owned), + })) + .unwrap(); + let payload = IlmRecoveryObservationReceipt { + schema: "rustfs-ilm-recovery-observation-receipt-v1".to_string(), + action: IlmRecoveryReceiptAction::Export, + mode: IlmRecoveryReceiptMode::Execute, + actor_sha256: hex_sha256(b"actor-a", ToOwned::to_owned), + issued_at_unix_nanos: 1, + expires_at_unix_nanos: 1 + ILM_RECOVERY_OBSERVATION_RECEIPT_TTL.whole_nanoseconds() as i64, + nonce: Uuid::new_v4(), + observation, + }; + let credentials = Credentials { + access_key: "root".to_string(), + secret_key: "secret".to_string(), + ..Default::default() + }; + let token = encode_recovery_receipt(&payload, &credentials).unwrap(); + assert!(!token.contains("actor-a")); + assert!(!token.contains("ilm/tier-delete-journal")); + assert_eq!(decode_recovery_receipt(&token, &credentials).unwrap(), payload); + assert!( + validate_recovery_observation_receipt( + payload.clone(), + &payload.actor_sha256, + &payload.observation.control_id, + IlmRecoveryReceiptAction::Export, + IlmRecoveryReceiptMode::Execute, + payload.issued_at_unix_nanos, + ) + .is_ok() + ); + let assert_denied = |receipt: IlmRecoveryObservationReceipt, actor: &str, control: &str, now: i64| { + let err = validate_recovery_observation_receipt( + receipt, + actor, + control, + IlmRecoveryReceiptAction::Export, + IlmRecoveryReceiptMode::Execute, + now, + ) + .expect_err("invalid observation receipt must be denied"); + assert_eq!(err.code(), &S3ErrorCode::AccessDenied); + }; + assert_denied( + payload.clone(), + &hex_sha256(b"actor-b", ToOwned::to_owned), + &payload.observation.control_id, + payload.issued_at_unix_nanos, + ); + assert_denied(payload.clone(), &payload.actor_sha256, &"cd".repeat(32), payload.issued_at_unix_nanos); + assert_denied( + payload.clone(), + &payload.actor_sha256, + &payload.observation.control_id, + payload.expires_at_unix_nanos, + ); + + let mut invalid = payload.clone(); + invalid.schema = "rustfs-ilm-recovery-observation-receipt-v2".to_string(); + assert_denied( + invalid, + &payload.actor_sha256, + &payload.observation.control_id, + payload.issued_at_unix_nanos, + ); + let mut invalid = payload.clone(); + invalid.action = IlmRecoveryReceiptAction::AbandonRemoteCleanup; + assert_denied( + invalid, + &payload.actor_sha256, + &payload.observation.control_id, + payload.issued_at_unix_nanos, + ); + let mut invalid = payload.clone(); + invalid.mode = IlmRecoveryReceiptMode::DryRun; + assert_denied( + invalid, + &payload.actor_sha256, + &payload.observation.control_id, + payload.issued_at_unix_nanos, + ); + let mut invalid = payload.clone(); + invalid.nonce = Uuid::nil(); + assert_denied( + invalid, + &payload.actor_sha256, + &payload.observation.control_id, + payload.issued_at_unix_nanos, + ); + let mut invalid = payload.clone(); + invalid.issued_at_unix_nanos += 1; + invalid.expires_at_unix_nanos += 1; + assert_denied( + invalid, + &payload.actor_sha256, + &payload.observation.control_id, + payload.issued_at_unix_nanos, + ); + let mut invalid = payload.clone(); + invalid.expires_at_unix_nanos += 1; + assert_denied( + invalid, + &payload.actor_sha256, + &payload.observation.control_id, + payload.issued_at_unix_nanos, + ); + + let mut tampered = token.into_bytes(); + let last = tampered.last_mut().unwrap(); + *last = if *last == b'a' { b'b' } else { b'a' }; + let err = decode_recovery_receipt(std::str::from_utf8(&tampered).unwrap(), &credentials) + .expect_err("tampered receipt must be denied"); + assert_eq!(err.code(), &S3ErrorCode::AccessDenied); + + let mut legacy_payload = serde_json::to_value(&payload).unwrap(); + legacy_payload.as_object_mut().unwrap().remove("mode"); + let legacy_payload: IlmRecoveryObservationReceipt = serde_json::from_value(legacy_payload).unwrap(); + assert_eq!(legacy_payload.mode, IlmRecoveryReceiptMode::Execute); + } + + #[test] + fn recovery_actor_binding_uses_the_authenticated_presented_access_key() { + let first = Credentials { + access_key: "operator-a".to_string(), + secret_key: "first-secret".to_string(), + ..Default::default() + }; + let same_actor_rotated_secret = Credentials { + access_key: first.access_key.clone(), + secret_key: "rotated-secret".to_string(), + ..Default::default() + }; + let other = Credentials { + access_key: "operator-b".to_string(), + secret_key: first.secret_key.clone(), + ..Default::default() + }; + + let actor = recovery_actor_sha256(&first); + assert_eq!(actor, recovery_actor_sha256(&same_actor_rotated_secret)); + assert_ne!(actor, recovery_actor_sha256(&other)); + assert!(!actor.contains(&first.access_key)); + + let production = include_str!("ilm_transition.rs") + .split("\n#[cfg(test)]\n") + .next() + .expect("production source must precede tests"); + let gate = extract_block_between_markers( + production, + "async fn authorize_recovery_admin_request", + "fn transition_transaction_id_from_params", + ); + assert!(gate.contains("let credentials = authorize_admin_request(")); + assert!(gate.contains("recovery_actor_sha256(&credentials)")); + assert!(!gate.contains("MaskedAccessKey")); + } + + #[test] + fn recovery_record_mutation_wire_contract_is_strict_and_mode_specific() { + let export = parse_recovery_record_mutation_request(br#"{"action":"export","observation_receipt":"opaque"}"#).unwrap(); + assert!(matches!( + validate_recovery_record_mutation_request(&export), + Ok(ValidatedIlmRecoveryRecordMutation::Export { + observation_receipt: "opaque" + }) + )); + + let export_id = "ab".repeat(32); + let export_sha256 = "cd".repeat(32); + let dry_run_json = format!( + r#"{{"action":"abandon_remote_cleanup","mode":"dry_run","observation_receipt":"opaque-dry-run","export_id":"{export_id}","export_sha256":"{export_sha256}","reason_code":"legacy_remote_cleanup_abandoned"}}"# + ); + let dry_run = parse_recovery_record_mutation_request(dry_run_json.as_bytes()).unwrap(); + assert!(matches!( + validate_recovery_record_mutation_request(&dry_run), + Ok(ValidatedIlmRecoveryRecordMutation::AbandonDryRun { + observation_receipt: "opaque-dry-run", + export_id: observed_export_id, + export_sha256: observed_export_sha256, + reason_code: IlmRecoveryDispositionReasonCode::LegacyRemoteCleanupAbandoned, + }) if observed_export_id == export_id && observed_export_sha256 == export_sha256 + )); + + let execute_json = format!( + r#"{{"action":"abandon_remote_cleanup","mode":"execute","confirm":true,"acknowledge_remote_cleanup_abandoned":true,"observation_receipt":"opaque-execute","export_id":"{export_id}","export_sha256":"{export_sha256}","reason_code":"legacy_remote_cleanup_abandoned"}}"# + ); + let execute = parse_recovery_record_mutation_request(execute_json.as_bytes()).unwrap(); + assert!(matches!( + validate_recovery_record_mutation_request(&execute), + Ok(ValidatedIlmRecoveryRecordMutation::AbandonExecute { + observation_receipt: "opaque-execute", + export_id: observed_export_id, + export_sha256: observed_export_sha256, + reason_code: IlmRecoveryDispositionReasonCode::LegacyRemoteCleanupAbandoned, + }) if observed_export_id == export_id && observed_export_sha256 == export_sha256 + )); + + let dry_run_with_confirmation = dry_run_json.replace(r#""mode":"dry_run""#, r#""mode":"dry_run","confirm":false"#); + assert!(parse_recovery_record_mutation_request(dry_run_with_confirmation.as_bytes()).is_err()); + assert!(parse_recovery_record_mutation_request(dry_run_json.replace('}', r#","confirm":null}"#).as_bytes()).is_err()); + + let uppercase_export_id = "AB".repeat(32); + for invalid in [ + execute_json.replace(r#""confirm":true,"#, ""), + execute_json.replace(r#""confirm":true"#, r#""confirm":false"#), + execute_json.replace( + r#""acknowledge_remote_cleanup_abandoned":true"#, + r#""acknowledge_remote_cleanup_abandoned":false"#, + ), + execute_json.replace(export_id.as_str(), uppercase_export_id.as_str()), + execute_json.replace(export_sha256.as_str(), "too-short"), + execute_json.replace("opaque-execute", ""), + ] { + if let Ok(request) = parse_recovery_record_mutation_request(invalid.as_bytes()) { + assert!( + validate_recovery_record_mutation_request(&request).is_err(), + "request should fail closed: {invalid}" + ); + } + } + + for invalid in [ + br#"{"action":"export","observation_receipt":"opaque","extra":true}"#.as_slice(), + br#"{"action":"abandon_remote_cleanup","mode":"preview"}"#.as_slice(), + br#"{"action":"unknown","observation_receipt":"opaque"}"#.as_slice(), + ] { + assert!(parse_recovery_record_mutation_request(invalid).is_err()); + } + } + + #[test] + fn recovery_disposition_response_wire_contract_is_closed() { + let export = IlmRecoveryExportCreateResponse { + export_id: "ab".repeat(32), + export_sha256: "cd".repeat(32), + download_url: "/rustfs/admin/v3/ilm/recovery/exports/export-id".to_string(), + outcome: "created", + }; + assert_eq!( + serde_json::to_value(&export).unwrap(), + serde_json::json!({ + "export_id": "ab".repeat(32), + "export_sha256": "cd".repeat(32), + "download_url": "/rustfs/admin/v3/ilm/recovery/exports/export-id", + "outcome": "created", + }) + ); + + let dry_run = IlmRecoveryDispositionDryRunResponse { + action: IlmRecoveryReceiptAction::AbandonRemoteCleanup, + mode: IlmRecoveryReceiptMode::DryRun, + status: IlmRecoveryDispositionDryRunStatus::Ready, + disposition_id: "ab".repeat(32), + export_id: "cd".repeat(32), + export_sha256: "ef".repeat(32), + source_generation_sha256: "12".repeat(32), + copy_set_sha256: "34".repeat(32), + source_copy_count: 2, + observation_receipt: "opaque-execute".to_string(), + observation_receipt_expires_at_unix_nanos: 900_000_000_001, + }; + let dry_run_json = serde_json::to_value(&dry_run).unwrap(); + assert_eq!(dry_run_json["action"], "abandon_remote_cleanup"); + assert_eq!(dry_run_json["mode"], "dry_run"); + assert_eq!(dry_run_json["status"], "ready"); + assert_eq!( + serde_json::from_value::(dry_run_json).unwrap(), + dry_run + ); + + let execute = IlmRecoveryDispositionExecuteResponse { + action: IlmRecoveryReceiptAction::AbandonRemoteCleanup, + mode: IlmRecoveryReceiptMode::Execute, + disposition_id: "ab".repeat(32), + state: IlmRecoveryDispositionState::Applying, + outcome: IlmRecoveryDispositionOutcome::AcceptedForRecovery, + confirmed_absent_copy_count: 1, + source_copy_count: 2, + }; + let execute_json = serde_json::to_value(&execute).unwrap(); + assert_eq!(execute_json["state"], "applying"); + assert_eq!(execute_json["outcome"], "accepted_for_recovery"); + assert_eq!( + serde_json::from_value::(execute_json).unwrap(), + execute + ); + + let mut unknown = serde_json::to_value(&dry_run).unwrap(); + unknown["unexpected"] = serde_json::json!(true); + assert!(serde_json::from_value::(unknown).is_err()); + } + + #[test] + fn recovery_export_download_headers_prevent_caching_and_force_attachment() { + let export_id = "ab".repeat(32); + let headers = recovery_export_download_headers(&export_id, 123).unwrap(); + assert_eq!(headers.get(header::CONTENT_TYPE).unwrap(), "application/json"); + assert_eq!(headers.get(header::CACHE_CONTROL).unwrap(), "no-store"); + assert_eq!(headers.get(header::CONTENT_LENGTH).unwrap(), "123"); + assert_eq!( + headers.get(header::CONTENT_DISPOSITION).unwrap(), + &format!("attachment; filename=\"ilm-recovery-export-{export_id}.json\"") + ); + } + fn manual_transition_job_request(method: Method, path: &'static str) -> S3Request { S3Request { input: Body::empty(), @@ -1628,6 +2480,19 @@ mod tests { assert_eq!(err.message(), Some("authentication required")); } + #[tokio::test] + async fn recovery_admin_gate_keeps_its_missing_credentials_response() { + let err = authorize_recovery_admin_request( + &manual_transition_job_request(Method::GET, "/rustfs/admin/v3/ilm/recovery/controls/control-123"), + AdminAction::ListTierAction, + ) + .await + .expect_err("a recovery admin request without credentials must fail"); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("authentication required")); + } + #[test] fn transition_admin_gate_routes_through_the_shared_gate() { let production = include_str!("ilm_transition.rs") diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 69c61768f..54d1715d6 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -507,6 +507,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ LIST_TIER, RouteRiskLevel::Sensitive, ), + admin( + HttpMethod::Post, + "/rustfs/admin/v3/ilm/recovery/records/{control_id}", + SET_TIER, + RouteRiskLevel::High, + ), + admin( + HttpMethod::Get, + "/rustfs/admin/v3/ilm/recovery/exports/{export_id}", + SET_TIER, + RouteRiskLevel::High, + ), admin(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SET_TIER, RouteRiskLevel::High), admin( HttpMethod::Get, @@ -2173,6 +2185,8 @@ mod tests { fn route_policy_uses_tier_actions_for_transition_routes() { assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/recovery/records", LIST_TIER); assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/recovery/records/{control_id}", LIST_TIER); + assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/recovery/records/{control_id}", SET_TIER); + assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/recovery/exports/{export_id}", SET_TIER); assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SET_TIER); assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER); assert_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER); diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index eec928968..8da3b43e2 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -215,6 +215,16 @@ fn expected_admin_route_matrix() -> Vec { "/v3/ilm/recovery/records/{control_id}", "/v3/ilm/recovery/records/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", ), + admin_route_sample( + Method::POST, + "/v3/ilm/recovery/records/{control_id}", + "/v3/ilm/recovery/records/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ), + admin_route_sample( + Method::GET, + "/v3/ilm/recovery/exports/{export_id}", + "/v3/ilm/recovery/exports/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + ), admin_route(Method::POST, "/v3/ilm/transition/run"), admin_route_sample( Method::GET, @@ -942,6 +952,16 @@ fn test_register_routes_cover_representative_admin_paths() { Method::GET, &admin_path("/v3/ilm/recovery/records/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), ); + assert_route( + &router, + Method::POST, + &admin_path("/v3/ilm/recovery/records/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ); + assert_route( + &router, + Method::GET, + &admin_path("/v3/ilm/recovery/exports/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ); assert_route(&router, Method::POST, &admin_path("/v3/ilm/transition/run")); assert_route( &router, diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 97e87c68c..c29159a47 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -233,7 +233,10 @@ pub(crate) mod lifecycle { super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunOptions; pub(crate) type ManualTransitionRunReport = super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunReport; pub(crate) use super::ecstore_bucket::lifecycle::recovery_control::{ - IlmRecoveryClassification, IlmRecoveryProtocol, inspect_recovery_control, list_recovery_controls, + IlmRecoveryClassification, IlmRecoveryControlView, IlmRecoveryProtocol, inspect_recovery_control, list_recovery_controls, + }; + pub(crate) use super::ecstore_bucket::lifecycle::recovery_export::{ + IlmRecoveryExportObservation, create_recovery_export, inspect_recovery_export_observation, load_recovery_export, }; pub(crate) use super::ecstore_bucket::lifecycle::transition_transaction::{ TransitionOperatorDeleteResult, TransitionOperatorError, delete_transition_candidate_for_operator, diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 25b200fb2..dd44f134d 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -187,6 +187,30 @@ static NODE_CAPABILITY_SERVER_EPOCH: LazyLock = LazyLock::new(Uuid::new_v4 // operations do not add another peer RPC. const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 4; +fn encode_heal_capability_response( + topology_member: &str, + remote_version_state_probe: bool, + recovery_export_probe: bool, +) -> Result, Status> { + if recovery_export_probe { + rustfs_protos::encode_remote_version_state_capability( + topology_member, + crate::storage::storage_api::ilm_recovery_export_local_process_epoch().as_bytes(), + ) + .map_err(|_| Status::internal("ILM recovery export capability length cannot be represented")) + } else if remote_version_state_probe { + rustfs_protos::encode_remote_version_state_capability(topology_member, NODE_CAPABILITY_SERVER_EPOCH.as_bytes()) + .map_err(|_| Status::internal("remote version state capability length cannot be represented")) + } else { + rustfs_protos::encode_cross_pool_fence_capability( + CROSS_POOL_FENCE_SUPPORTED_VERSION, + topology_member, + NODE_CAPABILITY_SERVER_EPOCH.as_bytes(), + ) + .map_err(|_| Status::internal("cross-pool fence capability length cannot be represented")) + } +} + fn admit_heal_control_replay( replay_cache: &mut HashMap>, request_id: &str, @@ -1003,7 +1027,8 @@ impl heal_control_service_server::HealControlService for HealControlRpcService { } let remote_version_state_probe = rustfs_protos::is_remote_version_state_capability_probe(&request.get_ref().command); let cross_pool_fence_probe = rustfs_protos::is_cross_pool_fence_capability_probe(&request.get_ref().command); - if remote_version_state_probe || cross_pool_fence_probe { + let recovery_export_probe = rustfs_protos::is_ilm_recovery_export_capability_probe(&request.get_ref().command); + if remote_version_state_probe || cross_pool_fence_probe || recovery_export_probe { let topology_member = self .endpoint_pools() .await @@ -1013,17 +1038,7 @@ impl heal_control_service_server::HealControlService for HealControlRpcService { if topology_member.is_empty() { return Err(Status::failed_precondition("local topology member identity is unavailable")); } - let result = if remote_version_state_probe { - rustfs_protos::encode_remote_version_state_capability(&topology_member, NODE_CAPABILITY_SERVER_EPOCH.as_bytes()) - .map_err(|_| Status::internal("remote version state capability length cannot be represented"))? - } else { - rustfs_protos::encode_cross_pool_fence_capability( - CROSS_POOL_FENCE_SUPPORTED_VERSION, - &topology_member, - NODE_CAPABILITY_SERVER_EPOCH.as_bytes(), - ) - .map_err(|_| Status::internal("cross-pool fence capability length cannot be represented"))? - }; + let result = encode_heal_capability_response(&topology_member, remote_version_state_probe, recovery_export_probe)?; let canonical_response = rustfs_protos::canonical_heal_control_response_body( request.get_ref().version, &request.get_ref().topology_fingerprint, @@ -4040,6 +4055,19 @@ mod tests { .expect_err("proof from one challenge must not be reusable"); } + #[test] + fn ilm_recovery_export_probe_uses_the_shared_local_process_epoch() { + let result = super::encode_heal_capability_response("node-a:9000", false, true) + .expect("ILM recovery export capability should encode"); + let (member, epoch) = + rustfs_protos::decode_remote_version_state_capability(&result).expect("ILM recovery export capability should decode"); + assert_eq!(member, "node-a:9000"); + assert_eq!( + Uuid::from_slice(epoch).expect("capability epoch should be a UUID"), + crate::storage::storage_api::ilm_recovery_export_local_process_epoch(), + ); + } + #[tokio::test] async fn cross_pool_fence_probe_authenticates_supported_v4_state() { let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string()); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index e35cf812d..886deb1cf 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -513,8 +513,8 @@ pub(crate) mod ecstore_notification { pub(crate) use rustfs_ecstore::api::notification::rotate_cross_pool_fence_fleet_proof_for_test; pub(crate) use rustfs_ecstore::api::notification::{ ClusterTierDailyStats, CrossPoolFenceFleetProofToken, NotificationSys, acquire_cross_pool_fence_fleet_proof, - cross_pool_fence_fleet_proof_matches, get_global_notification_sys, new_global_notification_sys, - start_remote_version_state_fleet_probe, + cross_pool_fence_fleet_proof_matches, get_global_notification_sys, ilm_recovery_export_local_process_epoch, + new_global_notification_sys, start_remote_version_state_fleet_probe, }; } @@ -1182,6 +1182,10 @@ pub(crate) fn start_remote_version_state_fleet_probe(topology_fingerprint: Strin ecstore_notification::start_remote_version_state_fleet_probe(topology_fingerprint); } +pub(crate) fn ilm_recovery_export_local_process_epoch() -> uuid::Uuid { + ecstore_notification::ilm_recovery_export_local_process_epoch() +} + pub(crate) async fn read_config(api: Arc, file: &str) -> Result> { ecstore_config::com::read_config(api, file).await }