diff --git a/.config/nextest.toml b/.config/nextest.toml index 097c79e3e..edf5b5df7 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -78,6 +78,12 @@ test-group = 'embedded-test-ports' filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)' test-group = 'ecstore-serial-flaky' +# The durable ILM decommission regressions build isolated multi-pool stores and +# deliberately take source or target disks offline while checking fencing. +[[profile.default.overrides]] +filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))' +test-group = 'ecstore-serial-flaky' + # Serialize the bucket-incarnation / lifecycle-fence tests. They drive # init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global # OnceLock state that serial_test's #[serial] cannot protect across nextest's @@ -190,6 +196,10 @@ test-group = 'embedded-test-ports' filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)' test-group = 'ecstore-serial-flaky' +[[profile.ci.overrides]] +filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))' +test-group = 'ecstore-serial-flaky' + # Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile # too (see the matching default-profile override near the top). No retries. [[profile.ci.overrides]] diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 1162787f7..48039a7c1 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -3739,17 +3739,55 @@ impl ManualTransitionRunReport { } pub fn merge_scan_report_preserving_worker(&mut self, scan_report: &ManualTransitionRunReport) { + let previous = self.clone(); + let resumed_after_checkpoint = previous.continuation_token.is_some() && scan_report.scanned < previous.scanned; let mut tier_failure_by_reason = self.tier_failure_by_reason.clone(); for (reason, count) in &scan_report.tier_failure_by_reason { let current = tier_failure_by_reason.get(reason).copied().unwrap_or_default(); - tier_failure_by_reason.insert(*reason, current.max(*count)); + let merged = if resumed_after_checkpoint { + current.saturating_add(*count) + } else { + current.max(*count) + }; + tier_failure_by_reason.insert(*reason, merged); } let transition_completed = self.transition_completed; let transition_failed = self.transition_failed; *self = scan_report.clone(); + if resumed_after_checkpoint { + self.scanned = previous.scanned.saturating_add(scan_report.scanned); + self.eligible = previous.eligible.saturating_add(scan_report.eligible); + self.enqueued = previous.enqueued.saturating_add(scan_report.enqueued); + self.dry_run_eligible = previous.dry_run_eligible.saturating_add(scan_report.dry_run_eligible); + self.skipped_not_transition = previous + .skipped_not_transition + .saturating_add(scan_report.skipped_not_transition); + self.skipped_tier = previous.skipped_tier.saturating_add(scan_report.skipped_tier); + self.skipped_delete_marker = previous + .skipped_delete_marker + .saturating_add(scan_report.skipped_delete_marker); + self.skipped_directory = previous.skipped_directory.saturating_add(scan_report.skipped_directory); + self.skipped_replication = previous.skipped_replication.saturating_add(scan_report.skipped_replication); + self.skipped_already_transitioned = previous + .skipped_already_transitioned + .saturating_add(scan_report.skipped_already_transitioned); + self.skipped_already_in_flight = previous + .skipped_already_in_flight + .saturating_add(scan_report.skipped_already_in_flight); + self.skipped_queue_full = previous.skipped_queue_full.saturating_add(scan_report.skipped_queue_full); + self.skipped_queue_closed = previous.skipped_queue_closed.saturating_add(scan_report.skipped_queue_closed); + self.skipped_queue_timeout = previous + .skipped_queue_timeout + .saturating_add(scan_report.skipped_queue_timeout); + self.tier_failure = previous.tier_failure.saturating_add(scan_report.tier_failure); + } + self.lifecycle_config_found = previous.lifecycle_config_found || scan_report.lifecycle_config_found; + self.truncated_by_limit = previous.truncated_by_limit || scan_report.truncated_by_limit; + self.truncated_by_duration = previous.truncated_by_duration || scan_report.truncated_by_duration; + self.cancelled = previous.cancelled || scan_report.cancelled; self.transition_completed = transition_completed; self.transition_failed = transition_failed; - self.tier_failure = scan_report.tier_failure.saturating_add(transition_failed); + self.tier_failure = self.tier_failure.saturating_add(transition_failed); self.tier_failure_by_reason = tier_failure_by_reason; } @@ -3765,7 +3803,10 @@ struct ManualTransitionContinuationToken { version_marker: Option, } -fn encode_manual_transition_continuation_token(marker: Option, version_marker: Option) -> Option { +pub(super) fn encode_manual_transition_continuation_token( + marker: Option, + version_marker: Option, +) -> Option { if marker.is_none() && version_marker.is_none() { return None; } @@ -9136,6 +9177,7 @@ mod tests { assert_eq!(loaded.report.scanned, 37); assert_eq!(loaded.report.eligible, 11); assert_eq!(loaded.report.enqueued, 5); + assert_eq!(loaded.cursor_revision, Some(37)); assert!(loaded.lease_expires_at_unix_nanos > 0); let token = loaded .report @@ -9154,6 +9196,30 @@ mod tests { assert_eq!(admission.lease_id, loaded.lease_id); assert_eq!(admission.lease_expires_at_unix_nanos, loaded.lease_expires_at_unix_nanos); + let mut same_marker_report = report.clone(); + same_marker_report.scanned += 1; + persist_manual_transition_page_checkpoint( + &checkpoint_options, + &same_marker_report, + Some("logs/page-end".to_string()), + Some("opaque-next-version".to_string()), + ) + .await + .expect("same-marker version checkpoint should persist through the durable progress sink"); + let same_marker_checkpointed = load_manual_transition_job_record(ecstore.clone(), job_id) + .await + .expect("same-marker version checkpoint should reload"); + assert_eq!(same_marker_checkpointed.cursor_revision, Some(38)); + let (_, version_marker) = decode_manual_transition_continuation_token( + same_marker_checkpointed + .report + .continuation_token + .as_deref() + .expect("same-marker version checkpoint should persist a cursor"), + ) + .expect("same-marker version cursor should decode"); + assert_eq!(version_marker.as_deref(), Some("opaque-next-version")); + create_test_bucket(&ecstore, &bucket).await; let lifecycle_xml = format!( r#" @@ -9210,6 +9276,7 @@ mod tests { assert_eq!(checkpointed.report.scanned, 1000); assert_eq!(checkpointed.report.eligible, 1000); assert_eq!(checkpointed.report.dry_run_eligible, 1000); + assert_eq!(checkpointed.cursor_revision, Some(1000)); let token = checkpointed .report .continuation_token diff --git a/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs new file mode 100644 index 000000000..606b5f536 --- /dev/null +++ b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs @@ -0,0 +1,1161 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeMap; + +use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::{ + bucket_lifecycle_ops::{ + ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token, + }, + manual_transition_job, tier_delete_journal, transition_transaction, +}; +use crate::error::{Error, Result}; + +pub(crate) const ILM_META_PREFIX: &str = "ilm"; +const ILM_META_OBJECT_PREFIX: &str = "ilm/"; +const MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE: usize = 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DurableIlmRecordKind { + TierDeleteJournal, + TransitionTransaction, + ManualTransitionJob, + ManualTransitionScope, + ManualTransitionTask, + ManualTransitionWorkerResult, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DurableIlmNamespace { + pub(crate) name: &'static str, + pub(crate) prefix: &'static str, + pub(crate) max_record_size: usize, + kind: DurableIlmRecordKind, +} + +pub(crate) const TIER_DELETE_JOURNAL_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "tier-delete-journal", + prefix: "ilm/tier-delete-journal/", + max_record_size: 64 * 1024, + kind: DurableIlmRecordKind::TierDeleteJournal, +}; +pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "transition-transaction", + prefix: "ilm/transition-transactions/records", + max_record_size: transition_transaction::MAX_TRANSITION_TRANSACTION_SIZE, + kind: DurableIlmRecordKind::TransitionTransaction, +}; +pub(crate) const MANUAL_TRANSITION_JOB_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "manual-transition-job", + prefix: "ilm/manual-transition/jobs", + max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE, + kind: DurableIlmRecordKind::ManualTransitionJob, +}; +pub(crate) const MANUAL_TRANSITION_SCOPE_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "manual-transition-scope", + prefix: "ilm/manual-transition/scopes", + max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE, + kind: DurableIlmRecordKind::ManualTransitionScope, +}; +pub(crate) const MANUAL_TRANSITION_TASK_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "manual-transition-task", + prefix: "ilm/manual-transition/tasks", + max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE, + kind: DurableIlmRecordKind::ManualTransitionTask, +}; +pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "manual-transition-worker-result", + prefix: "ilm/manual-transition/results", + max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE, + kind: DurableIlmRecordKind::ManualTransitionWorkerResult, +}; + +pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 6] = [ + TIER_DELETE_JOURNAL_NAMESPACE, + TRANSITION_TRANSACTION_NAMESPACE, + MANUAL_TRANSITION_JOB_NAMESPACE, + MANUAL_TRANSITION_SCOPE_NAMESPACE, + MANUAL_TRANSITION_TASK_NAMESPACE, + MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE, +]; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ValidatedDurableIlmRecord { + pub(crate) namespace: &'static str, + pub(crate) id_kind: &'static str, + pub(crate) id: String, + pub(crate) checkpoint: DurableIlmRecordCheckpoint, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManualTransitionJobProgressCheckpoint { + report: ManualTransitionRunReport, + queue_snapshot: ManualTransitionQueueSnapshot, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManualTransitionJobProgressProof { + scope_sha256: String, + scanned: u64, + eligible: u64, + enqueued: u64, + dry_run_eligible: u64, + skipped_not_transition: u64, + skipped_tier: u64, + skipped_delete_marker: u64, + skipped_directory: u64, + skipped_replication: u64, + skipped_already_transitioned: u64, + skipped_already_in_flight: u64, + skipped_queue_full: u64, + skipped_queue_closed: u64, + skipped_queue_timeout: u64, + transition_completed: u64, + transition_failed: u64, + tier_failure: u64, + tier_failure_by_reason: BTreeMap, + lifecycle_config_found: bool, + truncated_by_limit: bool, + truncated_by_duration: bool, + cancelled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + continuation_token_sha256: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cursor_marker: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cursor_revision: Option, + queue_snapshot: ManualTransitionQueueSnapshot, +} + +impl ValidatedDurableIlmRecord { + pub(crate) fn context(&self) -> String { + format!("namespace `{}` {} `{}`", self.namespace, self.id_kind, self.id) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub(crate) enum DurableIlmRecordCheckpoint { + TierDeleteJournal { + content_sha256: String, + identity_sha256: String, + committed: bool, + }, + TransitionTransaction { + content_sha256: String, + identity_sha256: String, + remote_version_sha256: String, + remote_version_known: bool, + revision: u64, + state: transition_transaction::TransitionTransactionState, + }, + ManualTransitionJob { + content_sha256: String, + identity_sha256: String, + updated_at_unix_nanos: i64, + state: manual_transition_job::ManualTransitionJobState, + scan_completed: bool, + cancel_requested: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + progress: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + progress_proof: Option>, + }, + ManualTransitionScope { + content_sha256: String, + identity_sha256: String, + updated_at_unix_nanos: i64, + }, + ManualTransitionTask { + content_sha256: String, + }, + ManualTransitionWorkerResult { + content_sha256: String, + }, +} + +impl DurableIlmRecordCheckpoint { + pub(crate) fn content_sha256(&self) -> &str { + match self { + Self::TierDeleteJournal { content_sha256, .. } + | Self::TransitionTransaction { content_sha256, .. } + | Self::ManualTransitionJob { content_sha256, .. } + | Self::ManualTransitionScope { content_sha256, .. } + | Self::ManualTransitionTask { content_sha256 } + | Self::ManualTransitionWorkerResult { content_sha256 } => content_sha256, + } + } + + pub(crate) fn compacted(&self) -> Result { + let mut checkpoint = self.clone(); + if let Self::ManualTransitionJob { + progress, + progress_proof, + .. + } = &mut checkpoint + { + match (progress.take(), progress_proof.take()) { + (Some(progress), None) => { + *progress_proof = Some(Box::new(ManualTransitionJobProgressProof::new( + &progress.report, + &progress.queue_snapshot, + None, + )?)); + } + (None, Some(proof)) if proof.is_valid() => *progress_proof = Some(proof), + (None, None) => {} + _ => return Err(Error::other("durable ILM manual transition checkpoint is invalid")), + } + } + Ok(checkpoint) + } + + pub(crate) fn validate_successor(&self, next: &Self) -> Result<()> { + if self == next { + if let Self::ManualTransitionJob { + progress, + progress_proof, + .. + } = self + && !manual_job_progress_checkpoint_is_valid(progress.as_deref(), progress_proof.as_deref()) + { + return Err(Error::other("durable ILM manual transition checkpoint is invalid")); + } + return Ok(()); + } + + let valid = match (self, next) { + ( + Self::TierDeleteJournal { + identity_sha256: previous_identity, + committed: previous_committed, + .. + }, + Self::TierDeleteJournal { + identity_sha256: next_identity, + committed: next_committed, + .. + }, + ) => { + previous_identity == next_identity + && (previous_committed == next_committed || (!previous_committed && *next_committed)) + } + ( + Self::TransitionTransaction { + identity_sha256: previous_identity, + remote_version_sha256: previous_remote_version, + remote_version_known: previous_remote_version_known, + revision: previous_revision, + state: previous_state, + .. + }, + Self::TransitionTransaction { + identity_sha256: next_identity, + remote_version_sha256: next_remote_version, + revision: next_revision, + state: next_state, + .. + }, + ) => { + previous_identity == next_identity + && transition_state_distance(*previous_state, *next_state) + .and_then(|distance| previous_revision.checked_add(distance)) + .is_some_and(|expected_revision| *next_revision == expected_revision) + && (!previous_remote_version_known || previous_remote_version == next_remote_version) + } + ( + Self::ManualTransitionJob { + content_sha256: previous_content, + identity_sha256: previous_identity, + updated_at_unix_nanos: previous_updated_at, + state: previous_state, + scan_completed: previous_scan_completed, + cancel_requested: previous_cancel_requested, + progress: previous_progress, + progress_proof: previous_progress_proof, + .. + }, + Self::ManualTransitionJob { + content_sha256: next_content, + identity_sha256: next_identity, + updated_at_unix_nanos: next_updated_at, + state: next_state, + scan_completed: next_scan_completed, + cancel_requested: next_cancel_requested, + progress: next_progress, + progress_proof: next_progress_proof, + .. + }, + ) => { + let same_generation = previous_content == next_content + && previous_identity == next_identity + && previous_updated_at == next_updated_at + && previous_state == next_state + && previous_scan_completed == next_scan_completed + && previous_cancel_requested == next_cancel_requested + && manual_job_progress_equivalent( + previous_progress.as_deref(), + previous_progress_proof.as_deref(), + next_progress.as_deref(), + next_progress_proof.as_deref(), + ); + same_generation + || (previous_identity == next_identity + && next_updated_at > previous_updated_at + && manual_job_state_reaches(*previous_state, *next_state) + && (!previous_scan_completed || *next_scan_completed) + && (!previous_cancel_requested || *next_cancel_requested) + && manual_job_progress_reaches( + previous_progress.as_deref(), + previous_progress_proof.as_deref(), + next_progress.as_deref(), + next_progress_proof.as_deref(), + *next_scan_completed, + )) + } + ( + Self::ManualTransitionScope { + identity_sha256: previous_identity, + updated_at_unix_nanos: previous_updated_at, + .. + }, + Self::ManualTransitionScope { + identity_sha256: next_identity, + updated_at_unix_nanos: next_updated_at, + .. + }, + ) => previous_identity == next_identity && next_updated_at > previous_updated_at, + _ => false, + }; + + if valid { + Ok(()) + } else { + Err(Error::other("durable ILM record generation is not a monotonic successor")) + } + } +} + +fn transition_state_distance( + from: transition_transaction::TransitionTransactionState, + to: transition_transaction::TransitionTransactionState, +) -> Option { + use transition_transaction::TransitionTransactionState::{ + AbortedNoRemote, CleanupPending, Committed, LocalCommitStarted, UploadOutcomeUnknown, UploadStarted, Uploaded, + }; + + match (from, to) { + (UploadStarted, UploadOutcomeUnknown | AbortedNoRemote | Uploaded) => Some(1), + (UploadStarted, LocalCommitStarted | CleanupPending) => Some(2), + (UploadStarted, Committed) => Some(3), + (UploadOutcomeUnknown, Uploaded | CleanupPending) => Some(1), + (UploadOutcomeUnknown, LocalCommitStarted) => Some(2), + (UploadOutcomeUnknown, Committed) => Some(3), + (Uploaded, LocalCommitStarted | CleanupPending) => Some(1), + (Uploaded, Committed) => Some(2), + (LocalCommitStarted, Committed | CleanupPending) => Some(1), + _ => None, + } +} + +fn manual_job_state_reaches( + from: manual_transition_job::ManualTransitionJobState, + to: manual_transition_job::ManualTransitionJobState, +) -> bool { + from == to || from == manual_transition_job::ManualTransitionJobState::Running +} + +impl ManualTransitionJobProgressProof { + fn new( + report: &ManualTransitionRunReport, + queue_snapshot: &ManualTransitionQueueSnapshot, + cursor_revision: Option, + ) -> Result { + let cursor_marker = match report.continuation_token.as_deref() { + Some(token) => { + let marker = decode_manual_transition_continuation_token(token)? + .0 + .ok_or_else(|| Error::other("durable ILM manual transition cursor marker is missing"))?; + (marker.len() <= MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE).then_some(marker) + } + None => None, + }; + if !manual_job_worker_results_are_valid(report) + || !manual_job_queue_snapshot_is_valid(queue_snapshot) + || (report.continuation_token.is_some() && cursor_revision == Some(0)) + { + return Err(Error::other("durable ILM manual transition progress is invalid")); + } + Ok(Self { + scope_sha256: checkpoint_hash(&(report.bucket.as_str(), report.prefix.as_str(), &report.tier, report.dry_run))?, + scanned: report.scanned, + eligible: report.eligible, + enqueued: report.enqueued, + dry_run_eligible: report.dry_run_eligible, + skipped_not_transition: report.skipped_not_transition, + skipped_tier: report.skipped_tier, + skipped_delete_marker: report.skipped_delete_marker, + skipped_directory: report.skipped_directory, + skipped_replication: report.skipped_replication, + skipped_already_transitioned: report.skipped_already_transitioned, + skipped_already_in_flight: report.skipped_already_in_flight, + skipped_queue_full: report.skipped_queue_full, + skipped_queue_closed: report.skipped_queue_closed, + skipped_queue_timeout: report.skipped_queue_timeout, + transition_completed: report.transition_completed, + transition_failed: report.transition_failed, + tier_failure: report.tier_failure, + tier_failure_by_reason: report.tier_failure_by_reason.clone(), + lifecycle_config_found: report.lifecycle_config_found, + truncated_by_limit: report.truncated_by_limit, + truncated_by_duration: report.truncated_by_duration, + cancelled: report.cancelled, + continuation_token_sha256: report + .continuation_token + .as_deref() + .map(|token| hex_sha256(token.as_bytes(), ToOwned::to_owned)), + cursor_marker, + cursor_revision, + queue_snapshot: *queue_snapshot, + }) + } + + fn is_valid(&self) -> bool { + let reason_total = self + .tier_failure_by_reason + .values() + .try_fold(0u64, |total, count| total.checked_add(*count)); + is_sha256_checksum(&self.scope_sha256) + && self.continuation_token_sha256.as_deref().is_none_or(is_sha256_checksum) + && match (&self.continuation_token_sha256, &self.cursor_marker) { + (None, None) | (Some(_), None) => true, + (Some(_), Some(marker)) => !marker.is_empty() && marker.len() <= MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE, + (None, Some(_)) => false, + } + && !(self.continuation_token_sha256.is_some() && self.cursor_revision == Some(0)) + && self + .transition_completed + .checked_add(self.transition_failed) + .is_some_and(|total| total <= self.enqueued) + && self.transition_failed <= self.tier_failure + && reason_total.is_some_and(|total| total <= self.tier_failure) + && manual_job_queue_snapshot_is_valid(&self.queue_snapshot) + } +} + +fn manual_job_progress_checkpoint_is_valid( + progress: Option<&ManualTransitionJobProgressCheckpoint>, + proof: Option<&ManualTransitionJobProgressProof>, +) -> bool { + match (progress, proof) { + (Some(progress), None) => manual_job_progress_is_valid(progress), + (None, Some(proof)) => proof.is_valid(), + (None, None) => true, + (Some(_), Some(_)) => false, + } +} + +fn manual_job_progress_proof( + progress: Option<&ManualTransitionJobProgressCheckpoint>, + proof: Option<&ManualTransitionJobProgressProof>, +) -> Option { + match (progress, proof) { + (Some(progress), None) => ManualTransitionJobProgressProof::new(&progress.report, &progress.queue_snapshot, None).ok(), + (None, Some(proof)) if proof.is_valid() => Some(proof.clone()), + _ => None, + } +} + +fn manual_job_progress_equivalent( + previous: Option<&ManualTransitionJobProgressCheckpoint>, + previous_proof: Option<&ManualTransitionJobProgressProof>, + next: Option<&ManualTransitionJobProgressCheckpoint>, + next_proof: Option<&ManualTransitionJobProgressProof>, +) -> bool { + if !manual_job_progress_checkpoint_is_valid(previous, previous_proof) + || !manual_job_progress_checkpoint_is_valid(next, next_proof) + { + return false; + } + match ( + manual_job_progress_proof(previous, previous_proof), + manual_job_progress_proof(next, next_proof), + ) { + (Some(previous), Some(next)) => previous == next, + (None, None) => true, + _ => false, + } +} + +fn manual_job_progress_reaches( + previous: Option<&ManualTransitionJobProgressCheckpoint>, + previous_proof: Option<&ManualTransitionJobProgressProof>, + next: Option<&ManualTransitionJobProgressCheckpoint>, + next_proof: Option<&ManualTransitionJobProgressProof>, + next_scan_completed: bool, +) -> bool { + if previous.is_none() && previous_proof.is_none() { + return (next.is_some() || next_proof.is_some()) && manual_job_progress_checkpoint_is_valid(next, next_proof); + } + let (Some(previous_compact), Some(next_compact)) = ( + manual_job_progress_proof(previous, previous_proof), + manual_job_progress_proof(next, next_proof), + ) else { + return false; + }; + + macro_rules! counters_do_not_regress { + ($($field:ident),+ $(,)?) => { + $(previous_compact.$field <= next_compact.$field)&&+ + }; + } + + let counters_monotonic = counters_do_not_regress!( + scanned, + eligible, + enqueued, + dry_run_eligible, + skipped_not_transition, + skipped_tier, + skipped_delete_marker, + skipped_directory, + skipped_replication, + skipped_already_transitioned, + skipped_already_in_flight, + skipped_queue_full, + skipped_queue_closed, + skipped_queue_timeout, + transition_completed, + transition_failed, + tier_failure, + ); + let failure_reasons_monotonic = previous_compact + .tier_failure_by_reason + .iter() + .all(|(reason, previous_count)| { + next_compact + .tier_failure_by_reason + .get(reason) + .is_some_and(|next_count| next_count >= previous_count) + }); + let flags_monotonic = (!previous_compact.lifecycle_config_found || next_compact.lifecycle_config_found) + && (!previous_compact.truncated_by_limit || next_compact.truncated_by_limit) + && (!previous_compact.truncated_by_duration || next_compact.truncated_by_duration) + && (!previous_compact.cancelled || next_compact.cancelled); + let cursor_monotonic = manual_job_cursor_reaches( + &previous_compact, + &next_compact, + previous.map(|progress| &progress.report), + next.map(|progress| &progress.report), + next_scan_completed, + ); + + previous_compact.scope_sha256 == next_compact.scope_sha256 + && counters_monotonic + && failure_reasons_monotonic + && flags_monotonic + && cursor_monotonic + && previous_compact.is_valid() + && next_compact.is_valid() +} + +fn manual_job_progress_is_valid(progress: &ManualTransitionJobProgressCheckpoint) -> bool { + manual_job_worker_results_are_valid(&progress.report) + && manual_job_queue_snapshot_is_valid(&progress.queue_snapshot) + && manual_job_cursor_is_valid(progress.report.continuation_token.as_deref()) +} + +fn manual_job_worker_results_are_valid(report: &ManualTransitionRunReport) -> bool { + let reason_total = report + .tier_failure_by_reason + .values() + .try_fold(0u64, |total, count| total.checked_add(*count)); + report + .transition_completed + .checked_add(report.transition_failed) + .is_some_and(|total| total <= report.enqueued) + && report.transition_failed <= report.tier_failure + && reason_total.is_some_and(|total| total <= report.tier_failure) +} + +fn manual_job_cursor_reaches( + previous: &ManualTransitionJobProgressProof, + next: &ManualTransitionJobProgressProof, + previous_legacy: Option<&ManualTransitionRunReport>, + next_legacy: Option<&ManualTransitionRunReport>, + next_scan_completed: bool, +) -> bool { + if previous.continuation_token_sha256 == next.continuation_token_sha256 { + return previous.cursor_marker == next.cursor_marker && previous.cursor_revision == next.cursor_revision; + } + match (&previous.continuation_token_sha256, &next.continuation_token_sha256) { + (None, Some(_)) => { + next.scanned > previous.scanned + && (manual_job_cursor_revision_advances(previous.cursor_revision, next.cursor_revision) + || (previous.cursor_revision.is_none() && next.cursor_revision.is_none())) + } + (Some(_), None) => next_scan_completed, + (Some(_), Some(_)) if next.scanned > previous.scanned => { + manual_job_cursor_revision_advances(previous.cursor_revision, next.cursor_revision) + || manual_job_legacy_cursor_reaches(previous, next, previous_legacy, next_legacy) + } + _ => false, + } +} + +fn manual_job_cursor_revision_advances(previous: Option, next: Option) -> bool { + match (previous, next) { + (Some(previous), Some(next)) => next > previous, + (None, Some(next)) => next > 0, + _ => false, + } +} + +fn manual_job_legacy_cursor_reaches( + previous_proof: &ManualTransitionJobProgressProof, + next_proof: &ManualTransitionJobProgressProof, + previous_legacy: Option<&ManualTransitionRunReport>, + next_legacy: Option<&ManualTransitionRunReport>, +) -> bool { + if let (Some(previous_marker), Some(next_marker)) = (&previous_proof.cursor_marker, &next_proof.cursor_marker) { + return next_marker > previous_marker; + } + let (Some(previous_token), Some(next_token)) = ( + previous_legacy.and_then(|report| report.continuation_token.as_deref()), + next_legacy.and_then(|report| report.continuation_token.as_deref()), + ) else { + return false; + }; + let (Ok((Some(previous_marker), _)), Ok((Some(next_marker), _))) = ( + decode_manual_transition_continuation_token(previous_token), + decode_manual_transition_continuation_token(next_token), + ) else { + return false; + }; + next_marker > previous_marker +} + +fn manual_job_cursor_is_valid(token: Option<&str>) -> bool { + let Some(token) = token else { + return true; + }; + matches!(decode_manual_transition_continuation_token(token), Ok((Some(_), _))) +} + +fn manual_job_queue_snapshot_is_valid(snapshot: &ManualTransitionQueueSnapshot) -> bool { + (snapshot.queue_capacity > 0 || snapshot.queued == 0) + && (snapshot.queue_capacity == 0 || snapshot.queued <= snapshot.queue_capacity) + && (snapshot.workers > 0 || snapshot.active == 0) + && (snapshot.workers == 0 || snapshot.active <= snapshot.workers) +} + +fn checkpoint_hash(value: &T) -> Result { + let encoded = serde_json::to_vec(value).map_err(Error::other)?; + Ok(hex_sha256(&encoded, ToOwned::to_owned)) +} + +fn path_is_in_namespace(path: &str, namespace: &DurableIlmNamespace) -> bool { + let Some(suffix) = path.strip_prefix(namespace.prefix) else { + return false; + }; + if namespace.prefix.ends_with('/') { + !suffix.is_empty() + } else { + suffix.starts_with('/') && suffix.len() > 1 + } +} + +pub(crate) fn classify_durable_ilm_record(path: &str) -> Result> { + if path != ILM_META_PREFIX && !path.starts_with(ILM_META_OBJECT_PREFIX) { + return Ok(None); + } + + DURABLE_ILM_NAMESPACES + .iter() + .find(|namespace| path_is_in_namespace(path, namespace)) + .map(Some) + .ok_or_else(|| Error::other(format!("unregistered durable ILM namespace for path `{path}`"))) +} + +fn parse_manual_sharded_record(path: &str, prefix: &str) -> Result<(Uuid, String)> { + let suffix = path + .strip_prefix(prefix) + .and_then(|suffix| suffix.strip_prefix('/')) + .ok_or_else(|| Error::other("manual transition record path has wrong prefix"))?; + let mut parts = suffix.split('/'); + let first = parts + .next() + .ok_or_else(|| Error::other("manual transition record first shard is missing"))?; + let second = parts + .next() + .ok_or_else(|| Error::other("manual transition record second shard is missing"))?; + let job_key = parts + .next() + .ok_or_else(|| Error::other("manual transition record job id is missing"))?; + let task_key = parts + .next() + .and_then(|file| file.strip_suffix(".json")) + .ok_or_else(|| Error::other("manual transition record task key is missing"))?; + if parts.next().is_some() + || job_key.len() != 32 + || first != &job_key[..2] + || second != &job_key[2..4] + || !job_key + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err(Error::other("manual transition record job id or shards are invalid")); + } + let job_id = Uuid::parse_str(job_key).map_err(|_| Error::other("manual transition record job id is invalid"))?; + Ok((job_id, task_key.to_string())) +} + +pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result { + let namespace = + classify_durable_ilm_record(path)?.ok_or_else(|| Error::other(format!("path `{path}` is not a durable ILM record")))?; + if data.len() > namespace.max_record_size { + return Err(Error::other(format!( + "durable ILM record exceeds {} byte limit", + namespace.max_record_size + ))); + } + + let content_sha256 = hex_sha256(data, ToOwned::to_owned); + let (id_kind, id, checkpoint) = match namespace.kind { + DurableIlmRecordKind::TierDeleteJournal => { + let entry = tier_delete_journal::decode_tier_delete_journal_entry(data)?; + if tier_delete_journal::tier_delete_journal_object_name(&entry) != path { + return Err(Error::other("tier delete journal content does not match its path")); + } + let operation_id = path + .strip_prefix(namespace.prefix) + .and_then(|suffix| suffix.strip_suffix(".json")) + .ok_or_else(|| Error::other("tier delete journal path is invalid"))?; + let identity_sha256 = checkpoint_hash(&( + &entry.obj_name, + &entry.version_id, + &entry.tier_name, + entry.backend_identity, + entry.version_id_exact, + entry.version_state, + &entry.source, + ))?; + ( + "operation_id", + operation_id.to_string(), + DurableIlmRecordCheckpoint::TierDeleteJournal { + content_sha256, + identity_sha256, + committed: entry.state == super::tier_sweeper::TierDeleteJournalState::Committed, + }, + ) + } + DurableIlmRecordKind::TransitionTransaction => { + let transaction = transition_transaction::decode_transition_transaction_record(path, data) + .map_err(|err| Error::other(err.to_string()))?; + let identity_sha256 = checkpoint_hash(&( + transaction.deployment_id, + transaction.transaction_id, + transaction.owner_epoch, + transaction.write_id, + &transaction.source, + &transaction.tier_name, + transaction.backend_fingerprint, + &transaction.remote_object, + transaction.not_after_unix_nanos, + ))?; + let remote_version_sha256 = checkpoint_hash(&transaction.remote_version)?; + ( + "transaction_id", + transaction.transaction_id.to_string(), + DurableIlmRecordCheckpoint::TransitionTransaction { + content_sha256, + identity_sha256, + remote_version_sha256, + remote_version_known: !transaction.remote_version.is_unknown(), + revision: transaction.revision, + state: transaction.state, + }, + ) + } + DurableIlmRecordKind::ManualTransitionJob => { + let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path) + .map_err(|err| Error::other(err.to_string()))?; + let canonical = manual_transition_job::manual_transition_job_record_object_name(job_id) + .map_err(|err| Error::other(err.to_string()))?; + if canonical != path { + return Err(Error::other("manual transition job path is not canonical")); + } + let job = manual_transition_job::ManualTransitionJobRecord::decode(job_id, data) + .map_err(|err| Error::other(err.to_string()))?; + let identity_sha256 = checkpoint_hash(&( + job.job_id, + &job.scope_key, + &job.bucket, + &job.prefix, + &job.tier, + job.dry_run, + job.max_objects, + job.max_duration, + job.created_at_unix_nanos, + ))?; + let progress_proof = ManualTransitionJobProgressProof::new(&job.report, &job.queue_snapshot, job.cursor_revision)?; + let updated_at_unix_nanos = i64::try_from(job.updated_at_unix_nanos) + .map_err(|_| Error::other("manual transition job updated_at exceeds durable ILM checkpoint range"))?; + ( + "job_id", + job_id.to_string(), + DurableIlmRecordCheckpoint::ManualTransitionJob { + content_sha256, + identity_sha256, + updated_at_unix_nanos, + state: job.state, + scan_completed: job.scan_completed, + cancel_requested: job.cancel_requested, + progress: None, + progress_proof: Some(Box::new(progress_proof)), + }, + ) + } + DurableIlmRecordKind::ManualTransitionScope => { + let admission: manual_transition_job::ManualTransitionScopeAdmission = + serde_json::from_slice(data).map_err(Error::other)?; + admission.validate().map_err(|err| Error::other(err.to_string()))?; + let canonical = manual_transition_job::manual_transition_scope_record_object_name(&admission.scope_key) + .map_err(|err| Error::other(err.to_string()))?; + if canonical != path { + return Err(Error::other("manual transition scope content does not match its path")); + } + let identity_sha256 = checkpoint_hash(&( + &admission.schema, + &admission.scope_key, + admission.job_id, + &admission.bucket, + &admission.prefix, + &admission.tier, + admission.dry_run, + ))?; + let updated_at_unix_nanos = i64::try_from(admission.updated_at_unix_nanos) + .map_err(|_| Error::other("manual transition scope updated_at exceeds durable ILM checkpoint range"))?; + ( + "job_id", + admission.job_id.to_string(), + DurableIlmRecordCheckpoint::ManualTransitionScope { + content_sha256, + identity_sha256, + updated_at_unix_nanos, + }, + ) + } + DurableIlmRecordKind::ManualTransitionTask => { + let (job_id, task_key) = parse_manual_sharded_record(path, namespace.prefix)?; + let canonical = manual_transition_job::manual_transition_task_object_name(job_id, &task_key) + .map_err(|err| Error::other(err.to_string()))?; + if canonical != path { + return Err(Error::other("manual transition task path is not canonical")); + } + manual_transition_job::ManualTransitionTaskRecord::decode(job_id, &task_key, data) + .map_err(|err| Error::other(err.to_string()))?; + ( + "job_id", + job_id.to_string(), + DurableIlmRecordCheckpoint::ManualTransitionTask { content_sha256 }, + ) + } + DurableIlmRecordKind::ManualTransitionWorkerResult => { + let (job_id, task_key) = parse_manual_sharded_record(path, namespace.prefix)?; + let canonical = manual_transition_job::manual_transition_worker_result_object_name(job_id, &task_key) + .map_err(|err| Error::other(err.to_string()))?; + if canonical != path { + return Err(Error::other("manual transition worker result path is not canonical")); + } + manual_transition_job::ManualTransitionWorkerResultRecord::decode(job_id, &task_key, data) + .map_err(|err| Error::other(err.to_string()))?; + ( + "job_id", + job_id.to_string(), + DurableIlmRecordCheckpoint::ManualTransitionWorkerResult { content_sha256 }, + ) + } + }; + + Ok(ValidatedDurableIlmRecord { + namespace: namespace.name, + id_kind, + id, + checkpoint, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn try_manual_job_checkpoint(job: &manual_transition_job::ManualTransitionJobRecord) -> Result { + let path = + manual_transition_job::manual_transition_job_record_object_name(job.job_id).expect("manual job path should build"); + let encoded = job.encode().expect("manual job should encode"); + Ok(validate_durable_ilm_record(&path, &encoded)?.checkpoint) + } + + fn manual_job_checkpoint(job: &manual_transition_job::ManualTransitionJobRecord) -> DurableIlmRecordCheckpoint { + try_manual_job_checkpoint(job).expect("manual job checkpoint should validate") + } + + fn continuation_token_with_version(marker: &str, version_marker: Option<&str>) -> String { + let encoded = serde_json::to_vec(&serde_json::json!({ "marker": marker, "version_marker": version_marker })) + .expect("continuation token should encode"); + base64_simd::URL_SAFE_NO_PAD.encode_to_string(&encoded) + } + + fn continuation_token(marker: &str) -> String { + continuation_token_with_version(marker, None) + } + + #[test] + fn unknown_ilm_record_requires_namespace_registration() { + let err = classify_durable_ilm_record("ilm/future-durable/jobs/one.json") + .expect_err("unknown durable ILM path must fail closed"); + + assert!(err.to_string().contains("ilm/future-durable/jobs/one.json")); + } + + #[test] + fn durable_ilm_registry_has_unique_non_overlapping_prefixes() { + for (index, namespace) in DURABLE_ILM_NAMESPACES.iter().enumerate() { + assert!(namespace.prefix.starts_with(ILM_META_OBJECT_PREFIX)); + assert!(namespace.max_record_size > 0); + for other in DURABLE_ILM_NAMESPACES.iter().skip(index + 1) { + assert_ne!(namespace.prefix, other.prefix); + assert!(!path_is_in_namespace(namespace.prefix, other)); + assert!(!path_is_in_namespace(other.prefix, namespace)); + } + } + } + + #[test] + fn manual_transition_job_checkpoint_compacts_legacy_progress_compatibly() { + let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default(); + let mut job = + manual_transition_job::ManualTransitionJobRecord::new(Uuid::new_v4(), "legacy-checkpoint-bucket", &options, "owner"); + job.cursor_revision = None; + job.updated_at_unix_nanos += 1; + job.report.scanned = 1; + job.report.continuation_token = Some(continuation_token("logs/a")); + let compact = manual_job_checkpoint(&job); + let mut legacy = compact.clone(); + let DurableIlmRecordCheckpoint::ManualTransitionJob { + progress, + progress_proof, + .. + } = &mut legacy + else { + panic!("manual job should produce a manual checkpoint"); + }; + *progress = Some(Box::new(ManualTransitionJobProgressCheckpoint { + report: job.report.clone(), + queue_snapshot: job.queue_snapshot, + })); + *progress_proof = None; + + compact + .validate_successor(&legacy) + .expect("bounded checkpoints should accept the same legacy generation"); + legacy + .validate_successor(&compact) + .expect("legacy checkpoints should accept the same bounded generation"); + assert_eq!(legacy.compacted().expect("legacy checkpoint should compact"), compact); + } + + #[test] + fn manual_transition_job_checkpoint_rejects_timestamp_outside_wire_range() { + let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default(); + let mut job = manual_transition_job::ManualTransitionJobRecord::new( + Uuid::new_v4(), + "checkpoint-timestamp-bucket", + &options, + "owner", + ); + job.updated_at_unix_nanos = i128::from(i64::MAX) + 1; + + let err = try_manual_job_checkpoint(&job).expect_err("out-of-range checkpoint timestamp must fail closed"); + + assert!(err.to_string().contains("updated_at exceeds durable ILM checkpoint range")); + } + + #[test] + fn manual_transition_scope_checkpoint_rejects_timestamp_outside_wire_range() { + let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default(); + let job = manual_transition_job::ManualTransitionJobRecord::new( + Uuid::new_v4(), + "scope-checkpoint-timestamp-bucket", + &options, + "owner", + ); + let mut admission = manual_transition_job::ManualTransitionScopeAdmission::from_job(&job); + admission.updated_at_unix_nanos = i128::from(i64::MAX) + 1; + let path = manual_transition_job::manual_transition_scope_record_object_name(&admission.scope_key) + .expect("manual transition scope path should build"); + let encoded = serde_json::to_vec(&admission).expect("manual transition scope should encode"); + + let err = + validate_durable_ilm_record(&path, &encoded).expect_err("out-of-range scope checkpoint timestamp must fail closed"); + + assert!(err.to_string().contains("updated_at exceeds durable ILM checkpoint range")); + } + + #[test] + fn manual_transition_job_checkpoint_rejects_progress_poison() { + let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default(); + let mut initial = + manual_transition_job::ManualTransitionJobRecord::new(Uuid::new_v4(), "manual-checkpoint-bucket", &options, "owner"); + let initial_checkpoint = manual_job_checkpoint(&initial); + let mut first_page = initial.report.clone(); + first_page.scanned = 1; + first_page.continuation_token = Some(continuation_token("logs/a")); + initial.update_running_progress(first_page, ManualTransitionQueueSnapshot::default()); + let first_page_checkpoint = manual_job_checkpoint(&initial); + initial_checkpoint + .validate_successor(&first_page_checkpoint) + .expect("the first durable cursor should advance from no cursor"); + + let mut legacy_checkpoint = initial_checkpoint; + let DurableIlmRecordCheckpoint::ManualTransitionJob { + progress, + progress_proof, + .. + } = &mut legacy_checkpoint + else { + panic!("manual job should produce a manual checkpoint"); + }; + *progress = None; + *progress_proof = None; + legacy_checkpoint + .validate_successor(&first_page_checkpoint) + .expect("legacy checkpoints should upgrade to validated progress"); + + let mut previous = initial; + let mut previous_report = previous.report.clone(); + previous_report.scanned = 10; + previous_report.eligible = 8; + previous_report.enqueued = 2; + previous_report.continuation_token = Some(continuation_token("logs/b")); + let previous_queue = ManualTransitionQueueSnapshot { + queue_capacity: 10, + queued: 1, + active: 1, + workers: 2, + queue_full: 2, + queue_send_timeout: 1, + ..Default::default() + }; + previous.update_running_progress(previous_report, previous_queue); + previous.report.transition_completed = 1; + let previous_checkpoint = manual_job_checkpoint(&previous); + + let mut next = previous.clone(); + let mut next_report = next.report.clone(); + next_report.scanned = 11; + next_report.eligible = 9; + next_report.continuation_token = Some(continuation_token("logs/c")); + let mut next_queue = next.queue_snapshot; + next_queue.queued = 0; + next_queue.active = 0; + next_queue.queue_full = 3; + next.update_running_progress(next_report, next_queue); + next.report.transition_completed = 2; + let next_checkpoint = manual_job_checkpoint(&next); + previous_checkpoint + .validate_successor(&next_checkpoint) + .expect("forward job progress should validate"); + + let mut counter_rollback = next.clone(); + counter_rollback.updated_at_unix_nanos += 1; + counter_rollback.report.scanned = 9; + assert!( + previous_checkpoint + .validate_successor(&manual_job_checkpoint(&counter_rollback)) + .is_err() + ); + + let mut cursor_rollback = previous.clone(); + cursor_rollback.updated_at_unix_nanos += 1; + cursor_rollback.report.scanned += 1; + cursor_rollback.report.continuation_token = Some(continuation_token("logs/a")); + assert!( + previous_checkpoint + .validate_successor(&manual_job_checkpoint(&cursor_rollback)) + .is_err() + ); + + let mut same_marker_version_previous = previous.clone(); + let mut same_marker_report = same_marker_version_previous.report.clone(); + same_marker_report.continuation_token = Some(continuation_token_with_version("logs/b", Some("opaque-z-version"))); + same_marker_version_previous.update_running_progress(same_marker_report, same_marker_version_previous.queue_snapshot); + let same_marker_version_previous_checkpoint = manual_job_checkpoint(&same_marker_version_previous); + let mut same_marker_version_next = same_marker_version_previous.clone(); + let mut same_marker_next_report = same_marker_version_next.report.clone(); + same_marker_next_report.scanned += 1; + same_marker_next_report.continuation_token = Some(continuation_token_with_version("logs/b", Some("opaque-a-version"))); + same_marker_version_next.update_running_progress(same_marker_next_report, same_marker_version_next.queue_snapshot); + same_marker_version_previous_checkpoint + .validate_successor(&manual_job_checkpoint(&same_marker_version_next)) + .expect("producer cursor revision should prove same-marker version progress"); + + let mut same_marker_version_rollback = same_marker_version_previous.clone(); + same_marker_version_rollback.updated_at_unix_nanos += 1; + same_marker_version_rollback.report.scanned += 1; + same_marker_version_rollback.report.continuation_token = + Some(continuation_token_with_version("logs/b", Some("opaque-arbitrary-version"))); + assert!( + same_marker_version_previous_checkpoint + .validate_successor(&manual_job_checkpoint(&same_marker_version_rollback)) + .is_err(), + "a different opaque version marker without producer evidence must fail closed" + ); + + let mut worker_result_rollback = next.clone(); + worker_result_rollback.updated_at_unix_nanos += 1; + worker_result_rollback.report.transition_completed = 0; + assert!( + previous_checkpoint + .validate_successor(&manual_job_checkpoint(&worker_result_rollback)) + .is_err() + ); + + let mut worker_result_overflow = next.clone(); + worker_result_overflow.updated_at_unix_nanos += 1; + worker_result_overflow.report.enqueued = u64::MAX; + worker_result_overflow.report.transition_completed = u64::MAX; + worker_result_overflow.report.transition_failed = 1; + worker_result_overflow.report.tier_failure = 1; + assert!(try_manual_job_checkpoint(&worker_result_overflow).is_err()); + + let mut invalid_cursor = next.clone(); + invalid_cursor.updated_at_unix_nanos += 1; + invalid_cursor.report.continuation_token = Some("not-base64".to_string()); + assert!(try_manual_job_checkpoint(&invalid_cursor).is_err()); + + let mut queue_state_poison = next; + queue_state_poison.updated_at_unix_nanos += 1; + queue_state_poison.queue_snapshot.queued = queue_state_poison.queue_snapshot.queue_capacity + 1; + assert!(try_manual_job_checkpoint(&queue_state_poison).is_err()); + } +} diff --git a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs index b1fe45cad..36a0120d5 100644 --- a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs +++ b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs @@ -20,10 +20,16 @@ use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use uuid::Uuid; +#[cfg(test)] +use crate::bucket::lifecycle::bucket_lifecycle_ops::encode_manual_transition_continuation_token; use crate::bucket::lifecycle::bucket_lifecycle_ops::{ ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, }; use crate::bucket::lifecycle::config_boundary; +use crate::bucket::lifecycle::durable_namespace::{ + MANUAL_TRANSITION_JOB_NAMESPACE, MANUAL_TRANSITION_SCOPE_NAMESPACE, MANUAL_TRANSITION_TASK_NAMESPACE, + MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE, +}; use crate::disk::RUSTFS_META_BUCKET; use crate::error::{Error, Result as EcstoreResult}; use crate::object_api::ObjectOptions; @@ -34,10 +40,10 @@ use crate::store::ECStore; pub const MANUAL_TRANSITION_JOB_SCHEMA: &str = "rustfs-manual-transition-job-v1"; pub const MANUAL_TRANSITION_TASK_SCHEMA: &str = "rustfs-manual-transition-task-v1"; pub const MANUAL_TRANSITION_WORKER_RESULT_SCHEMA: &str = "rustfs-manual-transition-worker-result-v1"; -pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = "ilm/manual-transition/jobs"; -pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = "ilm/manual-transition/scopes"; -pub const MANUAL_TRANSITION_TASK_PREFIX: &str = "ilm/manual-transition/tasks"; -pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = "ilm/manual-transition/results"; +pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = MANUAL_TRANSITION_JOB_NAMESPACE.prefix; +pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = MANUAL_TRANSITION_SCOPE_NAMESPACE.prefix; +pub const MANUAL_TRANSITION_TASK_PREFIX: &str = MANUAL_TRANSITION_TASK_NAMESPACE.prefix; +pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE.prefix; pub const MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE: usize = 64 * 1024; pub const MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE: usize = 16 * 1024; pub const MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE: usize = 8 * 1024; @@ -195,6 +201,8 @@ pub struct ManualTransitionJobRecord { pub updated_at_unix_nanos: i128, #[serde(default, skip_serializing_if = "Option::is_none")] pub completed_at_unix_nanos: Option, + #[serde(default, skip_serializing)] + pub cursor_revision: Option, pub report: ManualTransitionRunReport, pub queue_snapshot: ManualTransitionQueueSnapshot, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -224,6 +232,7 @@ impl ManualTransitionJobRecord { created_at_unix_nanos: now, updated_at_unix_nanos: now, completed_at_unix_nanos: None, + cursor_revision: None, report: ManualTransitionRunReport { bucket: bucket.to_string(), prefix: options.prefix.clone(), @@ -238,7 +247,7 @@ impl ManualTransitionJobRecord { pub fn complete(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) { self.scan_completed = true; - self.report.merge_scan_report_preserving_worker(&report); + self.merge_scan_report(&report); self.queue_snapshot = queue_snapshot; self.error = None; self.mark_terminal_if_worker_drained(); @@ -316,7 +325,7 @@ impl ManualTransitionJobRecord { } } self.queue_snapshot = queue_snapshot; - self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos(); + self.advance_updated_at(); self.mark_terminal_if_worker_drained(); } @@ -359,14 +368,14 @@ impl ManualTransitionJobRecord { self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed); self.report.tier_failure_by_reason = scan_tier_failure_by_reason; self.queue_snapshot = queue_snapshot; - self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos(); + self.advance_updated_at(); self.mark_terminal_if_worker_drained(); true } pub fn mark_cancel_requested(&mut self) { self.cancel_requested = true; - self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos(); + self.advance_updated_at(); } pub fn claim_recovery_lease(&mut self, owner_id: impl Into, queue_snapshot: ManualTransitionQueueSnapshot) { @@ -380,7 +389,7 @@ impl ManualTransitionJobRecord { pub fn abandon_recovery_lease(&mut self, lease_id: Uuid) { if self.state == ManualTransitionJobState::Running && self.lease_id == lease_id { self.lease_expires_at_unix_nanos = 0; - self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos(); + self.advance_updated_at(); } } @@ -398,7 +407,7 @@ impl ManualTransitionJobRecord { pub fn renew_lease(&mut self, queue_snapshot: ManualTransitionQueueSnapshot) { let now = OffsetDateTime::now_utc().unix_timestamp_nanos(); - self.updated_at_unix_nanos = now; + self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now); self.lease_expires_at_unix_nanos = manual_transition_job_lease_expires_at(now); self.queue_snapshot = queue_snapshot; } @@ -438,11 +447,16 @@ impl ManualTransitionJobRecord { pub fn update_running_progress(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) { if self.state == ManualTransitionJobState::Running { - self.report.merge_scan_report_preserving_worker(&report); + self.merge_scan_report(&report); self.renew_lease(queue_snapshot); } } + fn merge_scan_report(&mut self, report: &ManualTransitionRunReport) { + self.report.merge_scan_report_preserving_worker(report); + self.cursor_revision = manual_transition_cursor_revision(&self.report); + } + pub fn mark_unknown_if_unowned(&mut self) { if self.state == ManualTransitionJobState::Running { self.state = ManualTransitionJobState::Unknown; @@ -463,9 +477,13 @@ impl ManualTransitionJobRecord { } fn mark_updated_terminal(&mut self) { + self.advance_updated_at(); + self.completed_at_unix_nanos = Some(self.updated_at_unix_nanos); + } + + fn advance_updated_at(&mut self) { let now = OffsetDateTime::now_utc().unix_timestamp_nanos(); - self.updated_at_unix_nanos = now; - self.completed_at_unix_nanos = Some(now); + self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now); } fn mark_terminal_if_worker_drained(&mut self) { @@ -543,6 +561,7 @@ impl ManualTransitionJobRecord { if job.state == ManualTransitionJobState::Cancelled && job.cancel_requested { job.report.cancelled = true; } + job.cursor_revision = manual_transition_cursor_revision(&job.report); job.validate()?; Ok(job) } @@ -1109,7 +1128,8 @@ pub fn manual_transition_scope_record_object_name(scope_key: &str) -> Result, job: &ManualTransitionJobRecord) -> EcstoreResult<()> { let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?; let data = job.encode().map_err(manual_transition_job_store_error)?; - config_boundary::save_config(api, &object, data).await + config_boundary::save_config(api.clone(), &object, data.clone()).await?; + api.record_durable_ilm_decommission_progress(&object, &data).await } pub async fn load_manual_transition_job_record(api: Arc, job_id: Uuid) -> EcstoreResult { @@ -1142,9 +1162,9 @@ pub async fn save_manual_transition_job_record_if_current( let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?; let data = job.encode().map_err(manual_transition_job_store_error)?; config_boundary::save_config_with_opts_quiet( - api, + api.clone(), &object, - data, + data.clone(), &ObjectOptions { max_parity: true, http_preconditions: Some(HTTPPreconditions { @@ -1154,7 +1174,8 @@ pub async fn save_manual_transition_job_record_if_current( ..Default::default() }, ) - .await + .await?; + api.record_durable_ilm_decommission_progress(&object, &data).await } /// Applies a job-record mutation with optimistic concurrency control. @@ -1592,9 +1613,9 @@ pub async fn save_manual_transition_scope_admission_if_absent( let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?; let data = serde_json::to_vec(admission).map_err(Error::other)?; config_boundary::save_config_with_opts( - api, + api.clone(), &object, - data, + data.clone(), &ObjectOptions { max_parity: true, http_preconditions: Some(HTTPPreconditions { @@ -1604,7 +1625,8 @@ pub async fn save_manual_transition_scope_admission_if_absent( ..Default::default() }, ) - .await + .await?; + api.record_durable_ilm_decommission_progress(&object, &data).await } pub async fn load_manual_transition_scope_admission( @@ -1642,9 +1664,9 @@ pub async fn save_manual_transition_scope_admission_if_current( let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?; let data = serde_json::to_vec(admission).map_err(Error::other)?; match config_boundary::save_config_with_opts( - api, + api.clone(), &object, - data, + data.clone(), &ObjectOptions { max_parity: true, http_preconditions: Some(HTTPPreconditions { @@ -1660,7 +1682,8 @@ pub async fn save_manual_transition_scope_admission_if_current( Err(Error::PreconditionFailed) } result => result, - } + }?; + api.record_durable_ilm_decommission_progress(&object, &data).await } pub async fn claim_manual_transition_scope_admission( @@ -1953,13 +1976,15 @@ pub async fn delete_manual_transition_scope_admission_if_current( job_id: Uuid, lease_id: Uuid, ) -> EcstoreResult { - let etag = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await { - Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => etag, + let (admission, etag) = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await { + Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => (admission, etag), Ok(_) => return Ok(false), Err(Error::ConfigNotFound) => return Ok(true), Err(err) => return Err(err), }; let object = manual_transition_scope_record_object_name(scope_key).map_err(manual_transition_job_store_error)?; + let data = serde_json::to_vec(&admission).map_err(Error::other)?; + api.record_durable_ilm_decommission_terminal(&object, &data).await?; match config_boundary::delete_config_if_match(api, &object, &etag).await { Ok(()) | Err(Error::ConfigNotFound) => Ok(true), Err(Error::PreconditionFailed) => Ok(false), @@ -1971,6 +1996,11 @@ fn manual_transition_job_store_error(err: ManualTransitionJobError) -> Error { Error::other(err) } +fn manual_transition_cursor_revision(report: &ManualTransitionRunReport) -> Option { + report.continuation_token.as_ref()?; + (report.scanned > 0).then_some(report.scanned) +} + pub fn manual_transition_scope_admission_lease_expired(admission: &ManualTransitionScopeAdmission) -> bool { OffsetDateTime::now_utc().unix_timestamp_nanos() > admission.lease_expires_at_unix_nanos } @@ -2210,6 +2240,76 @@ mod tests { ); } + #[test] + fn manual_transition_job_scan_progress_accumulates_resumed_checkpoint_counters() { + let options = ManualTransitionRunOptions::default(); + let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER); + let first_token = + encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string())); + record.update_running_progress( + ManualTransitionRunReport { + bucket: "bucket".to_string(), + lifecycle_config_found: true, + scanned: 1000, + eligible: 800, + enqueued: 50, + dry_run_eligible: 10, + skipped_not_transition: 2, + skipped_tier: 3, + skipped_delete_marker: 4, + skipped_directory: 5, + skipped_replication: 6, + skipped_already_transitioned: 7, + skipped_already_in_flight: 8, + skipped_queue_full: 9, + skipped_queue_closed: 10, + skipped_queue_timeout: 11, + tier_failure: 12, + truncated_by_duration: true, + continuation_token: first_token, + ..Default::default() + }, + ManualTransitionQueueSnapshot::default(), + ); + + let next_token = + encode_manual_transition_continuation_token(Some("logs/page-b".to_string()), Some("version-b".to_string())); + record.update_running_progress( + ManualTransitionRunReport { + bucket: "bucket".to_string(), + scanned: 3, + eligible: 2, + enqueued: 1, + dry_run_eligible: 1, + skipped_not_transition: 1, + tier_failure: 1, + continuation_token: next_token.clone(), + ..Default::default() + }, + ManualTransitionQueueSnapshot::default(), + ); + + assert_eq!(record.report.scanned, 1003); + assert_eq!(record.report.eligible, 802); + assert_eq!(record.report.enqueued, 51); + assert_eq!(record.report.dry_run_eligible, 11); + assert_eq!(record.report.skipped_not_transition, 3); + assert_eq!(record.report.skipped_tier, 3); + assert_eq!(record.report.skipped_delete_marker, 4); + assert_eq!(record.report.skipped_directory, 5); + assert_eq!(record.report.skipped_replication, 6); + assert_eq!(record.report.skipped_already_transitioned, 7); + assert_eq!(record.report.skipped_already_in_flight, 8); + assert_eq!(record.report.skipped_queue_full, 9); + assert_eq!(record.report.skipped_queue_closed, 10); + assert_eq!(record.report.skipped_queue_timeout, 11); + assert_eq!(record.report.tier_failure, 13); + assert!(record.report.lifecycle_config_found); + assert!(record.report.truncated_by_duration); + assert_eq!(record.report.continuation_token, next_token); + assert_eq!(record.cursor_revision, Some(1003)); + } + #[test] fn manual_transition_job_apply_worker_result_counts_preserves_existing_failure_reasons() { let options = ManualTransitionRunOptions::default(); @@ -2577,6 +2677,98 @@ mod tests { assert!(decoded.report.tier_failure_by_reason.is_empty()); } + #[test] + fn manual_transition_job_record_derives_revision_from_legacy_cursor() { + let options = ManualTransitionRunOptions::default(); + let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER); + let continuation_token = + encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string())); + record.update_running_progress( + ManualTransitionRunReport { + bucket: "bucket".to_string(), + scanned: 9, + continuation_token, + ..Default::default() + }, + ManualTransitionQueueSnapshot::default(), + ); + let encoded = record.encode().expect("job record should encode"); + let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json"); + value["job"] + .as_object_mut() + .expect("job should be object") + .remove("cursor_revision"); + let record_bytes = serde_json::to_vec(&value["job"]).expect("legacy job should encode"); + value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned)); + let legacy = serde_json::to_vec(&value).expect("legacy envelope should encode"); + + let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job should decode"); + + assert_eq!(decoded.cursor_revision, Some(9)); + } + + #[test] + fn manual_transition_job_record_omits_cursor_revision_for_old_readers() { + #[allow(dead_code)] + #[derive(serde::Deserialize)] + #[serde(deny_unknown_fields)] + struct LegacyPersistedManualTransitionJobRecord { + schema: String, + content_sha256: String, + job: LegacyManualTransitionJobRecord, + } + + #[allow(dead_code)] + #[derive(serde::Deserialize)] + #[serde(deny_unknown_fields)] + struct LegacyManualTransitionJobRecord { + job_id: Uuid, + scope_key: String, + bucket: String, + prefix: String, + tier: Option, + dry_run: bool, + max_objects: Option, + max_duration: Option, + owner_id: String, + lease_id: Uuid, + lease_expires_at_unix_nanos: i128, + state: ManualTransitionJobState, + scan_completed: bool, + cancel_requested: bool, + created_at_unix_nanos: i128, + updated_at_unix_nanos: i128, + completed_at_unix_nanos: Option, + report: ManualTransitionRunReport, + queue_snapshot: ManualTransitionQueueSnapshot, + error: Option, + } + + let options = ManualTransitionRunOptions::default(); + let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER); + let continuation_token = + encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string())); + record.update_running_progress( + ManualTransitionRunReport { + bucket: "bucket".to_string(), + scanned: 7, + continuation_token: continuation_token.clone(), + ..Default::default() + }, + ManualTransitionQueueSnapshot::default(), + ); + assert_eq!(record.cursor_revision, Some(7)); + + let encoded = record.encode().expect("job record should encode"); + let value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json"); + assert!(value["job"].get("cursor_revision").is_none()); + let legacy: LegacyPersistedManualTransitionJobRecord = + serde_json::from_slice(&encoded).expect("old reader should accept new job record"); + + assert_eq!(legacy.job.job_id, record.job_id); + assert_eq!(legacy.job.report.continuation_token, continuation_token); + } + #[test] fn manual_transition_job_record_rejects_unknown_report_fields() { let options = ManualTransitionRunOptions::default(); diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index 6d8e64f1b..20956ca70 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -16,6 +16,7 @@ pub mod bucket_lifecycle_audit; pub mod bucket_lifecycle_ops; mod config_boundary; pub mod core; +mod durable_namespace; pub mod evaluator; pub mod manual_transition_job; mod metadata_boundary; @@ -31,3 +32,8 @@ pub mod tier_free_version_recovery; pub mod tier_last_day_stats; pub mod tier_sweeper; pub mod transition_transaction; + +pub(crate) use durable_namespace::{ + DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record, + validate_durable_ilm_record, +}; diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index 6308b3767..22bc2ca70 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -20,6 +20,7 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, warn}; use crate::bucket::lifecycle::config_boundary; +use crate::bucket::lifecycle::durable_namespace::TIER_DELETE_JOURNAL_NAMESPACE; use crate::bucket::lifecycle::runtime_boundary; use crate::bucket::lifecycle::tier_sweeper::{ Jentry, TierDeleteJournalState, TierDeleteSourceIdentity, @@ -49,7 +50,7 @@ const TIER_DELETE_JOURNAL_VERSION: u8 = 2; const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3; const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4; const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5; -pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/"; +pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] @@ -432,6 +433,21 @@ async fn process_committed_tier_delete_journal_entry(api: Arc, je: &Jen ) .await?; } + let path = tier_delete_journal_object_name(je); + let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?; + let target_pool_indices = api + .record_durable_ilm_decommission_terminal_target_pools(&path, &data) + .await + .map_err(std::io::Error::other)?; + if let Some(target_pool_indices) = target_pool_indices { + for target_pool_idx in target_pool_indices { + match config_boundary::delete_config(api.pools[target_pool_idx].clone(), &path).await { + Ok(()) | Err(Error::ConfigNotFound) => {} + Err(err) => return Err(std::io::Error::other(err)), + } + } + return Ok(()); + } remove_tier_delete_journal_entry(api, je).await } diff --git a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs index f9831d394..70bf4ed53 100644 --- a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs +++ b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs @@ -21,6 +21,7 @@ use tracing::{debug, warn}; use uuid::Uuid; use crate::bucket::lifecycle::config_boundary; +use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE; use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; use crate::bucket::lifecycle::tier_sweeper::{ delete_confirmed_transition_candidate_exact_with_lease_idempotent, @@ -42,7 +43,7 @@ const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(6 const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300); pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1"; pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions"; -pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = "ilm/transition-transactions/records"; +pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix; pub const MAX_TRANSITION_TRANSACTION_SIZE: usize = 64 * 1024; pub type Result = std::result::Result; @@ -584,7 +585,8 @@ pub(crate) async fn save_transition_transaction_record( let object = transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?; let data = transaction.encode().map_err(transition_transaction_store_error)?; - config_boundary::save_config(api, &object, data).await + config_boundary::save_config(api.clone(), &object, data.clone()).await?; + api.record_durable_ilm_decommission_progress(&object, &data).await } pub(crate) async fn load_transition_transaction_record( @@ -596,8 +598,14 @@ pub(crate) async fn load_transition_transaction_record( TransitionTransaction::decode(transaction_id, &data).map_err(transition_transaction_store_error) } -pub(crate) async fn delete_transition_transaction_record(api: Arc, transaction_id: Uuid) -> EcstoreResult<()> { - let object = transition_transaction_record_object_name(transaction_id).map_err(transition_transaction_store_error)?; +pub(crate) async fn delete_transition_transaction_record( + api: Arc, + transaction: &TransitionTransaction, +) -> EcstoreResult<()> { + let object = + transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?; + let data = transaction.encode().map_err(transition_transaction_store_error)?; + api.record_durable_ilm_decommission_terminal(&object, &data).await?; match config_boundary::delete_config(api, &object).await { Ok(()) | Err(Error::ConfigNotFound) => Ok(()), Err(err) => Err(err), @@ -813,7 +821,7 @@ pub async fn finalize_missing_transition_transaction_for_operator( if probe != TransitionOperatorProbe::Missing { return Err(TransitionOperatorError::CandidateNotMissing(probe)); } - delete_transition_transaction_record(api, transaction_id) + delete_transition_transaction_record(api, &transaction) .await .map_err(TransitionOperatorError::Store) } @@ -849,22 +857,22 @@ pub async fn process_transition_transaction_record( match transaction.state { TransitionTransactionState::Uploaded => { delete_transition_remote_candidate(api.clone(), transaction).await?; - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted) } TransitionTransactionState::CleanupPending => match local_commit_matches_transaction(api.clone(), transaction).await { Ok(true) => { - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RecordDeleted) } Ok(false) => { delete_transition_remote_candidate(api.clone(), transaction).await?; - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted) } Err(err) if transition_source_is_missing(&err) => { delete_transition_remote_candidate(api.clone(), transaction).await?; - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted) } Err(err) => Err(err), @@ -872,7 +880,7 @@ pub async fn process_transition_transaction_record( TransitionTransactionState::LocalCommitStarted => { match local_commit_matches_transaction(api.clone(), transaction).await { Ok(true) => { - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RecordDeleted) } Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained), @@ -881,7 +889,7 @@ pub async fn process_transition_transaction_record( } } TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => { - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RecordDeleted) } TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await, @@ -907,7 +915,7 @@ async fn recover_unknown_upload_outcome( .map_err(Error::other)? { TransitionCandidateProbe::Missing => { - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RecordDeleted) } TransitionCandidateProbe::UnversionedPresent => { @@ -925,7 +933,7 @@ async fn recover_unknown_upload_outcome( ) .await .map_err(Error::other)?; - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted) } TransitionCandidateProbe::VersionedPresent(version_id) => { @@ -958,7 +966,7 @@ async fn cleanup_recovered_unknown_upload_candidate( .map_err(transition_transaction_store_error)?; save_transition_transaction_record(api.clone(), &cleanup).await?; delete_transition_remote_candidate(api.clone(), &cleanup).await?; - delete_transition_transaction_record(api, cleanup.transaction_id).await?; + delete_transition_transaction_record(api, &cleanup).await?; Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted) } diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 3b2729b8b..472c87e63 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -406,6 +406,25 @@ where Ok(data) } +pub(crate) async fn read_config_limited_preserve_empty(api: Arc, file: &str, max_bytes: usize) -> Result> +where + S: EcstoreObjectIO, +{ + let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?; + Ok(data) +} + +pub(crate) async fn read_config_limited_preserve_empty_with_metadata( + api: Arc, + file: &str, + max_bytes: usize, +) -> Result<(Vec, ObjectInfo)> +where + S: EcstoreObjectIO, +{ + read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await +} + /// Read an existing config object without treating an empty payload as absent. /// Callers that validate their own payload format need to distinguish corruption /// from `ConfigNotFound`. diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 0c39d1f57..16e4394a5 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -16,19 +16,23 @@ use crate::bucket::replication::replication_state_from_filemeta; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::bucket::{ lifecycle::{ - LifecycleExpiryConfigs, + DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, ValidatedDurableIlmRecord, bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::{ LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle, lifecycle_delete_all_versions_blocked_by_replication, }, - get_expiry_configs, + classify_durable_ilm_record, get_expiry_configs, lifecycle::IlmAction, + validate_durable_ilm_record, }, metadata_sys, }; use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw}; -use crate::config::com::{CONFIG_PREFIX, read_config, read_config_no_lock, save_config, save_config_with_opts}; +use crate::config::com::{ + CONFIG_PREFIX, delete_config, read_config, read_config_limited_preserve_empty, + read_config_limited_preserve_empty_with_metadata, read_config_no_lock, save_config, save_config_with_opts, +}; use crate::data_movement; use crate::data_movement::backpressure::{self, DataMovementOperation}; use crate::data_usage::DATA_USAGE_CACHE_NAME; @@ -48,8 +52,9 @@ use crate::storage_api_contracts::{ admin::StorageAdminApi, bucket::{BucketOperations, BucketOptions, MakeBucketOptions}, heal::HealOperations as _, + list::ListOperations as _, namespace::NamespaceLocking as _, - object::{EcstoreObjectIO, ObjectIO as _, ObjectOperations as _}, + object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _, ObjectOperations as _}, }; use crate::{core::sets::Sets, store::ECStore}; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; @@ -60,6 +65,7 @@ use rmp_serde::Deserializer; use rmp_serde::Serializer; use rustfs_common::heal_channel::HealOpts; use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; +use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum}; use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration}; use serde::{Deserialize, Serialize}; @@ -97,10 +103,18 @@ const DECOMMISSION_ENTRY_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_ENTRY_CONC const DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP: usize = 8; const DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP: usize = 64; const DECOMMISSION_ENTRY_WORKERS_PER_SET: usize = 2; +const DECOMMISSION_META_PREFIXES: [&str; 3] = [CONFIG_PREFIX, BUCKET_META_PREFIX, ILM_META_PREFIX]; const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30; const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3; const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5); const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1); +const DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT: &str = "decommission/ilm-receipts"; +const DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT: &str = "decommission/ilm-manifests"; +const DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA: &str = "v2"; +const DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA: &str = "v1"; +const DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE: usize = 16 * 1024; +const DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE: usize = 4 * 1024; +const DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS: usize = 3; /// Background decommission walks must tolerate slow object migrations; the /// stall timeout is the drive-health bound, not the total listing duration. const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); @@ -130,6 +144,11 @@ impl DecommissionCanceler { } } + #[cfg(test)] + pub(crate) fn new_for_test(token: CancellationToken) -> Self { + Self::new(token) + } + fn token(&self) -> &CancellationToken { &self.operation.token } @@ -377,6 +396,19 @@ fn is_decommission_meta_bucket(bucket: &DecomBucketInfo) -> bool { bucket.name == RUSTFS_META_BUCKET } +fn decommission_meta_buckets() -> [DecomBucketInfo; DECOMMISSION_META_PREFIXES.len()] { + DECOMMISSION_META_PREFIXES.map(|prefix| DecomBucketInfo { + name: RUSTFS_META_BUCKET.to_owned(), + prefix: prefix.to_owned(), + }) +} + +fn reconcile_decommission_meta_buckets(meta: &mut PoolMeta, idx: usize) -> bool { + let before = meta.pending_buckets(idx).len(); + meta.queue_buckets(idx, decommission_meta_buckets().into()); + meta.pending_buckets(idx).len() != before +} + fn split_decommission_buckets(buckets: Vec) -> (Vec, Vec) { let mut regular = Vec::with_capacity(buckets.len()); let mut meta = Vec::new(); @@ -1022,6 +1054,362 @@ fn resolve_decommission_partial_listing_entry( )) } +fn validate_decommission_durable_ilm_copy( + path: &str, + source_record: &ValidatedDurableIlmRecord, + target: &[u8], +) -> Result { + let target_record = validate_durable_ilm_record(path, target).map_err(|err| { + Error::other(format!( + "target durable ILM record is invalid at path `{path}` {}: {err}", + source_record.context() + )) + })?; + source_record + .checkpoint + .validate_successor(&target_record.checkpoint) + .map_err(|err| { + Error::other(format!( + "target durable ILM record generation mismatch at path `{path}` {}: {err}", + source_record.context() + )) + })?; + Ok(target_record) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct DecommissionDurableIlmReceipt { + source_path: String, + namespace: String, + id_kind: String, + id: String, + checkpoint: DurableIlmRecordCheckpoint, + terminal_checkpoint: Option, +} + +impl DecommissionDurableIlmReceipt { + fn new(path: &str, record: &ValidatedDurableIlmRecord) -> Self { + Self { + source_path: path.to_string(), + namespace: record.namespace.to_string(), + id_kind: record.id_kind.to_string(), + id: record.id.clone(), + checkpoint: record.checkpoint.clone(), + terminal_checkpoint: None, + } + } + + fn context(&self) -> String { + format!("namespace `{}` {} `{}`", self.namespace, self.id_kind, self.id) + } + + fn validate(&self) -> Result<()> { + let namespace = classify_durable_ilm_record(&self.source_path)? + .ok_or_else(|| Error::other(format!("receipt source path `{}` is not a durable ILM record", self.source_path)))?; + if namespace.name != self.namespace { + return Err(Error::other(format!( + "receipt namespace `{}` does not match source path `{}`", + self.namespace, self.source_path + ))); + } + if self.id_kind.is_empty() || self.id.is_empty() { + return Err(Error::other(format!( + "receipt identity is missing for source path `{}`", + self.source_path + ))); + } + if !is_sha256_checksum(self.checkpoint.content_sha256()) { + return Err(Error::other(format!( + "receipt target checksum is invalid for source path `{}` {}", + self.source_path, + self.context() + ))); + } + if let Some(terminal_checkpoint) = &self.terminal_checkpoint { + self.checkpoint.validate_successor(terminal_checkpoint).map_err(|err| { + Error::other(format!( + "receipt terminal checkpoint is invalid for source path `{}` {}: {err}", + self.source_path, + self.context() + )) + })?; + } + Ok(()) + } + + fn encode(&self) -> Result> { + let mut receipt = self.clone(); + receipt.checkpoint = receipt.checkpoint.compacted()?; + receipt.terminal_checkpoint = receipt + .terminal_checkpoint + .as_ref() + .map(DurableIlmRecordCheckpoint::compacted) + .transpose()?; + receipt.validate()?; + let receipt_bytes = serde_json::to_vec(&receipt)?; + let persisted = PersistedDecommissionDurableIlmReceipt { + schema: DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA.to_string(), + content_sha256: hex_sha256(&receipt_bytes, ToOwned::to_owned), + receipt, + }; + let encoded = serde_json::to_vec(&persisted)?; + if encoded.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE { + return Err(Error::other(format!( + "durable ILM receipt exceeds maximum size for source path `{}` {}", + self.source_path, + self.context() + ))); + } + Ok(encoded) + } + + fn decode(data: &[u8]) -> Result { + if data.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE { + return Err(Error::other("durable ILM receipt exceeds maximum size")); + } + let persisted: PersistedDecommissionDurableIlmReceipt = serde_json::from_slice(data)?; + if persisted.schema != DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA { + return Err(Error::other(format!("unsupported durable ILM receipt schema `{}`", persisted.schema))); + } + if !is_sha256_checksum(&persisted.content_sha256) { + return Err(Error::other("durable ILM receipt checksum is invalid")); + } + let receipt_bytes = serde_json::to_vec(&persisted.receipt)?; + let actual_checksum = hex_sha256(&receipt_bytes, ToOwned::to_owned); + if persisted.content_sha256 != actual_checksum { + return Err(Error::other("durable ILM receipt checksum mismatch")); + } + persisted.receipt.validate()?; + Ok(persisted.receipt) + } +} + +fn merge_decommission_durable_ilm_receipts( + existing: &DecommissionDurableIlmReceipt, + incoming: &DecommissionDurableIlmReceipt, +) -> Result { + if existing.source_path != incoming.source_path + || existing.namespace != incoming.namespace + || existing.id_kind != incoming.id_kind + || existing.id != incoming.id + { + return Err(Error::other(format!( + "durable ILM receipt identity conflict for source path `{}` {}; incoming {}", + existing.source_path, + existing.context(), + incoming.context() + ))); + } + + let checkpoint = + if existing.checkpoint == incoming.checkpoint || incoming.checkpoint.validate_successor(&existing.checkpoint).is_ok() { + existing.checkpoint.clone() + } else { + existing.checkpoint.validate_successor(&incoming.checkpoint).map_err(|err| { + Error::other(format!( + "durable ILM receipt checkpoint conflict for source path `{}` {}: {err}", + existing.source_path, + existing.context() + )) + })?; + incoming.checkpoint.clone() + }; + let terminal_checkpoint = match (&existing.terminal_checkpoint, &incoming.terminal_checkpoint) { + (Some(existing_terminal), Some(incoming_terminal)) if existing_terminal == incoming_terminal => { + Some(existing_terminal.clone()) + } + (Some(existing_terminal), Some(incoming_terminal)) if incoming_terminal.validate_successor(existing_terminal).is_ok() => { + Some(existing_terminal.clone()) + } + (Some(existing_terminal), Some(incoming_terminal)) => { + existing_terminal.validate_successor(incoming_terminal).map_err(|err| { + Error::other(format!( + "durable ILM receipt terminal checkpoint conflict for source path `{}` {}: {err}", + existing.source_path, + existing.context() + )) + })?; + Some(incoming_terminal.clone()) + } + (Some(existing_terminal), None) => Some(existing_terminal.clone()), + (None, Some(incoming_terminal)) => Some(incoming_terminal.clone()), + (None, None) => None, + }; + let merged = DecommissionDurableIlmReceipt { + source_path: existing.source_path.clone(), + namespace: existing.namespace.clone(), + id_kind: existing.id_kind.clone(), + id: existing.id.clone(), + checkpoint, + terminal_checkpoint, + }; + merged.validate()?; + Ok(merged) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedDecommissionDurableIlmReceipt { + schema: String, + content_sha256: String, + receipt: DecommissionDurableIlmReceipt, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct DecommissionDurableIlmManifest { + schema: String, + run_token: String, + receipt_count: u64, + receipt_paths_sha256: String, +} + +impl DecommissionDurableIlmManifest { + fn new(run_token: &str, receipt_paths: &[String]) -> Result { + let manifest = Self { + schema: DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA.to_string(), + run_token: run_token.to_string(), + receipt_count: u64::try_from(receipt_paths.len()) + .map_err(|_| Error::other("durable ILM expected manifest receipt count exceeds u64"))?, + receipt_paths_sha256: decommission_durable_ilm_manifest_paths_sha256(receipt_paths)?, + }; + manifest.validate(run_token, receipt_paths)?; + Ok(manifest) + } + + fn validate(&self, run_token: &str, receipt_paths: &[String]) -> Result<()> { + if self.schema != DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA { + return Err(Error::other(format!( + "unsupported durable ILM expected manifest schema `{}`", + self.schema + ))); + } + if self.run_token != run_token || !is_sha256_checksum(&self.run_token) { + return Err(Error::other("durable ILM expected manifest run token is invalid")); + } + let receipt_count = u64::try_from(receipt_paths.len()) + .map_err(|_| Error::other("durable ILM expected manifest receipt count exceeds u64"))?; + if self.receipt_count != receipt_count { + return Err(Error::other(format!( + "durable ILM expected manifest receipt count mismatch: expected {}, found {receipt_count}", + self.receipt_count + ))); + } + if !is_sha256_checksum(&self.receipt_paths_sha256) + || self.receipt_paths_sha256 != decommission_durable_ilm_manifest_paths_sha256(receipt_paths)? + { + return Err(Error::other("durable ILM expected manifest receipt paths checksum mismatch")); + } + Ok(()) + } + + fn encode(&self) -> Result> { + let encoded = serde_json::to_vec(self)?; + if encoded.len() > DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE { + return Err(Error::other("durable ILM expected manifest exceeds maximum size")); + } + Ok(encoded) + } + + fn decode(data: &[u8], run_token: &str, receipt_paths: &[String]) -> Result { + if data.len() > DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE { + return Err(Error::other("durable ILM expected manifest exceeds maximum size")); + } + let manifest: Self = serde_json::from_slice(data)?; + manifest.validate(run_token, receipt_paths)?; + Ok(manifest) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct DecommissionDurableIlmReceiptLocator { + run_token: String, + source_path: String, + id_kind: String, + id: String, +} + +impl DecommissionDurableIlmReceiptLocator { + fn context(&self) -> String { + format!("source path `{}` {} `{}`", self.source_path, self.id_kind, self.id) + } +} + +fn decommission_durable_ilm_receipt_run_token(cmd_line: &str, start_time: OffsetDateTime) -> String { + let identity = format!("{cmd_line}\0{}", start_time.unix_timestamp_nanos()); + hex_sha256(identity.as_bytes(), ToOwned::to_owned) +} + +fn decommission_durable_ilm_receipt_run_prefix(run_token: &str) -> String { + format!("{DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT}/{run_token}/") +} + +fn decommission_durable_ilm_receipt_path(run_token: &str, source_path: &str, id_kind: &str, id: &str) -> String { + format!( + "{}{}/{}/{}.json", + decommission_durable_ilm_receipt_run_prefix(run_token), + source_path, + id_kind, + id + ) +} + +fn decommission_durable_ilm_manifest_path(run_token: &str) -> String { + format!("{DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT}/{run_token}.json") +} + +fn decommission_durable_ilm_manifest_paths_sha256(receipt_paths: &[String]) -> Result { + let mut sorted_paths = receipt_paths.iter().map(String::as_str).collect::>(); + sorted_paths.sort_unstable(); + let encoded = serde_json::to_vec(&sorted_paths)?; + Ok(hex_sha256(&encoded, ToOwned::to_owned)) +} + +fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result { + let prefix = format!("{DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT}/"); + let suffix = path + .strip_prefix(&prefix) + .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has the wrong root")))?; + let (run_token, record_path) = suffix + .split_once('/') + .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record path")))?; + let mut parts = record_path.rsplitn(3, '/'); + let id = parts + .next() + .and_then(|file| file.strip_suffix(".json")) + .filter(|id| !id.is_empty()) + .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record id")))?; + let id_kind = parts + .next() + .filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id")) + .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has an invalid id kind")))?; + let source_path = parts + .next() + .filter(|source_path| !source_path.is_empty()) + .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its source path")))?; + if !is_sha256_checksum(run_token) { + return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid run token"))); + } + match id_kind { + "operation_id" if !is_sha256_checksum(id) => { + return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid operation id"))); + } + "transaction_id" | "job_id" if uuid::Uuid::parse_str(id).is_err() => { + return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid UUID"))); + } + _ => {} + } + classify_durable_ilm_record(source_path)? + .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` does not identify a durable ILM source path")))?; + Ok(DecommissionDurableIlmReceiptLocator { + run_token: run_token.to_string(), + source_path: source_path.to_string(), + id_kind: id_kind.to_string(), + id: id.to_string(), + }) +} + fn resolve_decommission_pool_meta_reload_result(result: Result<()>, stage: &str) -> Result<()> { result.map_err(|err| Error::other(format!("decommission pool meta reload failed during {stage}: {err}"))) } @@ -3101,6 +3489,8 @@ impl ECStore { async fn decommission_cancel_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> { ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?; let _start_guard = self.start_gate.lock().await; + let operation_gate = self.ctx.decommission_operation_gate(); + let operation_guard = operation_gate.write().await; // Lock order: decommission_cancelers before pool_meta. Holding both makes // owner validation and the terminal transition one atomic operation. @@ -3159,8 +3549,6 @@ impl ECStore { ); } - self.wait_for_decommission_side_effects().await; - if should_save_pool_meta && let Err(err) = self.save_current_pool_meta().await { if let Some(previous_pool_meta) = previous_pool_meta { let mut pool_meta = self.pool_meta.write().await; @@ -3168,6 +3556,7 @@ impl ECStore { } return Err(err); } + drop(operation_guard); if let Some(canceler) = terminal_canceler.as_ref() { self.release_decommission_canceler_slot(idx, canceler).await; @@ -3226,24 +3615,24 @@ impl ECStore { } async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result { - // Serialize promotion and generation capture with clear/restart transitions. - let (promoted, generation, save_error) = { + let (changed, generation, save_error) = { let _start_guard = self.start_gate.lock().await; let mut pool_meta = self.pool_meta.write().await; if pool_meta.pools.get(idx).is_none() { return Err(Error::other("failed to start decommission: target pool was not found")); } + let reconciled = reconcile_decommission_meta_buckets(&mut pool_meta, idx); let promoted = pool_meta.promote_queued_decommission(idx); + let changed = reconciled || promoted; drop(pool_meta); - let save_error = if promoted { + let save_error = if changed { self.save_current_pool_meta().await.err() } else { None }; - let generation = self.active_decommission_generation(idx).await?; - (promoted, generation, save_error) + (changed, generation, save_error) }; if let Some(err) = save_error { @@ -3255,7 +3644,7 @@ impl ECStore { return Err(err); } - if promoted && let Some(notification_sys) = runtime_sources::notification_sys() { + if changed && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("promote_queued_decommission for pool {idx}"); if let Err(err) = resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) @@ -3272,6 +3661,12 @@ impl ECStore { Ok(generation) } + #[cfg(test)] + pub(crate) async fn promote_queued_decommission_for_test(&self, idx: usize) -> Result<()> { + let owner = DecommissionCanceler::new(CancellationToken::new()); + self.promote_queued_decommission(idx, &owner).await.map(|_| ()) + } + async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> { let changed = { let mut pool_meta = self.pool_meta.write().await; @@ -3812,6 +4207,12 @@ impl ECStore { ); return Ok(()); } + let durable_ilm_record = if bucket == RUSTFS_META_BUCKET { + classify_durable_ilm_record(&entry.name) + .map_err(|err| with_decommission_entry_context("durable_ilm_namespace", &bucket, &entry.name, err))? + } else { + None + }; if self.decommission_cancel_requested(idx, &rx).await { rx.cancel(); } @@ -4138,7 +4539,8 @@ impl ECStore { } } - if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) { + if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) && durable_ilm_record.is_none() + { if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) { return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup")); } @@ -4193,6 +4595,17 @@ impl ECStore { }) .await; resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())? + } else if durable_ilm_record.is_some() { + debug!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %entry.name, + state = "retained_for_final_verification", + "Decommission durable ILM source retained for final verification" + ); } else if decommissioned != fivs.versions.len() || expired > 0 { warn!( event = EVENT_DECOMMISSION_ENTRY, @@ -4442,6 +4855,18 @@ impl ECStore { Ok(()) } + #[cfg(test)] + pub(crate) async fn decommission_pool_for_test( + self: &Arc, + rx: CancellationToken, + idx: usize, + pool: Arc, + bucket: DecomBucketInfo, + ) -> Result<()> { + self.decommission_pool(rx, idx, pool, bucket, Arc::new(Semaphore::new(decommission_entry_concurrency_limit()))) + .await + } + #[tracing::instrument(skip(self, canceler))] pub async fn do_decommission_in_routine( self: &Arc, @@ -4602,7 +5027,10 @@ impl ECStore { state = "verifying_completion", "Decommission completion verification started" ); - if let Err(err) = self.check_after_decommission(idx).await { + if let Err(err) = self.check_after_decommission(idx, &rx, generation).await { + if is_err_operation_canceled(&err) { + return Err(err); + } resolve_decommission_terminal_mark_result( self.decommission_failed_for_operation(idx, canceler).await, "failed", @@ -4612,13 +5040,6 @@ impl ECStore { "failed to finalize decommission for pool {cmd_line}: post-check failed: {err}" ))); } - - if self.decommission_cancel_requested(idx, &rx).await { - rx.cancel(); - } - decommission_cancel_signal_result(rx.is_cancelled())?; - self.ensure_decommission_generation_current(idx, generation).await?; - info!( event = EVENT_DECOMMISSION_STATE, component = LOG_COMPONENT_ECSTORE, @@ -4628,11 +5049,14 @@ impl ECStore { state = "marking_completed", "Decommission marking completed state" ); - resolve_decommission_terminal_mark_result( - self.complete_decommission_for_operation(idx, canceler).await, - "completed", - &cmd_line, - )?; + if let Err(err) = self.complete_decommission_for_operation(idx, canceler).await { + resolve_decommission_terminal_mark_result( + self.decommission_failed_for_operation(idx, canceler).await, + "failed", + &cmd_line, + )?; + return Err(Error::other(format!("failed to finalize decommission for pool {cmd_line}: {err}"))); + } } DecommissionFinalState::Failed => { warn!( @@ -4774,11 +5198,18 @@ impl ECStore { async fn complete_decommission_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> { ensure_decommission_terminal_operation_supported(self.single_pool(), "complete decommission")?; + ensure_valid_decommission_pool_index(self.pools.len(), idx)?; + if let Some(owner) = owner { + let cancelers = self.decommission_cancelers.read().await; + if !decommission_canceler_is_owned_by(cancelers.as_slice(), idx, owner) { + owner.release(); + return Ok(()); + } + } + self.verify_decommission_durable_ilm_receipts(idx).await?; let _start_guard = self.start_gate.lock().await; - // Lock order: decommission_cancelers before pool_meta. Holding both makes - // owner validation and the terminal transition one atomic operation. - let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = { + let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = { let cancelers = self.decommission_cancelers.read().await; let mut pool_meta = self.pool_meta.write().await; let previous_pool_meta = pool_meta.clone(); @@ -4789,12 +5220,17 @@ impl ECStore { else { return Ok(()); }; + let completed = pool_meta + .pools + .get(idx) + .and_then(|pool| pool.decommission.as_ref()) + .is_some_and(|decommission| decommission.complete); let terminal_canceler = if let Some(owner) = owner { Some(owner.clone()) } else { cancelers.get(idx).and_then(Option::as_ref).cloned() }; - (changed, changed.then_some(previous_pool_meta), terminal_canceler) + (changed, completed, changed.then_some(previous_pool_meta), terminal_canceler) }; if should_reload_pool_meta && let Err(err) = self.save_current_pool_meta().await { @@ -4846,6 +5282,18 @@ impl ECStore { } } + if completed && let Err(err) = self.cleanup_decommission_durable_ilm_receipts(idx).await { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "receipt_cleanup_failed", + error = %err, + "Decommission durable ILM receipt cleanup failed" + ); + } + Ok(()) } @@ -5007,17 +5455,16 @@ impl ECStore { let decom_buckets = self.get_buckets_to_decommission().await?; + let mut healed_buckets = HashSet::with_capacity(decom_buckets.len()); for bk in decom_buckets.iter() { - resolve_decommission_preflight_heal_result(&bk.name, self.heal_bucket(&bk.name, &HealOpts::default()).await)?; + if healed_buckets.insert(bk.name.as_str()) { + resolve_decommission_preflight_heal_result(&bk.name, self.heal_bucket(&bk.name, &HealOpts::default()).await)?; + } } - let meta_buckets = [ - path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(CONFIG_PREFIX)]), - path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(BUCKET_META_PREFIX)]), - ]; - let meta_bucket_opts = decommission_meta_bucket_options(); - for bk in meta_buckets.iter() { + for prefix in DECOMMISSION_META_PREFIXES { + let bk = path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(prefix)]); if let Err(err) = self .make_bucket(bk.to_string_lossy().to_string().as_str(), &meta_bucket_opts) .await @@ -5136,19 +5583,955 @@ impl ECStore { }) .collect(); - ret.push(DecomBucketInfo { - name: RUSTFS_META_BUCKET.to_owned(), - prefix: CONFIG_PREFIX.to_owned(), - }); - ret.push(DecomBucketInfo { - name: RUSTFS_META_BUCKET.to_owned(), - prefix: BUCKET_META_PREFIX.to_owned(), - }); + ret.extend(decommission_meta_buckets()); Ok(ret) } - async fn check_after_decommission(self: &Arc, idx: usize) -> Result<()> { + async fn durable_ilm_receipt_run_token(&self, source_pool_idx: usize) -> Result { + let pool_meta = self.pool_meta.read().await; + let pool = pool_meta + .pools + .get(source_pool_idx) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), source_pool_idx))?; + let start_time = pool + .decommission + .as_ref() + .and_then(|info| info.start_time) + .ok_or_else(|| Error::other(format!("decommission run identity is missing for pool {source_pool_idx}")))?; + Ok(decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time)) + } + + async fn load_decommissioned_durable_ilm_target( + &self, + source_pool_idx: usize, + path: &str, + max_record_size: usize, + record_context: &str, + ) -> Result)>> { + let mut target = None::<(usize, Vec)>; + let mut first_read_error = None; + for (target_pool_idx, pool) in self.pools.iter().enumerate() { + if target_pool_idx == source_pool_idx { + continue; + } + match read_config_limited_preserve_empty(pool.clone(), path, max_record_size).await { + Ok(data) => { + if let Some((existing_pool_idx, existing)) = target.as_ref() + && existing != &data + { + return Err(Error::other(format!( + "divergent target durable ILM records at path `{path}` {record_context} in pools {existing_pool_idx} and {target_pool_idx}" + ))); + } + target = Some((target_pool_idx, data)); + } + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => {} + Err(err) => { + first_read_error.get_or_insert_with(|| { + Error::other(format!( + "failed to read target durable ILM record at path `{path}` {record_context} from pool {target_pool_idx}: {err}" + )) + }); + } + } + } + + if let Some(err) = first_read_error { + return Err(err); + } + Ok(target) + } + + async fn list_decommission_durable_ilm_receipt_paths_in_pool(&self, pool_idx: usize, prefix: &str) -> Result> { + let pool = self + .pools + .get(pool_idx) + .ok_or_else(|| invalid_decommission_pool_index_error(self.pools.len(), pool_idx))?; + let mut receipts = Vec::new(); + let mut continuation = None; + loop { + let page = pool + .clone() + .list_objects_v2(RUSTFS_META_BUCKET, prefix, continuation, None, 1000, false, None, false) + .await + .map_err(|err| { + Error::other(format!( + "failed to list durable ILM decommission receipts under `{prefix}` in pool {pool_idx}: {err}" + )) + })?; + receipts.extend(page.objects.into_iter().map(|object| object.name)); + if !page.is_truncated { + break; + } + continuation = Some(page.next_continuation_token.ok_or_else(|| { + Error::other(format!( + "durable ILM decommission receipt listing under `{prefix}` in pool {pool_idx} was truncated without a continuation token" + )) + })?); + } + Ok(receipts) + } + + async fn list_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result> { + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let prefix = decommission_durable_ilm_receipt_run_prefix(&run_token); + let mut receipts = Vec::new(); + for pool_idx in 0..self.pools.len() { + if pool_idx == source_pool_idx { + continue; + } + for receipt_path in self + .list_decommission_durable_ilm_receipt_paths_in_pool(pool_idx, &prefix) + .await? + { + let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?; + if locator.run_token != run_token { + return Err(Error::other(format!( + "durable ILM receipt path `{receipt_path}` has an unexpected run token" + ))); + } + receipts.push((pool_idx, receipt_path)); + } + } + Ok(receipts) + } + + async fn list_decommission_durable_ilm_manifest_receipts(&self, source_pool_idx: usize) -> Result> { + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let prefix = decommission_durable_ilm_receipt_run_prefix(&run_token); + let receipt_paths = self + .list_decommission_durable_ilm_receipt_paths_in_pool(source_pool_idx, &prefix) + .await?; + for receipt_path in &receipt_paths { + let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?; + if locator.run_token != run_token { + return Err(Error::other(format!( + "durable ILM expected manifest receipt path `{receipt_path}` has an unexpected run token" + ))); + } + } + Ok(receipt_paths) + } + + async fn persist_decommission_durable_ilm_manifest(&self, source_pool_idx: usize) -> Result<()> { + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let receipt_paths = self.list_decommission_durable_ilm_manifest_receipts(source_pool_idx).await?; + for receipt_path in &receipt_paths { + self.read_decommission_durable_ilm_receipt(source_pool_idx, receipt_path) + .await?; + } + let manifest = DecommissionDurableIlmManifest::new(&run_token, &receipt_paths)?; + let manifest_path = decommission_durable_ilm_manifest_path(&run_token); + let encoded = manifest.encode()?; + let mut attempt = 1; + loop { + match read_config_limited_preserve_empty( + self.pools[source_pool_idx].clone(), + &manifest_path, + DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE, + ) + .await + { + Ok(existing) => { + DecommissionDurableIlmManifest::decode(&existing, &run_token, &receipt_paths).map_err(|err| { + Error::other(format!( + "durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx} is invalid: {err}" + )) + })?; + return Ok(()); + } + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => {} + Err(err) => { + return Err(Error::other(format!( + "failed to read durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}" + ))); + } + } + match save_config_with_opts( + self.pools[source_pool_idx].clone(), + &manifest_path, + encoded.clone(), + &ObjectOptions { + max_parity: true, + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + { + Ok(()) => return Ok(()), + Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => { + attempt += 1; + } + Err(Error::PreconditionFailed) => { + return Err(Error::other(format!( + "failed to persist durable ILM expected manifest `{manifest_path}` after concurrent updates" + ))); + } + Err(err) => { + return Err(Error::other(format!( + "failed to persist durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx}: {err}" + ))); + } + } + } + } + + async fn load_decommission_durable_ilm_manifest( + &self, + source_pool_idx: usize, + ) -> Result> { + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let receipt_paths = self.list_decommission_durable_ilm_manifest_receipts(source_pool_idx).await?; + let manifest_path = decommission_durable_ilm_manifest_path(&run_token); + let data = read_config_limited_preserve_empty( + self.pools[source_pool_idx].clone(), + &manifest_path, + DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE, + ) + .await + .map_err(|err| { + Error::other(format!( + "failed to read durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}" + )) + })?; + DecommissionDurableIlmManifest::decode(&data, &run_token, &receipt_paths).map_err(|err| { + Error::other(format!( + "durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx} is invalid: {err}" + )) + })?; + + let mut receipts = HashMap::with_capacity(receipt_paths.len()); + for receipt_path in receipt_paths { + let receipt = self + .read_decommission_durable_ilm_receipt(source_pool_idx, &receipt_path) + .await?; + if receipts.insert(receipt_path.clone(), receipt).is_some() { + return Err(Error::other(format!( + "durable ILM expected manifest contains duplicate receipt path `{receipt_path}`" + ))); + } + } + Ok(receipts) + } + + async fn persist_decommission_durable_ilm_receipt( + &self, + source_pool_idx: usize, + target_pool_idx: usize, + receipt: &DecommissionDurableIlmReceipt, + ) -> Result<()> { + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let receipt_path = decommission_durable_ilm_receipt_path(&run_token, &receipt.source_path, &receipt.id_kind, &receipt.id); + let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?; + let mut attempt = 1; + loop { + let (merged, http_preconditions) = match read_config_limited_preserve_empty_with_metadata( + self.pools[target_pool_idx].clone(), + &receipt_path, + DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, + ) + .await + { + Ok((existing_data, metadata)) => { + let existing = DecommissionDurableIlmReceipt::decode(&existing_data).map_err(|err| { + Error::other(format!( + "durable ILM decommission receipt `{receipt_path}` in pool {target_pool_idx} for {} is invalid: {err}", + locator.context() + )) + })?; + Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &existing)?; + let merged = merge_decommission_durable_ilm_receipts(&existing, receipt)?; + if merged == existing { + return Ok(()); + } + let etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| { + Error::other(format!( + "durable ILM decommission receipt `{receipt_path}` in pool {target_pool_idx} is missing an ETag" + )) + })?; + ( + merged, + HTTPPreconditions { + if_match: Some(etag), + ..Default::default() + }, + ) + } + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => + { + ( + receipt.clone(), + HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }, + ) + } + Err(err) => { + return Err(Error::other(format!( + "failed to read durable ILM decommission receipt `{receipt_path}` from pool {target_pool_idx} for {}: {err}", + locator.context() + ))); + } + }; + let encoded = merged.encode().map_err(|err| { + Error::other(format!( + "failed to encode durable ILM decommission receipt `{receipt_path}` for source path `{}` {}: {err}", + receipt.source_path, + receipt.context() + )) + })?; + match save_config_with_opts( + self.pools[target_pool_idx].clone(), + &receipt_path, + encoded, + &ObjectOptions { + max_parity: true, + http_preconditions: Some(http_preconditions), + ..Default::default() + }, + ) + .await + { + Ok(()) => return Ok(()), + Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => { + attempt += 1; + } + Err(Error::PreconditionFailed) => { + return Err(Error::other(format!( + "failed to persist durable ILM decommission receipt `{receipt_path}` for {} after concurrent updates", + locator.context() + ))); + } + Err(err) => { + return Err(Error::other(format!( + "failed to persist durable ILM decommission receipt `{receipt_path}` for {}: {err}", + locator.context() + ))); + } + } + } + } + + fn validate_decommission_durable_ilm_receipt_locator( + receipt_path: &str, + locator: &DecommissionDurableIlmReceiptLocator, + receipt: &DecommissionDurableIlmReceipt, + ) -> Result<()> { + if locator.source_path != receipt.source_path || locator.id_kind != receipt.id_kind || locator.id != receipt.id { + return Err(Error::other(format!( + "durable ILM decommission receipt path `{receipt_path}` identity {} does not match receipt {}", + locator.context(), + receipt.context() + ))); + } + Ok(()) + } + + async fn read_decommission_durable_ilm_receipt( + &self, + receipt_pool_idx: usize, + receipt_path: &str, + ) -> Result { + let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?; + let data = read_config_limited_preserve_empty( + self.pools[receipt_pool_idx].clone(), + receipt_path, + DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, + ) + .await + .map_err(|err| { + Error::other(format!( + "failed to read durable ILM decommission receipt `{receipt_path}` from pool {receipt_pool_idx} for {}: {err}", + locator.context() + )) + })?; + let receipt = DecommissionDurableIlmReceipt::decode(&data).map_err(|err| { + Error::other(format!( + "durable ILM decommission receipt `{receipt_path}` in pool {receipt_pool_idx} for {} is invalid: {err}", + locator.context() + )) + })?; + Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?; + Ok(receipt) + } + + async fn load_decommission_durable_ilm_terminal_receipt( + &self, + source_pool_idx: usize, + path: &str, + source_record: &ValidatedDurableIlmRecord, + ) -> Result> { + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, source_record.id_kind, &source_record.id); + let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?; + let mut proof = None::; + for pool_idx in 0..self.pools.len() { + if pool_idx == source_pool_idx { + continue; + } + let data = match read_config_limited_preserve_empty( + self.pools[pool_idx].clone(), + &receipt_path, + DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, + ) + .await + { + Ok(data) => data, + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => + { + continue; + } + Err(err) => { + return Err(Error::other(format!( + "failed to read terminal durable ILM decommission receipt `{receipt_path}` from pool {pool_idx} for {}: {err}", + source_record.context() + ))); + } + }; + let receipt = DecommissionDurableIlmReceipt::decode(&data).map_err(|err| { + Error::other(format!( + "terminal durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} for {} is invalid: {err}", + source_record.context() + )) + })?; + Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?; + if receipt.namespace != source_record.namespace + || receipt.id_kind != source_record.id_kind + || receipt.id != source_record.id + { + return Err(Error::other(format!( + "terminal durable ILM decommission receipt identity mismatch at path `{path}` {}; receipt {}", + source_record.context(), + receipt.context() + ))); + } + source_record + .checkpoint + .validate_successor(&receipt.checkpoint) + .map_err(|err| { + Error::other(format!( + "terminal durable ILM decommission receipt does not cover source at path `{path}` {}: {err}", + source_record.context() + )) + })?; + if receipt.terminal_checkpoint.is_some() { + proof = Some(match proof { + Some(existing) => merge_decommission_durable_ilm_receipts(&existing, &receipt)?, + None => receipt, + }); + } + } + Ok(proof) + } + + async fn verify_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> { + let expected_receipts = self.load_decommission_durable_ilm_manifest(source_pool_idx).await?; + let receipt_paths = self.list_decommission_durable_ilm_receipts(source_pool_idx).await?; + let present_receipt_paths = receipt_paths + .iter() + .map(|(_, receipt_path)| receipt_path.as_str()) + .collect::>(); + for (expected_path, expected) in &expected_receipts { + if !present_receipt_paths.contains(expected_path.as_str()) { + return Err(Error::other(format!( + "durable ILM decommission receipt is missing at `{expected_path}` for source path `{}` {}", + expected.source_path, + expected.context() + ))); + } + } + + for (receipt_pool_idx, receipt_path) in receipt_paths { + let expected = expected_receipts.get(&receipt_path).ok_or_else(|| { + Error::other(format!( + "durable ILM decommission receipt `{receipt_path}` in pool {receipt_pool_idx} is absent from the expected manifest" + )) + })?; + let receipt = self + .read_decommission_durable_ilm_receipt(receipt_pool_idx, &receipt_path) + .await?; + if receipt.source_path != expected.source_path + || receipt.namespace != expected.namespace + || receipt.id_kind != expected.id_kind + || receipt.id != expected.id + { + return Err(Error::other(format!( + "durable ILM decommission receipt identity mismatch at `{receipt_path}` for source path `{}` {}; decoded {}", + expected.source_path, + expected.context(), + receipt.context() + ))); + } + expected.checkpoint.validate_successor(&receipt.checkpoint).map_err(|err| { + Error::other(format!( + "durable ILM decommission receipt generation mismatch at `{receipt_path}` for source path `{}` {}: {err}", + expected.source_path, + expected.context() + )) + })?; + match (&expected.terminal_checkpoint, &receipt.terminal_checkpoint) { + (Some(expected_terminal), Some(receipt_terminal)) => { + expected_terminal.validate_successor(receipt_terminal).map_err(|err| { + Error::other(format!( + "durable ILM decommission terminal receipt generation mismatch at `{receipt_path}` for source path `{}` {}: {err}", + expected.source_path, + expected.context() + )) + })?; + } + (Some(_), None) => { + return Err(Error::other(format!( + "durable ILM decommission terminal receipt is missing at `{receipt_path}` for source path `{}` {}", + expected.source_path, + expected.context() + ))); + } + (None, _) => {} + } + let namespace = classify_durable_ilm_record(&receipt.source_path)? + .ok_or_else(|| Error::other(format!("path `{}` is not a durable ILM record", receipt.source_path)))?; + let target = self + .load_decommissioned_durable_ilm_target( + source_pool_idx, + &receipt.source_path, + namespace.max_record_size, + &receipt.context(), + ) + .await?; + if let Some((_, target)) = target { + let target_record = validate_durable_ilm_record(&receipt.source_path, &target).map_err(|err| { + Error::other(format!( + "target durable ILM record is invalid at path `{}` {}: {err}", + receipt.source_path, + receipt.context() + )) + })?; + let identity_matches = target_record.namespace == receipt.namespace + && target_record.id_kind == receipt.id_kind + && target_record.id == receipt.id; + let reused_manual_scope = receipt.terminal_checkpoint.is_some() + && matches!( + (&receipt.checkpoint, &target_record.checkpoint), + ( + DurableIlmRecordCheckpoint::ManualTransitionScope { .. }, + DurableIlmRecordCheckpoint::ManualTransitionScope { .. } + ) + ); + if !identity_matches && !reused_manual_scope { + return Err(Error::other(format!( + "target durable ILM record identity mismatch at path `{}` {}; decoded {}", + receipt.source_path, + receipt.context(), + target_record.context() + ))); + } + if identity_matches { + receipt + .terminal_checkpoint + .as_ref() + .unwrap_or(&receipt.checkpoint) + .validate_successor(&target_record.checkpoint) + .map_err(|err| { + Error::other(format!( + "target durable ILM record generation mismatch at path `{}` {}: {err}", + receipt.source_path, + receipt.context() + )) + })?; + } + } else if receipt.terminal_checkpoint.is_none() { + return Err(Error::other(format!( + "target durable ILM record is missing at path `{}` {} without a recovery terminal checkpoint", + receipt.source_path, + receipt.context() + ))); + } + } + Ok(()) + } + + async fn advance_durable_ilm_decommission_receipt( + &self, + pool_idx: usize, + receipt_path: &str, + record: &ValidatedDurableIlmRecord, + terminal: bool, + ) -> Result { + let stage = if terminal { "terminal" } else { "progress" }; + let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?; + let mut attempt = 1; + loop { + let (receipt_data, metadata) = match read_config_limited_preserve_empty_with_metadata( + self.pools[pool_idx].clone(), + receipt_path, + DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, + ) + .await + { + Ok(receipt) => receipt, + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => + { + return Ok(false); + } + Err(err) => { + return Err(Error::other(format!( + "failed to read durable ILM decommission receipt `{receipt_path}` from pool {pool_idx} for {}: {err}", + locator.context() + ))); + } + }; + let mut receipt = DecommissionDurableIlmReceipt::decode(&receipt_data).map_err(|err| { + Error::other(format!( + "durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} for {} is invalid: {err}", + locator.context() + )) + })?; + Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?; + receipt.checkpoint.validate_successor(&record.checkpoint).map_err(|err| { + Error::other(format!( + "{stage} durable ILM record generation mismatch at path `{}` {}: {err}", + receipt.source_path, + receipt.context() + )) + })?; + + if terminal { + if let Some(existing) = &receipt.terminal_checkpoint { + if existing == &record.checkpoint || record.checkpoint.validate_successor(existing).is_ok() { + return Ok(true); + } + existing.validate_successor(&record.checkpoint).map_err(|err| { + Error::other(format!( + "terminal durable ILM record checkpoint conflicts at path `{}` {}: {err}", + receipt.source_path, + receipt.context() + )) + })?; + } + receipt.terminal_checkpoint = Some(record.checkpoint.clone()); + } else { + if let Some(existing) = &receipt.terminal_checkpoint { + if existing == &record.checkpoint { + return Ok(true); + } + existing.validate_successor(&record.checkpoint).map_err(|err| { + Error::other(format!( + "progress durable ILM record conflicts with terminal checkpoint at path `{}` {}: {err}", + receipt.source_path, + receipt.context() + )) + })?; + receipt.terminal_checkpoint = None; + } + if receipt.checkpoint == record.checkpoint { + return Ok(true); + } + receipt.checkpoint = record.checkpoint.clone(); + } + + let etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| { + Error::other(format!( + "durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} is missing an ETag" + )) + })?; + let encoded = receipt.encode()?; + match save_config_with_opts( + self.pools[pool_idx].clone(), + receipt_path, + encoded, + &ObjectOptions { + max_parity: true, + http_preconditions: Some(HTTPPreconditions { + if_match: Some(etag), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + { + Ok(()) => return Ok(true), + Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => { + attempt += 1; + continue; + } + Err(Error::PreconditionFailed) => { + return Err(Error::other(format!( + "failed to persist {stage} durable ILM decommission receipt `{receipt_path}` for {} after concurrent updates", + locator.context() + ))); + } + Err(err) => { + return Err(Error::other(format!( + "failed to persist {stage} durable ILM decommission receipt `{receipt_path}` for {}: {err}", + locator.context() + ))); + } + } + } + } + + async fn advance_durable_ilm_decommission_receipts( + &self, + path: &str, + data: &[u8], + terminal: bool, + ) -> Result>> { + let active_runs = { + let pool_meta = self.pool_meta.read().await; + pool_meta + .pools + .iter() + .enumerate() + .filter_map(|(pool_idx, pool)| { + pool.decommission + .as_ref() + .filter(|info| info.has_decommission_state() && !info.complete) + .and_then(|info| info.start_time) + .map(|start_time| (pool_idx, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))) + }) + .collect::>() + }; + if active_runs.is_empty() { + return Ok(None); + } + + let stage = if terminal { "terminal" } else { "progress" }; + let record = validate_durable_ilm_record(path, data) + .map_err(|err| Error::other(format!("{stage} durable ILM record is invalid at path `{path}`: {err}")))?; + let active_source_pool_indices = active_runs.iter().map(|(pool_idx, _)| *pool_idx).collect::>(); + let mut terminal_target_pool_indices = Vec::new(); + for (source_pool_idx, run_token) in active_runs { + let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, record.id_kind, &record.id); + let mut receipt_found = false; + for pool_idx in 0..self.pools.len() { + if pool_idx != source_pool_idx { + let found = self + .advance_durable_ilm_decommission_receipt(pool_idx, &receipt_path, &record, terminal) + .await?; + receipt_found |= found; + if terminal + && found + && !active_source_pool_indices.contains(&pool_idx) + && !terminal_target_pool_indices.contains(&pool_idx) + { + terminal_target_pool_indices.push(pool_idx); + } + } + } + if terminal && !receipt_found { + return Err(Error::other(format!( + "terminal durable ILM record at path `{path}` {} is retained until its decommission receipt is committed", + record.context() + ))); + } + } + Ok(Some(terminal_target_pool_indices)) + } + + pub(crate) async fn record_durable_ilm_decommission_progress(&self, path: &str, data: &[u8]) -> Result<()> { + self.advance_durable_ilm_decommission_receipts(path, data, false) + .await + .map(|_| ()) + } + + pub(crate) async fn record_durable_ilm_decommission_terminal(&self, path: &str, data: &[u8]) -> Result<()> { + self.record_durable_ilm_decommission_terminal_target_pools(path, data) + .await + .map(|_| ()) + } + + /// Record terminal proof and return its non-source receipt pools for targeted cleanup. + pub(crate) async fn record_durable_ilm_decommission_terminal_target_pools( + &self, + path: &str, + data: &[u8], + ) -> Result>> { + self.advance_durable_ilm_decommission_receipts(path, data, true).await + } + + async fn cleanup_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> { + for (pool_idx, receipt_path) in self.list_decommission_durable_ilm_receipts(source_pool_idx).await? { + match delete_config(self.pools[pool_idx].clone(), &receipt_path).await { + Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {} + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} + Err(err) => { + return Err(Error::other(format!( + "failed to clean durable ILM decommission receipt `{receipt_path}` from pool {pool_idx}: {err}" + ))); + } + } + } + for receipt_path in self.list_decommission_durable_ilm_manifest_receipts(source_pool_idx).await? { + match delete_config(self.pools[source_pool_idx].clone(), &receipt_path).await { + Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {} + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} + Err(err) => { + return Err(Error::other(format!( + "failed to clean durable ILM expected manifest receipt `{receipt_path}` from source pool {source_pool_idx}: {err}" + ))); + } + } + } + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let manifest_path = decommission_durable_ilm_manifest_path(&run_token); + match delete_config(self.pools[source_pool_idx].clone(), &manifest_path).await { + Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {} + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} + Err(err) => { + return Err(Error::other(format!( + "failed to clean durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}" + ))); + } + } + Ok(()) + } + + async fn verify_and_cleanup_decommissioned_durable_ilm_record( + &self, + source_pool_idx: usize, + source_set: Arc, + path: &str, + ) -> Result<()> { + let namespace = classify_durable_ilm_record(path)? + .ok_or_else(|| Error::other(format!("path `{path}` is not a durable ILM record")))?; + let source_versions = source_set + .load_file_info_versions_exact(RUSTFS_META_BUCKET, path) + .await + .map_err(|err| Error::other(format!("failed to load source durable ILM versions at path `{path}`: {err}")))? + .ok_or_else(|| Error::other(format!("source durable ILM record is missing at path `{path}`")))?; + let source = read_config_limited_preserve_empty(source_set.clone(), path, namespace.max_record_size) + .await + .map_err(|err| Error::other(format!("failed to read source durable ILM record at path `{path}`: {err}")))?; + let source_record = validate_durable_ilm_record(path, &source) + .map_err(|err| Error::other(format!("source durable ILM record is invalid at path `{path}`: {err}")))?; + let target = self + .load_decommissioned_durable_ilm_target(source_pool_idx, path, namespace.max_record_size, &source_record.context()) + .await?; + let manifest_receipt = if let Some((target_pool_idx, target)) = target { + let target_record = validate_decommission_durable_ilm_copy(path, &source_record, &target)?; + let receipt = DecommissionDurableIlmReceipt::new(path, &target_record); + self.persist_decommission_durable_ilm_receipt(source_pool_idx, target_pool_idx, &receipt) + .await?; + receipt + } else { + self.load_decommission_durable_ilm_terminal_receipt(source_pool_idx, path, &source_record) + .await? + .ok_or_else(|| { + Error::other(format!( + "target durable ILM record is missing at path `{path}` {} without a matching terminal receipt", + source_record.context() + )) + })? + }; + self.persist_decommission_durable_ilm_receipt(source_pool_idx, source_pool_idx, &manifest_receipt) + .await?; + + let cleanup_result = data_movement::cleanup_source_entry_if_unchanged( + source_set, + RUSTFS_META_BUCKET, + path, + &source_versions, + &[], + data_movement::SourceCleanupBucketFence::default(), + "decommission durable ILM final sweep", + ) + .await + .map_err(|err| { + Error::other(format!( + "source durable ILM cleanup failed at path `{path}` {}: {err}", + source_record.context() + )) + }); + resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path) + } + + #[cfg(test)] + pub(crate) async fn verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + &self, + source_pool_idx: usize, + source_set: Arc, + path: &str, + ) -> Result<()> { + self.verify_and_cleanup_decommissioned_durable_ilm_record(source_pool_idx, source_set, path) + .await + } + + #[cfg(test)] + pub(crate) async fn decommission_durable_ilm_receipt_count_for_test(&self, source_pool_idx: usize) -> Result { + Ok(self.list_decommission_durable_ilm_receipts(source_pool_idx).await?.len()) + } + + #[cfg(test)] + pub(crate) async fn decommission_durable_ilm_receipt_paths_for_test( + &self, + source_pool_idx: usize, + ) -> Result> { + self.list_decommission_durable_ilm_receipts(source_pool_idx).await + } + + #[cfg(test)] + pub(crate) async fn persist_decommission_durable_ilm_receipt_for_test( + &self, + source_pool_idx: usize, + target_pool_idx: usize, + source_path: &str, + record: &ValidatedDurableIlmRecord, + terminal: bool, + ) -> Result { + let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record); + if terminal { + receipt.terminal_checkpoint = Some(record.checkpoint.clone()); + } + self.persist_decommission_durable_ilm_receipt(source_pool_idx, target_pool_idx, &receipt) + .await?; + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + Ok(decommission_durable_ilm_receipt_path(&run_token, source_path, record.id_kind, &record.id)) + } + + #[cfg(test)] + pub(crate) async fn persist_decommission_durable_ilm_manifest_for_test(&self, source_pool_idx: usize) -> Result<()> { + self.persist_decommission_durable_ilm_manifest(source_pool_idx).await + } + + #[cfg(test)] + pub(crate) async fn cleanup_decommission_durable_ilm_receipts_for_test(&self, source_pool_idx: usize) -> Result<()> { + self.cleanup_decommission_durable_ilm_receipts(source_pool_idx).await + } + + async fn check_after_decommission( + self: &Arc, + idx: usize, + rx: &CancellationToken, + generation: OffsetDateTime, + ) -> Result<()> { + self.ensure_decommission_generation_current(idx, generation).await?; + let operation_gate = self.ctx.decommission_operation_gate(); + run_decommission_side_effect(rx, &operation_gate, || self.check_after_decommission_unfenced(idx)).await + } + + async fn check_after_decommission_unfenced(self: &Arc, idx: usize) -> Result<()> { let buckets = self.get_buckets_to_decommission().await?; let pool = self.pools[idx].clone(); @@ -5164,22 +6547,27 @@ impl ECStore { let versions_found = Arc::new(AtomicUsize::new(0)); let entry_error = Arc::new(tokio::sync::Mutex::new(None::)); + let first_remaining_path = Arc::new(tokio::sync::Mutex::new(None::)); let callback_rx = CancellationToken::new(); let versions_found_cb = versions_found.clone(); let entry_error_cb = entry_error.clone(); + let first_remaining_path_cb = first_remaining_path.clone(); let bucket_name = bucket_info.name.clone(); let lifecycle_config_cb = lifecycle_config.clone(); let object_lock_config_cb = object_lock_config.clone(); let store = Arc::clone(self); + let source_set = set.clone(); let callback_rx_cb = callback_rx.clone(); let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| { let versions_found = versions_found_cb.clone(); let entry_error = entry_error_cb.clone(); + let first_remaining_path = first_remaining_path_cb.clone(); let bucket_name = bucket_name.clone(); let lifecycle_config = lifecycle_config_cb.clone(); let object_lock_config = object_lock_config_cb.clone(); let store = Arc::clone(&store); + let source_set = source_set.clone(); let callback_rx = callback_rx_cb.clone(); Box::pin(async move { if callback_rx.is_cancelled() { @@ -5194,6 +6582,41 @@ impl ECStore { return; } + let durable_ilm_record = if bucket_name == RUSTFS_META_BUCKET { + match classify_durable_ilm_record(&entry.name) { + Ok(record) => record, + Err(err) => { + let mut first_err = entry_error.lock().await; + if first_err.is_none() { + *first_err = Some(with_decommission_entry_context( + "check_after_decommission.durable_ilm_namespace", + &bucket_name, + &entry.name, + err, + )); + callback_rx.cancel(); + } + return; + } + } + } else { + None + }; + + if durable_ilm_record.is_some() { + if let Err(err) = store + .verify_and_cleanup_decommissioned_durable_ilm_record(idx, source_set, &entry.name) + .await + { + let mut first_err = entry_error.lock().await; + if first_err.is_none() { + *first_err = Some(err); + callback_rx.cancel(); + } + } + return; + } + let fivs = match load_decommission_entry_versions( &entry, &bucket_name, @@ -5242,6 +6665,13 @@ impl ECStore { remaining += 1; } + if remaining > 0 { + let mut first_path = first_remaining_path.lock().await; + if first_path.is_none() { + *first_path = Some(format!("{bucket_name}/{}", entry.name)); + } + } + versions_found.fetch_add(remaining, Ordering::Relaxed); }) }); @@ -5254,17 +6684,32 @@ impl ECStore { let versions_found = versions_found.load(Ordering::Relaxed); if versions_found > 0 { + let first_remaining_path = first_remaining_path + .lock() + .await + .clone() + .unwrap_or_else(|| format!("{}/", bucket_info.name)); return Err(Error::other(format!( - "at least {versions_found} object(s)/version(s) were found in bucket `{}` after decommissioning", - bucket_info.name + "at least {versions_found} object(s)/version(s) were found in bucket `{}` after decommissioning; first remaining path `{first_remaining_path}`", + bucket_info.name, ))); } } } + self.persist_decommission_durable_ilm_manifest(idx).await?; + self.verify_decommission_durable_ilm_receipts(idx).await?; + Ok(()) } + #[cfg(test)] + pub(crate) async fn check_after_decommission_for_test(self: &Arc, idx: usize) -> Result<()> { + let generation = self.active_decommission_generation(idx).await?; + self.check_after_decommission(idx, &CancellationToken::new(), generation) + .await + } + #[tracing::instrument(skip(self, rd))] async fn decommission_object( self: Arc, @@ -6357,14 +7802,18 @@ mod pools_tests { use super::DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF; use super::record_decommission_entry_error; use super::resolve_decommission_listing_error; + use super::resolve_decommission_partial_listing_entry; use super::{ - DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP, + DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, + DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP, DECOMMISSION_META_PREFIXES, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DecomBucketInfo, DecommissionCanceler, - DecommissionEntryEnqueueResult, DecommissionStartPoolState, DecommissionTerminalState, ListCallback, - PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, QueuedDecommissionEntry, apply_decommission_status_space_info, - await_decommission_worker, bind_decommission_cancelers, bind_missing_decommission_cancelers, - cancel_decommission_canceler, clamp_decommission_entry_concurrency, classify_decommission_terminal_state, - count_decommission_item, decommission_cancel_signal_result, decommission_entry_queue_capacity, decommission_item_size, + DecommissionDurableIlmReceipt, DecommissionEntryEnqueueResult, DecommissionStartPoolState, DecommissionTerminalState, + ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, QueuedDecommissionEntry, + apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers, + bind_missing_decommission_cancelers, cancel_decommission_canceler, clamp_decommission_entry_concurrency, + classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result, + decommission_durable_ilm_receipt_path, decommission_durable_ilm_receipt_run_prefix, + decommission_durable_ilm_receipt_run_token, decommission_entry_queue_capacity, decommission_item_size, decommission_meta_bucket_options, decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency, default_decommission_entry_concurrency, drain_decommission_entry_queue, enqueue_decommission_entry, ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_generation, @@ -6375,25 +7824,32 @@ mod pools_tests { ensure_local_decommission_pool_leaders, ensure_valid_decommission_pool_index, first_resumable_decommission_queue_indices, get_by_index, guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested, load_decommission_entry_versions, local_decommission_queue_prefix, - mark_decommission_bucket_done, merge_pool_status_refresh, missing_decommission_worker_prefix, - observe_decommission_terminal_reload_result, pool_meta_has_active_decommission, require_decommission_store, - reserve_decommission_start_cancelers, resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state, + mark_decommission_bucket_done, merge_decommission_durable_ilm_receipts, merge_pool_status_refresh, + missing_decommission_worker_prefix, observe_decommission_terminal_reload_result, pool_meta_has_active_decommission, + reconcile_decommission_meta_buckets, require_decommission_store, reserve_decommission_start_cancelers, + resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state, resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result, resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result, resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result, - resolve_decommission_partial_listing_entry, resolve_decommission_pool_meta_reload_result, - resolve_decommission_preflight_heal_result, resolve_decommission_progress_save_result, - resolve_decommission_terminal_mark_after_error_result, resolve_decommission_terminal_mark_result, - resolve_decommission_update_after_result, resolve_start_decommission_pool_meta_reload_result, - rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, run_decommission_listing_with_retry, - run_decommission_listing_with_retry_and_drain, run_decommission_side_effect, should_cleanup_decommission_source_entry, - should_continue_decommission_queue, should_count_decommission_version_complete, - should_preserve_decommission_canceled_state, should_reject_decommission_cancel_as_terminal, - should_retry_decommission_cancel_reload, should_retry_decommission_listing, should_skip_canceled_decommission_routine, - spawn_decommission_index_cancelers, split_decommission_buckets, take_and_cancel_decommission_canceler, - take_decommission_canceler, track_decommission_current_object, track_decommission_current_object_stage, - update_decommission_for_operation, validate_start_decommission_request, wait_decommission_listing_retry, - wait_decommission_worker_drain, with_decommission_entry_context, + resolve_decommission_pool_meta_reload_result, resolve_decommission_preflight_heal_result, + resolve_decommission_progress_save_result, resolve_decommission_terminal_mark_after_error_result, + resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result, + resolve_start_decommission_pool_meta_reload_result, rollback_start_decommission_pool_meta, + run_decommission_buckets_bounded, run_decommission_listing_with_retry, run_decommission_listing_with_retry_and_drain, + run_decommission_side_effect, should_cleanup_decommission_source_entry, should_continue_decommission_queue, + should_count_decommission_version_complete, should_preserve_decommission_canceled_state, + should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload, + should_retry_decommission_listing, should_skip_canceled_decommission_routine, spawn_decommission_index_cancelers, + split_decommission_buckets, take_and_cancel_decommission_canceler, take_decommission_canceler, + track_decommission_current_object, track_decommission_current_object_stage, update_decommission_for_operation, + validate_start_decommission_request, wait_decommission_listing_retry, wait_decommission_worker_drain, + with_decommission_entry_context, + }; + use crate::bucket::lifecycle::{ + DurableIlmRecordCheckpoint, + bucket_lifecycle_ops::{ManualTransitionQueueSnapshot, ManualTransitionRunOptions}, + manual_transition_job::{ManualTransitionJobRecord, manual_transition_job_record_object_name}, + validate_durable_ilm_record, }; use crate::data_movement; use crate::disk::endpoint::Endpoint; @@ -6462,6 +7918,93 @@ mod pools_tests { } } + #[test] + fn decommission_receipt_run_token_changes_with_persisted_start_time() { + let first = OffsetDateTime::from_unix_timestamp(1_000).expect("first run timestamp should be valid"); + let second = OffsetDateTime::from_unix_timestamp(2_000).expect("second run timestamp should be valid"); + let first_token = decommission_durable_ilm_receipt_run_token("pool-0", first); + let second_token = decommission_durable_ilm_receipt_run_token("pool-0", second); + + assert_ne!(first_token, second_token); + assert_eq!(first_token, decommission_durable_ilm_receipt_run_token("pool-0", first)); + let operation_id = "a".repeat(64); + let old_receipt = decommission_durable_ilm_receipt_path( + &first_token, + &format!("ilm/tier-delete-journal/{operation_id}.json"), + "operation_id", + &operation_id, + ); + assert!(!old_receipt.starts_with(&decommission_durable_ilm_receipt_run_prefix(&second_token))); + } + + #[test] + fn decommission_receipt_merge_preserves_terminal_proof() { + let operation_id = "a".repeat(64); + let source_path = format!("ilm/tier-delete-journal/{operation_id}.json"); + let checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal { + content_sha256: "b".repeat(64), + identity_sha256: "c".repeat(64), + committed: false, + }; + let terminal_checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal { + content_sha256: "d".repeat(64), + identity_sha256: "c".repeat(64), + committed: true, + }; + let incoming = DecommissionDurableIlmReceipt { + source_path, + namespace: "tier-delete-journal".to_string(), + id_kind: "operation_id".to_string(), + id: operation_id, + checkpoint: checkpoint.clone(), + terminal_checkpoint: None, + }; + let existing = DecommissionDurableIlmReceipt { + terminal_checkpoint: Some(terminal_checkpoint.clone()), + ..incoming.clone() + }; + + let merged = merge_decommission_durable_ilm_receipts(&existing, &incoming) + .expect("retry receipt must merge with a terminal receipt"); + + assert_eq!(merged.checkpoint, checkpoint); + assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint)); + } + + #[test] + fn decommission_manual_job_receipt_compacts_large_progress() { + let prefix = "p".repeat(12 * 1024); + let options = ManualTransitionRunOptions { + prefix, + ..Default::default() + }; + let mut job = ManualTransitionJobRecord::new(uuid::Uuid::new_v4(), "bounded-receipt-bucket", &options, "owner"); + let token_bytes = serde_json::to_vec(&serde_json::json!({ + "marker": "m".repeat(12 * 1024), + "version_marker": "opaque-version" + })) + .expect("large continuation token should encode"); + let mut report = job.report.clone(); + report.scanned = 1; + report.continuation_token = Some(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&token_bytes)); + job.update_running_progress(report, ManualTransitionQueueSnapshot::default()); + let path = manual_transition_job_record_object_name(job.job_id).expect("manual job path should build"); + let job_bytes = job.encode().expect("large manual job should remain within its record limit"); + assert!(job_bytes.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE); + let record = validate_durable_ilm_record(&path, &job_bytes).expect("large manual job should validate"); + let expected_checkpoint = record.checkpoint.clone(); + let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record); + receipt.terminal_checkpoint = Some(record.checkpoint); + + let encoded = receipt.encode().expect("bounded progress proof should fit the receipt limit"); + let decoded = DecommissionDurableIlmReceipt::decode(&encoded).expect("bounded receipt should round trip"); + + assert!(encoded.len() <= DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE); + assert_eq!(decoded.source_path, path); + assert_eq!(decoded.checkpoint, expected_checkpoint); + assert_eq!(decoded.terminal_checkpoint, Some(expected_checkpoint)); + } + #[test] fn test_apply_decommission_status_space_info_adds_idle_pool_usage() { let status = apply_decommission_status_space_info( @@ -6736,6 +8279,10 @@ mod pools_tests { name: crate::disk::RUSTFS_META_BUCKET.to_string(), prefix: crate::disk::BUCKET_META_PREFIX.to_string(), }, + DecomBucketInfo { + name: crate::disk::RUSTFS_META_BUCKET.to_string(), + prefix: crate::bucket::lifecycle::ILM_META_PREFIX.to_string(), + }, ]); assert_eq!( @@ -6744,10 +8291,42 @@ mod pools_tests { ); assert_eq!( meta.iter().map(|bucket| bucket.prefix.as_str()).collect::>(), - vec![crate::config::com::CONFIG_PREFIX, crate::disk::BUCKET_META_PREFIX,] + vec![ + crate::config::com::CONFIG_PREFIX, + crate::disk::BUCKET_META_PREFIX, + crate::bucket::lifecycle::ILM_META_PREFIX, + ] ); } + #[test] + fn test_resume_reconciles_missing_decommission_meta_prefixes() { + let mut meta = PoolMeta { + pools: vec![decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + queued_buckets: vec![ + format!("{}/{}", crate::disk::RUSTFS_META_BUCKET, crate::config::com::CONFIG_PREFIX), + format!("{}/{}", crate::disk::RUSTFS_META_BUCKET, crate::disk::BUCKET_META_PREFIX), + ], + ..Default::default() + }), + )], + ..Default::default() + }; + + assert!(reconcile_decommission_meta_buckets(&mut meta, 0)); + assert_eq!( + meta.pending_buckets(0) + .iter() + .filter(|bucket| bucket.name == crate::disk::RUSTFS_META_BUCKET) + .map(|bucket| bucket.prefix.as_str()) + .collect::>(), + DECOMMISSION_META_PREFIXES + ); + assert!(!reconcile_decommission_meta_buckets(&mut meta, 0)); + } + #[tokio::test] async fn test_run_decommission_buckets_bounded_respects_limit() { let rx = CancellationToken::new(); diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index bb3c9fbf9..1ceb8acdf 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -37,7 +37,7 @@ use crate::bucket::lifecycle::{ transition_transaction::{ TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit, TransitionTransactionState, delete_transition_transaction_record, - save_transition_transaction_record, + load_transition_transaction_record, save_transition_transaction_record, }, }; use crate::bucket::quota::reservation; @@ -4248,7 +4248,12 @@ fn record_transition_uploaded_save_attempt(transaction: &TransitionTransaction, async fn delete_transition_transaction_if_available(api: Option<&Arc>, transaction_id: Uuid) -> Result<()> { if let Some(api) = api { - return delete_transition_transaction_record(api.clone(), transaction_id).await; + let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await { + Ok(transaction) => transaction, + Err(Error::ConfigNotFound) => return Ok(()), + Err(err) => return Err(err), + }; + return delete_transition_transaction_record(api.clone(), &transaction).await; } Ok(()) } diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 451e42242..1d143201c 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -555,10 +555,18 @@ mod tests { #[cfg(feature = "test-util")] use crate::{ bucket::lifecycle::{ + DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, + bucket_lifecycle_ops::{ManualTransitionRunOptions, recover_manual_transition_jobs_once}, lifecycle::{TRANSITION_PENDING, TransitionOptions}, + manual_transition_job::{ + ManualTransitionJobRecord, ManualTransitionScopeAdmission, ManualTransitionTaskRecord, + ManualTransitionWorkerResult, ManualTransitionWorkerResultRecord, manual_transition_job_record_object_name, + manual_transition_scope_record_object_name, manual_transition_task_object_name, + manual_transition_worker_result_object_name, manual_transition_worker_result_task_key, + }, tier_delete_journal::{ - TIER_DELETE_JOURNAL_PREFIX, persist_tier_delete_journal_entry, recover_tier_delete_journal_entries, - tier_delete_journal_object_name, + TIER_DELETE_JOURNAL_PREFIX, encode_tier_delete_journal_entry, persist_tier_delete_journal_entry, + recover_tier_delete_journal_entries, tier_delete_journal_object_name, }, tier_sweeper::{ Jentry, TierDeleteJournalState, TierDeleteSourceIdentity, transitioned_delete_journal_entry_for_source, @@ -570,12 +578,16 @@ mod tests { delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator, load_transition_transaction_record, recover_transition_transaction_records, save_transition_transaction_record, + transition_transaction_record_object_name, }, + validate_durable_ilm_record, }, bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG}, client::transition_api::ReaderImpl, config::com, - disk::{RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE}, + core::pools::DecomBucketInfo, + data_movement::SourceCleanupDeleteBarrier, + disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE}, runtime::{global::set_object_store_resolver, sources as runtime_sources}, services::tier::{ test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier}, @@ -608,6 +620,8 @@ mod tests { range::HTTPRangeSpec, }, }; + #[cfg(feature = "test-util")] + use futures::{StreamExt as _, TryStreamExt as _}; use http::HeaderMap; use rustfs_config::server_config::KVS; #[cfg(feature = "test-util")] @@ -4386,6 +4400,1079 @@ mod tests { )); } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success() { + let temp_dir = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-target-read-error", &[4, 4, 4])) + .await; + let job_id = uuid::Uuid::new_v4(); + let job = + ManualTransitionJobRecord::new(job_id, "manual-target-read-error", &ManualTransitionRunOptions::default(), "owner"); + let path = manual_transition_job_record_object_name(job_id).expect("manual job path should build"); + let data = job.encode().expect("manual job should encode"); + for pool in &store.pools { + com::save_config(pool.clone(), &path, data.clone()) + .await + .expect("manual job fixture should persist in every pool"); + } + store.pool_meta.write().await.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + + let failing_target = store.pools[2].get_disks_by_key(&path); + let original_disks = { + let mut disks = failing_target.disks.write().await; + let original = disks.clone(); + for disk in disks.iter_mut().take(3) { + *disk = None; + } + original + }; + let error = store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, store.pools[0].get_disks_by_key(&path), &path) + .await + .expect_err("one target read-quorum error must fail closed despite another target success") + .to_string(); + *failing_target.disks.write().await = original_disks; + + assert!(error.contains(&path)); + assert!(error.contains("pool 2")); + assert_eq!( + com::read_config(store.pools[0].clone(), &path) + .await + .expect("target read error must retain the source"), + data + ); + assert_eq!( + store + .decommission_durable_ilm_receipt_count_for_test(0) + .await + .expect("failed target verification should not create a receipt"), + 0 + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup() { + let temp_dir = tempfile::tempdir().expect("create temp store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-terminal-receipt", &[4, 4])).await; + let tier_name = "DECOMMISSION-RECEIPT"; + let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("tier lease should resolve") + .backend_identity(); + let entry = Jentry { + obj_name: "receipt-recovery-object".to_string(), + version_id: "receipt-recovery-version".to_string(), + tier_name: tier_name.to_string(), + backend_identity: Some(backend_identity), + version_id_exact: true, + version_state: rustfs_filemeta::TransitionVersionState::Exact, + state: TierDeleteJournalState::Committed, + source: None, + }; + let path = tier_delete_journal_object_name(&entry); + let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode"); + com::save_config(store.pools[0].clone(), &path, data.clone()) + .await + .expect("source tier journal should persist"); + com::save_config(store.pools[1].clone(), &path, data.clone()) + .await + .expect("target tier journal should persist"); + let active_pool_meta = { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + pool_meta.clone() + }; + active_pool_meta + .save(store.pools.clone()) + .await + .expect("active decommission run identity should persist"); + + let source_set = store.pools[0].get_disks_by_key(&path); + let barrier = SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &path); + let cleanup_store = store.clone(); + let cleanup_set = source_set.clone(); + let cleanup_path = path.clone(); + let cleanup = tokio::spawn(async move { + cleanup_store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, cleanup_set, &cleanup_path) + .await + }); + barrier.wait_until_paused().await; + let original_source_disks = { + let mut disks = source_set.disks.write().await; + let original = disks.clone(); + for disk in disks.iter_mut().take(3) { + *disk = None; + } + original + }; + barrier.release(); + let cleanup_error = cleanup + .await + .expect("source cleanup task should not panic") + .expect_err("injected source delete quorum failure must fail cleanup") + .to_string(); + *source_set.disks.write().await = original_source_disks; + drop(barrier); + + assert!(cleanup_error.contains("source durable ILM cleanup failed")); + assert_eq!( + store + .decommission_durable_ilm_receipt_count_for_test(0) + .await + .expect("receipt should persist before source cleanup"), + 1 + ); + assert_eq!( + com::read_config(store.pools[0].clone(), &path) + .await + .expect("failed cleanup must retain the source"), + data + ); + + let mut restarted_pool_meta = PoolMeta::default(); + restarted_pool_meta + .load(store.pools[0].clone(), store.pools.clone()) + .await + .expect("decommission run identity should reload after restart"); + *store.pool_meta.write().await = restarted_pool_meta; + let stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("target recovery should commit terminal proof and delete the target"); + assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0)); + assert!(matches!( + com::read_config(store.pools[1].clone(), &path).await, + Err(Error::ConfigNotFound) + )); + assert_eq!( + com::read_config(store.pools[0].clone(), &path) + .await + .expect("target recovery must not delete the decommission source"), + data + ); + + store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, source_set, &path) + .await + .expect("terminal receipt should authorize cleanup after target deletion"); + assert!(matches!( + com::read_config(store.pools[0].clone(), &path).await, + Err(Error::ConfigNotFound) + )); + assert!(backend.remove_versions().await.contains(&(entry.obj_name, entry.version_id))); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn decommission_final_sweep_blocks_cancel_until_source_cleanup_finishes() { + let temp_dir = tempfile::tempdir().expect("create final sweep gate store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-final-sweep-gate", &[4, 4])).await; + let job_id = uuid::Uuid::new_v4(); + let job = ManualTransitionJobRecord::new(job_id, "final-sweep-gate", &ManualTransitionRunOptions::default(), "owner"); + let path = manual_transition_job_record_object_name(job_id).expect("manual job path should build"); + let data = job.encode().expect("manual job should encode"); + for pool in &store.pools { + com::save_config(pool.clone(), &path, data.clone()) + .await + .expect("manual job fixture should persist in both pools"); + } + let active_pool_meta = { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + pool_meta.clone() + }; + active_pool_meta + .save(store.pools.clone()) + .await + .expect("active decommission run identity should persist"); + + let barrier = SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &path); + let final_sweep = tokio::spawn({ + let store = store.clone(); + async move { store.check_after_decommission_for_test(0).await } + }); + barrier.wait_until_paused().await; + + let mut cancel = tokio::spawn({ + let store = store.clone(); + async move { store.decommission_cancel(0).await } + }); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut cancel).await.is_err(), + "cancel must wait for the final sweep source cleanup" + ); + { + let pool_meta = store.pool_meta.read().await; + let decommission = pool_meta.pools[0] + .decommission + .as_ref() + .expect("decommission state should remain present"); + assert!( + !decommission.canceled, + "cancel must not publish terminal state before the final sweep drains" + ); + assert!( + decommission.start_time.is_some(), + "cancel must preserve the run identity until the final sweep drains" + ); + } + + barrier.release(); + final_sweep + .await + .expect("final sweep task should not panic") + .expect("final sweep should finish after the barrier releases"); + cancel + .await + .expect("cancel task should not panic") + .expect("cancel should complete after the final sweep releases the operation gate"); + let pool_meta = store.pool_meta.read().await; + let decommission = pool_meta.pools[0] + .decommission + .as_ref() + .expect("decommission state should remain present"); + assert!(decommission.canceled); + assert!(decommission.start_time.is_none()); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn decommission_durable_ilm_recovery_keeps_multiple_active_sources() { + let temp_dir = tempfile::tempdir().expect("create multi-source recovery store dir"); + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "durable-ilm-multi-source-recovery", + &[4, 4, 4], + )) + .await; + let tier_name = "DECOMMISSION-MULTI-SOURCE"; + let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("tier lease should resolve") + .backend_identity(); + let entry = Jentry { + obj_name: "multi-source-recovery-object".to_string(), + version_id: "multi-source-recovery-version".to_string(), + tier_name: tier_name.to_string(), + backend_identity: Some(backend_identity), + version_id_exact: true, + version_state: rustfs_filemeta::TransitionVersionState::Exact, + state: TierDeleteJournalState::Committed, + source: None, + }; + let path = tier_delete_journal_object_name(&entry); + let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode"); + for pool in &store.pools { + com::save_config(pool.clone(), &path, data.clone()) + .await + .expect("source and target tier journals should persist"); + } + + let active_pool_meta = { + let mut pool_meta = store.pool_meta.write().await; + let start_time = OffsetDateTime::now_utc(); + for pool_idx in [0, 1] { + pool_meta.pools[pool_idx].decommission = Some(PoolDecommissionInfo { + start_time: Some(start_time), + ..Default::default() + }); + } + pool_meta.clone() + }; + active_pool_meta + .save(store.pools.clone()) + .await + .expect("multiple active decommission runs should persist"); + let mut restarted_pool_meta = PoolMeta::default(); + restarted_pool_meta + .load(store.pools[0].clone(), store.pools.clone()) + .await + .expect("multiple active decommission runs should reload"); + *store.pool_meta.write().await = restarted_pool_meta; + + let record = validate_durable_ilm_record(&path, &data).expect("tier journal should validate"); + let source_zero_receipt = store + .persist_decommission_durable_ilm_receipt_for_test(0, 1, &path, &record, false) + .await + .expect("source pool zero receipt should persist on the other active source"); + let source_one_receipt = store + .persist_decommission_durable_ilm_receipt_for_test(1, 0, &path, &record, false) + .await + .expect("source pool one receipt should persist on the other active source"); + assert_ne!( + source_zero_receipt, source_one_receipt, + "active source runs must have distinct receipt paths" + ); + + let stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("cross-source receipts should not remove active source journals"); + assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0)); + for pool in &store.pools { + assert_eq!( + com::read_config(pool.clone(), &path) + .await + .expect("cross-source receipts alone must retain every journal copy"), + data + ); + } + + store + .persist_decommission_durable_ilm_receipt_for_test(0, 2, &path, &record, false) + .await + .expect("source pool zero receipt should persist on the target"); + store + .persist_decommission_durable_ilm_receipt_for_test(1, 2, &path, &record, false) + .await + .expect("source pool one receipt should persist on the target"); + + let stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("multi-source tier journal recovery should complete"); + assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0)); + assert_eq!( + com::read_config(store.pools[0].clone(), &path) + .await + .expect("first active source must remain after target recovery"), + data + ); + assert_eq!( + com::read_config(store.pools[1].clone(), &path) + .await + .expect("second active source must remain after target recovery"), + data + ); + assert!(matches!( + com::read_config(store.pools[2].clone(), &path).await, + Err(Error::ConfigNotFound) + )); + assert!(backend.remove_versions().await.contains(&(entry.obj_name, entry.version_id))); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page() { + const RECEIPT_COUNT: usize = 1001; + + let temp_dir = tempfile::tempdir().expect("create paginated receipt store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-receipt-pages", &[4, 4])).await; + store.pool_meta.write().await.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + + futures::stream::iter(0..RECEIPT_COUNT) + .map(|index| { + let store = store.clone(); + async move { + let id = format!("{index:064x}"); + let source_path = format!("ilm/tier-delete-journal/{id}.json"); + let record = ValidatedDurableIlmRecord { + namespace: "tier-delete-journal", + id_kind: "operation_id", + id, + checkpoint: DurableIlmRecordCheckpoint::TierDeleteJournal { + content_sha256: format!("{:064x}", index + RECEIPT_COUNT), + identity_sha256: "f".repeat(64), + committed: false, + }, + }; + store + .persist_decommission_durable_ilm_receipt_for_test(0, 0, &source_path, &record, true) + .await?; + store + .persist_decommission_durable_ilm_receipt_for_test(0, 1, &source_path, &record, true) + .await?; + Ok::<(), Error>(()) + } + }) + .buffer_unordered(32) + .try_collect::>() + .await + .expect("more than one receipt page should persist"); + store + .persist_decommission_durable_ilm_manifest_for_test(0) + .await + .expect("paginated source receipts should produce a manifest"); + + let target_receipts = store + .decommission_durable_ilm_receipt_paths_for_test(0) + .await + .expect("paginated target receipts should list"); + assert_eq!(target_receipts.len(), RECEIPT_COUNT); + let (target_pool_idx, second_page_path) = target_receipts + .get(1000) + .cloned() + .expect("the real 1000-item page boundary should expose a second page receipt"); + let receipt_bytes = com::read_config(store.pools[target_pool_idx].clone(), &second_page_path) + .await + .expect("second page receipt should be readable"); + + com::delete_config(store.pools[target_pool_idx].clone(), &second_page_path) + .await + .expect("second page receipt should delete"); + let missing = store + .complete_decommission(0) + .await + .expect_err("a missing second page receipt must block completion") + .to_string(); + assert!(missing.contains(&second_page_path)); + assert!( + !store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .expect("source pool should remain in decommission") + .complete + ); + assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok()); + + com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone()) + .await + .expect("second page receipt should restore"); + com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec()) + .await + .expect("second page receipt should corrupt deterministically"); + let corrupt = store + .complete_decommission(0) + .await + .expect_err("a corrupt second page receipt must block completion") + .to_string(); + assert!(corrupt.contains(&second_page_path)); + assert!(corrupt.contains("invalid")); + assert!( + !store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .expect("source pool should remain in decommission") + .complete + ); + assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok()); + } + + #[cfg(feature = "test-util")] + #[test] + #[serial_test::serial(storage_class_env)] + fn decommission_migrates_and_verifies_registered_durable_ilm_records() { + std::thread::Builder::new() + .name("durable-ilm-decommission-test".to_string()) + .stack_size(16 * 1024 * 1024) + .spawn(|| { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(2) + .build() + .expect("durable ILM decommission runtime should build"); + runtime.block_on(decommission_migrates_and_verifies_registered_durable_ilm_records_scenario()); + }) + .expect("durable ILM decommission scenario thread should spawn") + .join() + .expect("durable ILM decommission scenario should not panic"); + } + + #[cfg(feature = "test-util")] + async fn decommission_migrates_and_verifies_registered_durable_ilm_records_scenario() { + let temp_dir = tempfile::tempdir().expect("create temp store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-decommission", &[4, 4])).await; + + let tier_name = "DECOMMISSION-ILM"; + let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("tier lease should resolve") + .backend_identity(); + let tier_entry = Jentry { + obj_name: "decommissioned-remote-object".to_string(), + version_id: "decommissioned-remote-version".to_string(), + tier_name: tier_name.to_string(), + backend_identity: Some(backend_identity), + version_id_exact: true, + version_state: rustfs_filemeta::TransitionVersionState::Exact, + state: TierDeleteJournalState::Committed, + source: None, + }; + let tier_path = tier_delete_journal_object_name(&tier_entry); + let tier_bytes = encode_tier_delete_journal_entry(&tier_entry).expect("tier journal should encode"); + + let mut transaction = TransitionTransaction::new(TransitionTransactionInit { + deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"), + transaction_id: uuid::Uuid::new_v4(), + owner_epoch: uuid::Uuid::new_v4(), + write_id: uuid::Uuid::new_v4(), + source: TransitionSourceIdentity { + bucket: "source-bucket".to_string(), + object: "source-object".to_string(), + version_id: Some(uuid::Uuid::new_v4()), + data_dir: uuid::Uuid::new_v4(), + mod_time_unix_nanos: 1_770_000_000_000_000_000, + size: 42, + etag: "source-etag".to_string(), + version_mode: TransitionSourceVersionMode::Versioned, + }, + tier_name: tier_name.to_string(), + backend_fingerprint: backend_identity, + not_after_unix_nanos: 1_780_000_000_000_000_000, + }) + .expect("transition transaction should build"); + let regressed_transaction = transaction.clone(); + transaction + .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None) + .expect("transition transaction should advance before migration"); + let transaction_path = transition_transaction_record_object_name(transaction.transaction_id) + .expect("transition transaction path should build"); + let transaction_bytes = transaction.encode().expect("transition transaction should encode"); + + let manual_job_id = uuid::Uuid::new_v4(); + let manual_bucket = format!("manual-decommission-{}", manual_job_id.simple()); + let manual_options = ManualTransitionRunOptions { + prefix: "logs/".to_string(), + tier: Some(tier_name.to_string()), + ..Default::default() + }; + let mut manual_job = ManualTransitionJobRecord::new(manual_job_id, &manual_bucket, &manual_options, "old-owner"); + manual_job.scan_completed = true; + manual_job.report.enqueued = 1; + manual_job.lease_expires_at_unix_nanos = 0; + let manual_scope = ManualTransitionScopeAdmission::from_job(&manual_job); + let task_key = manual_transition_worker_result_task_key(&manual_bucket, "logs/a", None); + let manual_task = ManualTransitionTaskRecord::new(manual_job_id, &task_key, &manual_bucket, "logs/a", None, tier_name); + let manual_result = + ManualTransitionWorkerResultRecord::new(manual_job_id, &task_key, ManualTransitionWorkerResult::Completed); + + let manual_job_path = manual_transition_job_record_object_name(manual_job_id).expect("manual job path should build"); + let manual_scope_path = + manual_transition_scope_record_object_name(&manual_scope.scope_key).expect("manual scope path should build"); + let manual_task_path = + manual_transition_task_object_name(manual_job_id, &task_key).expect("manual task path should build"); + let manual_result_path = manual_transition_worker_result_object_name(manual_job_id, &task_key) + .expect("manual worker result path should build"); + let manual_job_bytes = manual_job.encode().expect("manual job should encode"); + let manual_scope_bytes = serde_json::to_vec(&manual_scope).expect("manual scope should encode"); + let manual_task_bytes = manual_task.encode().expect("manual task should encode"); + let manual_result_bytes = manual_result.encode().expect("manual result should encode"); + + let records = vec![ + (tier_path.clone(), tier_bytes.clone()), + (transaction_path.clone(), transaction_bytes.clone()), + (manual_job_path.clone(), manual_job_bytes.clone()), + (manual_scope_path.clone(), manual_scope_bytes.clone()), + (manual_task_path.clone(), manual_task_bytes.clone()), + (manual_result_path.clone(), manual_result_bytes.clone()), + ]; + for (path, data) in &records { + com::save_config(store.pools[0].clone(), path, data.clone()) + .await + .expect("durable ILM source record should persist"); + } + + let legacy_queue = [com::CONFIG_PREFIX, BUCKET_META_PREFIX] + .into_iter() + .map(|prefix| { + DecomBucketInfo { + name: RUSTFS_META_BUCKET.to_string(), + prefix: prefix.to_string(), + } + .to_string() + }) + .collect(); + let legacy_pool_meta = { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + queued: true, + queued_buckets: legacy_queue, + ..Default::default() + }); + pool_meta.clone() + }; + legacy_pool_meta + .save(store.pools.clone()) + .await + .expect("legacy decommission queue should persist before restart"); + let mut restarted_pool_meta = PoolMeta::default(); + restarted_pool_meta + .load(store.pools[0].clone(), store.pools.clone()) + .await + .expect("legacy decommission queue should reload after restart"); + *store.pool_meta.write().await = restarted_pool_meta; + store + .promote_queued_decommission_for_test(0) + .await + .expect("legacy queued decommission should resume"); + let expected_ilm_queue = DecomBucketInfo { + name: RUSTFS_META_BUCKET.to_string(), + prefix: ILM_META_PREFIX.to_string(), + } + .to_string(); + { + let pool_meta = store.pool_meta.read().await; + let decommission = pool_meta.pools[0] + .decommission + .as_ref() + .expect("decommission state should remain present"); + assert!(!decommission.queued); + assert!(decommission.queued_buckets.contains(&expected_ilm_queue)); + } + + let ilm_bucket = DecomBucketInfo { + name: RUSTFS_META_BUCKET.to_string(), + prefix: ILM_META_PREFIX.to_string(), + }; + for _ in 0..2 { + store + .decommission_pool_for_test(CancellationToken::new(), 0, store.pools[0].clone(), ilm_bucket.clone()) + .await + .expect("durable ILM decommission should be idempotent"); + } + for (path, expected) in &records { + assert_eq!( + com::read_config(store.pools[0].clone(), path) + .await + .expect("source should remain until the final sweep"), + *expected + ); + assert_eq!( + com::read_config(store.pools[1].clone(), path) + .await + .expect("target should contain the migrated record"), + *expected + ); + } + assert_eq!( + store + .decommission_durable_ilm_receipt_count_for_test(0) + .await + .expect("no receipt should exist before the final sweep"), + 0 + ); + let isolated_tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("tier recovery should retain a terminal record until its receipt is committed"); + assert!(isolated_tier_stats.scanned >= 1); + assert_eq!(isolated_tier_stats.deleted, 0); + assert!(isolated_tier_stats.failed >= 1); + assert_eq!( + com::read_config(store.pools[1].clone(), &tier_path) + .await + .expect("receipt isolation must retain the target tier journal"), + tier_bytes + ); + + com::delete_config(store.pools[1].clone(), &manual_job_path) + .await + .expect("target manual job should delete"); + let missing = store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + 0, + store.pools[0].get_disks_by_key(&manual_job_path), + &manual_job_path, + ) + .await + .expect_err("missing target must block source cleanup"); + let missing = missing.to_string(); + assert!(missing.contains(&manual_job_path) && missing.contains(&manual_job_id.to_string())); + assert_eq!( + com::read_config(store.pools[0].clone(), &manual_job_path) + .await + .expect("missing target must retain source"), + manual_job_bytes + ); + com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone()) + .await + .expect("target manual job should restore"); + + com::save_config(store.pools[1].clone(), &transaction_path, b"{corrupt".to_vec()) + .await + .expect("target transaction should corrupt deterministically"); + let corrupt = store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + 0, + store.pools[0].get_disks_by_key(&transaction_path), + &transaction_path, + ) + .await + .expect_err("corrupt target must block source cleanup"); + let corrupt = corrupt.to_string(); + assert!(corrupt.contains(&transaction_path) && corrupt.contains(&transaction.transaction_id.to_string())); + assert_eq!( + com::read_config(store.pools[0].clone(), &transaction_path) + .await + .expect("corrupt target must retain source"), + transaction_bytes + ); + com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone()) + .await + .expect("target transaction should restore"); + + com::save_config(store.pools[1].clone(), &manual_scope_path, manual_scope_bytes.clone()) + .await + .expect("target scope rewrite should invalidate cached metadata before the quorum check"); + let target_scope_set = store.pools[1].get_disks_by_key(&manual_scope_path); + let original_target_scope_disks = { + let mut disks = target_scope_set.disks.write().await; + let original = disks.clone(); + for disk in disks.iter_mut().take(3) { + *disk = None; + } + original + }; + let quorum_error = store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + 0, + store.pools[0].get_disks_by_key(&manual_scope_path), + &manual_scope_path, + ) + .await + .expect_err("target below read quorum must block source cleanup"); + *target_scope_set.disks.write().await = original_target_scope_disks; + let quorum_error = quorum_error.to_string(); + assert!(quorum_error.contains(&manual_scope_path) && quorum_error.contains(&manual_job_id.to_string())); + assert!(com::read_config(store.pools[0].clone(), &manual_scope_path).await.is_ok()); + + com::save_config(store.pools[1].clone(), &manual_task_path, manual_task_bytes.clone()) + .await + .expect("target task rewrite should invalidate cached metadata before the quorum check"); + let target_task_set = store.pools[1].get_disks_by_key(&manual_task_path); + let original_target_task_disks = { + let mut disks = target_task_set.disks.write().await; + let original = disks.clone(); + for disk in disks.iter_mut().take(2) { + *disk = None; + } + original + }; + let receipt_quorum_error = store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + 0, + store.pools[0].get_disks_by_key(&manual_task_path), + &manual_task_path, + ) + .await + .expect_err("target read quorum without receipt write quorum must retain the source"); + *target_task_set.disks.write().await = original_target_task_disks; + let receipt_quorum_error = receipt_quorum_error.to_string(); + assert!(receipt_quorum_error.contains("receipt")); + assert!(receipt_quorum_error.contains(&manual_task_path)); + assert!(receipt_quorum_error.contains(&manual_job_id.to_string())); + assert!(com::read_config(store.pools[0].clone(), &manual_task_path).await.is_ok()); + store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + 0, + store.pools[0].get_disks_by_key(&manual_task_path), + &manual_task_path, + ) + .await + .expect("healthy target should persist the receipt before source cleanup"); + + let unknown_path = "ilm/future-durable/jobs/one.json"; + com::save_config(store.pools[0].clone(), unknown_path, b"{}".to_vec()) + .await + .expect("unknown durable ILM record should persist for the guard test"); + let unknown_migration = store + .decommission_pool_for_test(CancellationToken::new(), 0, store.pools[0].clone(), ilm_bucket) + .await + .expect_err("unregistered durable ILM namespace must block migration"); + assert!(unknown_migration.to_string().contains(unknown_path)); + let unknown_final_sweep = store + .check_after_decommission_for_test(0) + .await + .expect_err("unregistered durable ILM namespace must block completion"); + assert!(unknown_final_sweep.to_string().contains(unknown_path)); + com::delete_config(store.pools[0].clone(), unknown_path) + .await + .expect("unknown guard fixture should be removed before the successful final sweep"); + + store + .check_after_decommission_for_test(0) + .await + .expect("production final sweep should validate every target before cleanup"); + assert_eq!( + store + .decommission_durable_ilm_receipt_count_for_test(0) + .await + .expect("durable ILM receipts should be listable"), + records.len(), + "every cleaned source record must have a durable validation receipt" + ); + for (path, expected) in &records { + assert!( + matches!(com::read_config(store.pools[0].clone(), path).await, Err(Error::ConfigNotFound)), + "final sweep should remove the validated source `{path}`" + ); + assert_eq!( + com::read_config(store.pools[1].clone(), path) + .await + .expect("final sweep must preserve the target"), + *expected + ); + } + + let mut crash_restarted_pool_meta = PoolMeta::default(); + crash_restarted_pool_meta + .load(store.pools[0].clone(), store.pools.clone()) + .await + .expect("pool metadata should reload after the simulated pre-complete crash"); + *store.pool_meta.write().await = crash_restarted_pool_meta; + + let (manual_job_receipt_pool, manual_job_receipt_path) = store + .decommission_durable_ilm_receipt_paths_for_test(0) + .await + .expect("durable ILM receipt paths should be listable") + .into_iter() + .find(|(_, path)| path.contains(&manual_job_path)) + .expect("manual job should have one target receipt"); + let manual_job_receipt_bytes = com::read_config(store.pools[manual_job_receipt_pool].clone(), &manual_job_receipt_path) + .await + .expect("manual job receipt should be readable before deletion"); + com::delete_config(store.pools[manual_job_receipt_pool].clone(), &manual_job_receipt_path) + .await + .expect("manual job receipt should delete after source cleanup"); + com::delete_config(store.pools[1].clone(), &manual_job_path) + .await + .expect("post-crash target manual job should delete"); + let missing_after_crash = store + .complete_decommission(0) + .await + .expect_err("completion must reject a missing target after source cleanup and restart") + .to_string(); + assert!(missing_after_crash.contains("receipt")); + assert!(missing_after_crash.contains(&manual_job_path)); + assert!(missing_after_crash.contains(&manual_job_id.to_string())); + assert!( + !store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .expect("decommission state should survive restart") + .complete + ); + com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone()) + .await + .expect("post-crash target manual job should restore"); + com::save_config( + store.pools[manual_job_receipt_pool].clone(), + &manual_job_receipt_path, + manual_job_receipt_bytes, + ) + .await + .expect("manual job receipt should restore after the missing-receipt check"); + + com::save_config(store.pools[1].clone(), &transaction_path, b"{corrupt".to_vec()) + .await + .expect("post-crash target transaction should corrupt deterministically"); + let corrupt_after_crash = store + .complete_decommission(0) + .await + .expect_err("completion must reject a corrupt target after source cleanup and restart") + .to_string(); + assert!(corrupt_after_crash.contains(&transaction_path)); + assert!(corrupt_after_crash.contains(&transaction.transaction_id.to_string())); + com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone()) + .await + .expect("post-crash target transaction should restore"); + + let mut wrong_manual_job = manual_job.clone(); + wrong_manual_job.job_id = uuid::Uuid::new_v4(); + com::save_config( + store.pools[1].clone(), + &manual_job_path, + wrong_manual_job.encode().expect("wrong-id job should encode"), + ) + .await + .expect("post-crash target manual job should accept the wrong-id fixture"); + let wrong_id_after_crash = store + .complete_decommission(0) + .await + .expect_err("completion must reject a target record with the wrong id") + .to_string(); + assert!(wrong_id_after_crash.contains(&manual_job_path)); + assert!(wrong_id_after_crash.contains(&manual_job_id.to_string())); + com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone()) + .await + .expect("post-crash target manual job should restore after the wrong-id check"); + + com::save_config( + store.pools[1].clone(), + &transaction_path, + regressed_transaction + .encode() + .expect("regressed transition transaction should encode"), + ) + .await + .expect("post-crash target transaction should accept the regression fixture"); + let regression_after_crash = store + .complete_decommission(0) + .await + .expect_err("completion must reject a lower transition transaction revision") + .to_string(); + assert!(regression_after_crash.contains("generation mismatch")); + assert!(regression_after_crash.contains(&transaction_path)); + assert!(regression_after_crash.contains(&transaction.transaction_id.to_string())); + com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone()) + .await + .expect("post-crash target transaction should restore after the regression check"); + + let (manual_task_receipt_pool, manual_task_receipt_path) = store + .decommission_durable_ilm_receipt_paths_for_test(0) + .await + .expect("durable ILM receipt paths should be listable") + .into_iter() + .find(|(_, path)| path.contains(&manual_task_path)) + .expect("manual task receipt should retain its reversible source path"); + let manual_task_receipt_bytes = + com::read_config(store.pools[manual_task_receipt_pool].clone(), &manual_task_receipt_path) + .await + .expect("manual task receipt should be readable before corruption"); + com::save_config( + store.pools[manual_task_receipt_pool].clone(), + &manual_task_receipt_path, + b"{corrupt".to_vec(), + ) + .await + .expect("manual task receipt should corrupt deterministically"); + let corrupt_receipt = store + .complete_decommission(0) + .await + .expect_err("completion must fail closed on a corrupt receipt") + .to_string(); + assert!(corrupt_receipt.contains(&manual_task_path)); + assert!(corrupt_receipt.contains(&manual_job_id.to_string())); + com::save_config( + store.pools[manual_task_receipt_pool].clone(), + &manual_task_receipt_path, + manual_task_receipt_bytes, + ) + .await + .expect("manual task receipt should restore after the corruption check"); + + let tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("tier journal recovery should consume the migrated record before completion"); + assert_eq!((tier_stats.scanned, tier_stats.deleted, tier_stats.failed), (1, 1, 0)); + assert!(matches!(com::read_config(store.clone(), &tier_path).await, Err(Error::ConfigNotFound))); + + let recovered_transition_version = "recovered-transition-version".to_string(); + backend + .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::VersionedPresent( + recovered_transition_version.clone(), + ))) + .await; + let transaction_stats = recover_transition_transaction_records(store.clone(), 100, None) + .await + .expect("transition recovery should advance and consume the migrated transaction before completion"); + backend.set_transition_candidate_probe_override(None).await; + assert_eq!( + ( + transaction_stats.scanned, + transaction_stats.recovered, + transaction_stats.retained, + transaction_stats.failed, + ), + (1, 1, 0, 0) + ); + assert!(matches!( + com::read_config(store.clone(), &transaction_path).await, + Err(Error::ConfigNotFound) + )); + com::save_config( + store.pools[1].clone(), + &transaction_path, + regressed_transaction + .encode() + .expect("post-terminal transition rollback should encode"), + ) + .await + .expect("target should accept the post-terminal rollback fixture"); + let post_terminal_regression = store + .complete_decommission(0) + .await + .expect_err("terminal proof must not mask a lower transition revision") + .to_string(); + assert!(post_terminal_regression.contains("generation mismatch")); + assert!(post_terminal_regression.contains(&transaction_path)); + assert!(post_terminal_regression.contains(&transaction.transaction_id.to_string())); + com::delete_config(store.pools[1].clone(), &transaction_path) + .await + .expect("post-terminal rollback fixture should be removed"); + + let manual_stats = recover_manual_transition_jobs_once(store.clone(), 100, None) + .await + .expect("manual recovery should advance the migrated job and consume its scope before completion"); + assert_eq!( + (manual_stats.scanned, manual_stats.resumed, manual_stats.skipped, manual_stats.failed,), + (1, 1, 0, 0) + ); + assert!(matches!( + com::read_config(store.clone(), &manual_scope_path).await, + Err(Error::ConfigNotFound) + )); + let recovered_manual_job_bytes = com::read_config(store.pools[1].clone(), &manual_job_path) + .await + .expect("manual recovery should retain the advanced job record"); + assert_ne!(recovered_manual_job_bytes, manual_job_bytes); + + com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone()) + .await + .expect("target manual job should accept the rollback fixture"); + let manual_regression = store + .complete_decommission(0) + .await + .expect_err("completion must reject a manual job generation rollback") + .to_string(); + assert!(manual_regression.contains("generation mismatch")); + assert!(manual_regression.contains(&manual_job_path)); + assert!(manual_regression.contains(&manual_job_id.to_string())); + com::save_config(store.pools[1].clone(), &manual_job_path, recovered_manual_job_bytes) + .await + .expect("target manual job should restore its recovered generation"); + + store + .complete_decommission(0) + .await + .expect("completion should persist before receipt cleanup"); + assert!( + store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .expect("completed decommission state should remain present") + .complete + ); + assert_eq!( + store + .decommission_durable_ilm_receipt_count_for_test(0) + .await + .expect("receipt cleanup should be observable"), + 0 + ); + store + .cleanup_decommission_durable_ilm_receipts_for_test(0) + .await + .expect("receipt cleanup should be idempotent"); + let removed_versions = backend.remove_versions().await; + assert!(removed_versions.contains(&(tier_entry.obj_name.clone(), tier_entry.version_id.clone()))); + assert!(removed_versions.contains(&(transaction.remote_object.clone(), recovered_transition_version))); + } + #[cfg(feature = "test-util")] async fn tier_delete_journal_count(store: Arc) -> usize { store diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index c7a4e35c6..cc9d123c7 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -2301,7 +2301,9 @@ mod tests { #[serial_test::serial] async fn peer_pool_meta_reload_keeps_active_worker_progress_over_newer_snapshot() { let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-worker", &[2]).await; - *store.decommission_cancelers.write().await = vec![Some(CancellationToken::new())]; + *store.decommission_cancelers.write().await = vec![Some(crate::core::pools::DecommissionCanceler::new_for_test( + CancellationToken::new(), + ))]; let worker_time = OffsetDateTime::now_utc(); let newer_time = worker_time + TimeDuration::seconds(30);