From caeaa4bf340d41167d034c12fa9bbd26fa350676 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 26 Jul 2026 18:06:24 +0800 Subject: [PATCH] test(ilm): cover durable transition progress recovery (#5286) Co-authored-by: heihutu --- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 90 +++++++++++++++++-- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 6c711c5e6..d5e7aa4d2 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -4605,11 +4605,11 @@ mod tests { lifecycle_rule_has_date_expiration, lifecycle_version_purge_state_from_completed_targets, manual_transition_duration_elapsed, manual_transition_has_more_after_limit, manual_transition_version_marker, mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate, - persist_manual_transition_page_checkpoint, recover_manual_transition_job, recover_manual_transition_jobs, - recover_manual_transition_jobs_once, replication_state_for_delete, resolve_tier_free_version_recovery_enabled, - resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count, - resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop, select_restore_s3_location, - set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer, + persist_manual_transition_job_progress, persist_manual_transition_page_checkpoint, recover_manual_transition_job, + recover_manual_transition_jobs, recover_manual_transition_jobs_once, replication_state_for_delete, + resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, + resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop, + select_restore_s3_location, set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update, should_reuse_lifecycle_delete_replication_state, transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery, }; @@ -8394,6 +8394,86 @@ mod tests { assert!(!after_terminal_cancel.cancel_requested); } + #[tokio::test] + #[serial] + async fn manual_transition_progress_checkpoint_survives_reload_and_recovery() { + let (_paths, ecstore) = setup_test_env().await; + let job_id = Uuid::new_v4(); + let bucket = "manual-progress-checkpoint-bucket"; + let continuation_token = + encode_manual_transition_continuation_token(Some("logs/page-end".to_string()), Some("null".to_string())) + .expect("resume token should encode"); + let options = ManualTransitionRunOptions { + prefix: "logs/".to_string(), + ..Default::default() + }; + let mut record = ManualTransitionJobRecord::new(job_id, bucket, &options, "owner-a"); + record.lease_expires_at_unix_nanos = 0; + save_manual_transition_job_record(ecstore.clone(), &record) + .await + .expect("running job record should save"); + save_manual_transition_scope_admission_if_absent(ecstore.clone(), &ManualTransitionScopeAdmission::from_job(&record)) + .await + .expect("running scope admission should save"); + + let checkpointed = persist_manual_transition_job_progress( + ecstore.clone(), + job_id, + &ManualTransitionRunReport { + bucket: bucket.to_string(), + prefix: "logs/".to_string(), + scanned: 2, + eligible: 2, + enqueued: 2, + continuation_token: Some(continuation_token.clone()), + truncated_by_limit: true, + ..Default::default() + }, + ManualTransitionQueueSnapshot::default(), + ) + .await + .expect("running progress checkpoint should persist"); + + assert_eq!(checkpointed.report.continuation_token.as_deref(), Some(continuation_token.as_str())); + assert_eq!(checkpointed.report.enqueued, 2); + + let reloaded = load_manual_transition_job_record(ecstore.clone(), job_id) + .await + .expect("checkpointed job should reload"); + assert_eq!(reloaded.report.continuation_token.as_deref(), Some(continuation_token.as_str())); + assert_eq!(reloaded.report.scanned, 2); + assert_eq!(reloaded.report.enqueued, 2); + + let mut expired = reloaded.clone(); + expired.lease_expires_at_unix_nanos = 0; + save_manual_transition_job_record(ecstore.clone(), &expired) + .await + .expect("checkpointed job owner loss should persist"); + + let outcome = recover_manual_transition_job(ecstore.clone(), job_id, ManualTransitionQueueSnapshot::default()) + .await + .expect("recovery should fail closed when checkpointed page has pending worker results"); + assert_eq!(outcome, ManualTransitionJobRecoveryOutcome::Unknown); + let recovered = load_manual_transition_job_record(ecstore.clone(), job_id) + .await + .expect("unknown checkpointed job should reload"); + assert_eq!(recovered.state, ManualTransitionJobState::Unknown); + assert!( + recovered + .error + .as_deref() + .is_some_and(|error| error.contains("page/task journal")), + "unknown checkpoint recovery must retain page/task journal context: {recovered:#?}" + ); + assert!( + matches!( + load_manual_transition_scope_admission(ecstore, &record.scope_key).await, + Err(Error::ConfigNotFound) + ), + "unknown checkpoint recovery must release scope admission" + ); + } + #[tokio::test] #[serial] async fn manual_transition_job_lost_worker_results_mark_unknown_and_release_admission() {