feat(scripts): add runbook entrypoints for #1508 #1510 (#5326)

* feat(ecstore): attribute manual transition worker failures

Add manual transition worker failure reason tracking and persistence recovery compatibility for checksum validation.

Co-Authored-By: heihutu <heihutu@gmail.com>

* chore(ilm): add manual transition diagnostics scripts

Co-Authored-By: heihutu <heihutu@gmail.com>

* style(ecstore): format manual transition attribution

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(ecstore): avoid copying failure reasons via clone

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(manual-transition): improve matrix verification scripts

- Add strict mixed rollout phase ratio validation.
- Add strict read/write ratio validation for soak mix.
- Inline generated admin-check command blocks into mixed-rollout run script.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(scripts): add runbook entrypoints for #1508 #1510

---------

Co-authored-by: heihutu <heihutu@gmail.com>

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -9376,6 +9376,9 @@ mod tests {
                 ..Default::default()
             };
             let mut stale = ManualTransitionJobRecord::new(Uuid::new_v4(), &bucket, &options, "old-owner");
+            if stale_case == "missing-job" {
+                stale.lease_expires_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos() - 1;
+            }
             if stale_case == "terminal-job" {
                 stale.complete(
                     ManualTransitionRunReport {
@@ -9412,6 +9415,48 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn manual_transition_admission_concurrent_same_scope_writes_is_singleton() {
+        let (_paths, ecstore) = setup_test_env().await;
+        let options = ManualTransitionRunOptions {
+            prefix: "logs/".to_string(),
+            tier: Some("warm".to_string()),
+            ..Default::default()
+        };
+        let first = ManualTransitionJobRecord::new(Uuid::new_v4(), "manual-concurrent-scope-bucket", &options, "owner-a");
+        let first_admission = ManualTransitionScopeAdmission::from_job(&first);
+        let second = ManualTransitionJobRecord::new(Uuid::new_v4(), "manual-concurrent-scope-bucket", &options, "owner-b");
+        let second_admission = ManualTransitionScopeAdmission::from_job(&second);
+
+        let first_claim = claim_manual_transition_scope_admission(ecstore.clone(), &first_admission);
+        let second_claim = claim_manual_transition_scope_admission(ecstore.clone(), &second_admission);
+        let (first_result, second_result) = tokio::join!(first_claim, second_claim);
+        let first_claim = first_result.expect("first concurrent claim should resolve");
+        let second_claim = second_result.expect("second concurrent claim should resolve");
+
+        let mut claimed = 0;
+        let mut conflicted = 0;
+        let mut active_job_id = None;
+        for item in [first_claim, second_claim] {
+            match item {
+                ManualTransitionScopeAdmissionClaim::Claimed => claimed += 1,
+                ManualTransitionScopeAdmissionClaim::Conflict(active) => {
+                    conflicted += 1;
+                    active_job_id = Some(active.job_id);
+                }
+            }
+        }
+
+        assert_eq!(claimed, 1, "only one concurrent same-scope claim should be accepted");
+        assert_eq!(conflicted, 1, "only one concurrent same-scope claim should report conflict");
+        let active = load_manual_transition_scope_admission(ecstore.clone(), &first.scope_key)
+            .await
+            .expect("scope admission should remain");
+        assert!(active.job_id == first.job_id || active.job_id == second.job_id);
+        assert_eq!(active_job_id, Some(active.job_id), "conflict response must carry active owner");
+    }
+
     #[tokio::test]
     async fn existing_object_lifecycle_allows_expired_marker_after_replication_completed() {
         let lc = expired_delete_marker_lifecycle();

crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs

@@ -1405,7 +1405,9 @@ pub async fn claim_manual_transition_scope_admission(
                 Ok(active_job) => {
                     active_job.is_terminal() || (scope_lease_expired && manual_transition_job_lease_expired(&active_job))
                 }
-                Err(Error::ConfigNotFound) => true,
+                // Missing active job metadata can be transient (for example, immediately after admission creation);
+                // require an expired scope lease before treating it as reclaimable.
+                Err(Error::ConfigNotFound) => scope_lease_expired,
                 Err(err) => return Err(err),
             }
         };