fix(ilm): retry transient scope admission races (#5280)

Co-authored-by: heihutu <heihutu@gmail.com>

crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs

@@ -1015,13 +1015,18 @@ pub async fn claim_manual_transition_scope_admission(
     api: Arc<ECStore>,
     admission: &ManualTransitionScopeAdmission,
 ) -> EcstoreResult<ManualTransitionScopeAdmissionClaim> {
+    loop {
         match save_manual_transition_scope_admission_if_absent(api.clone(), admission).await {
             Ok(()) => return finish_manual_transition_scope_admission_claim(api, admission).await,
             Err(Error::PreconditionFailed) => {}
             Err(err) => return Err(err),
         }
 
-    let (active, etag) = load_manual_transition_scope_admission_with_etag(api.clone(), &admission.scope_key).await?;
+        let (active, etag) = match load_manual_transition_scope_admission_with_etag(api.clone(), &admission.scope_key).await {
+            Ok(active) => active,
+            Err(Error::ConfigNotFound) => continue,
+            Err(err) => return Err(err),
+        };
         let scope_lease_expired = manual_transition_scope_admission_lease_expired(&active);
         let active_job_reclaimable = if active.job_id == admission.job_id {
             scope_lease_expired
@@ -1035,14 +1040,15 @@ pub async fn claim_manual_transition_scope_admission(
             }
         };
         if active_job_reclaimable {
-        return match save_manual_transition_scope_admission_if_current(api.clone(), admission, &etag).await {
+            match save_manual_transition_scope_admission_if_current(api.clone(), admission, &etag).await {
-            Ok(()) => finish_manual_transition_scope_admission_claim(api, admission).await,
+                Ok(()) => return finish_manual_transition_scope_admission_claim(api, admission).await,
-            Err(Error::PreconditionFailed) => Ok(ManualTransitionScopeAdmissionClaim::Conflict(Box::new(active))),
+                Err(Error::PreconditionFailed) => continue,
-            Err(err) => Err(err),
+                Err(err) => return Err(err),
-        };
+            }
         }
 
-    Ok(ManualTransitionScopeAdmissionClaim::Conflict(Box::new(active)))
+        return Ok(ManualTransitionScopeAdmissionClaim::Conflict(Box::new(active)));
+    }
 }
 
 async fn finish_manual_transition_scope_admission_claim(