Merge branch 'main' into houseme/test/scanner-heal-v2-w20

Handle exact threshold comparisons without binary floating point boundary drift and mark unstable P1 walk controls inconclusive.

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

.config/make/tests.mak

@@ -31,6 +31,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_object_batch_bench_enhanced.sh
 	./scripts/test_hotpath_warp_ab_gate.sh
 	./scripts/test_hotpath_warp_abba.sh
+	./scripts/test_scanner_validation_harness.sh
 	./scripts/test_exact_1mib_handoff_abba.sh
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh

crates/e2e_test/src/on_demand_migration/concurrency_test.rs

@@ -20,10 +20,9 @@
 //! journal (`count_requests`) carries the assertion in every one of them.
 
 use super::common::{BoxError, OdmTestEnv, RawResponse, SeedObject, start_configured_env};
-use crate::fake_s3_target::{FaultAction, Operation};
+use crate::fake_s3_target::Operation;
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 use bytes::Bytes;
-use futures::{StreamExt, TryStreamExt};
 use std::time::Duration;
 
 type TestResult = Result<(), BoxError>;
@@ -146,38 +145,14 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
     .await?;
 
     let body = payload(128 * 1024);
-    let blocker = "queue/blocker.bin";
-    env.seed_source(SOURCE_BUCKET, &[SeedObject::new(blocker, body.clone())]);
-    // The one-chunk range completes immediately; its full background pull
-    // occupies the only slot while the remaining requests fill the queue.
-    env.source.inject_for_key(
-        Operation::GetObject,
-        blocker,
-        FaultAction::SlowSendBody {
-            chunk_bytes: 1024,
-            delay: Duration::from_millis(100),
-        },
-        2,
-    );
-    let response = env
-        .raw_object_request(http::Method::GET, bucket, blocker, &[("range", "bytes=0-1023")])
-        .await?;
-    assert_eq!(response.status, 206);
-    assert_eq!(response.body, body.slice(0..1024));
-    env.wait_for_status_counter(bucket, "/inflight_pulls", 1, SETTLE).await?;
-
     let keys: Vec<String> = (0..REQUESTS).map(|index| format!("queue/object-{index:03}.bin")).collect();
     let seeds: Vec<SeedObject> = keys.iter().map(|key| SeedObject::new(key.clone(), body.clone())).collect();
     env.seed_source(SOURCE_BUCKET, &seeds);
 
-    // Bound source connections below the fixture's limit while still
-    // submitting all 100 requests to the eight-slot background queue.
-    let responses: Vec<RawResponse> = futures::stream::iter(
+    let responses: Vec<RawResponse> = futures::future::try_join_all(
         keys.iter()
             .map(|key| env.raw_object_request(http::Method::GET, bucket, key, &[("range", "bytes=0-1023")])),
     )
-    .buffered(16)
-    .try_collect()
     .await?;
     for (key, response) in keys.iter().zip(&responses) {
         assert_eq!(response.status, 206, "{key}: {}", String::from_utf8_lossy(&response.body));
@@ -193,15 +168,6 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
         .wait_for_status_counter(bucket, "/counters/pull_failures_total/queue_full", 1, SETTLE)
         .await?;
     assert!(queue_full > 0, "a 100-deep burst must overflow an 8-slot queue");
-    let queue_full = usize::try_from(queue_full)?;
-    assert!(queue_full <= REQUESTS);
-    env.wait_for_status_counter(
-        bucket,
-        "/counters/pulled_objects_total/background",
-        u64::try_from(REQUESTS + 1 - queue_full)?,
-        SETTLE,
-    )
-    .await?;
 
     let ranged_reads: usize = keys.iter().map(|key| source_get_count(&env, key)).sum();
     assert!(
@@ -209,6 +175,9 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
         "every reader is served from the source: {ranged_reads} GETs for {REQUESTS} readers"
     );
     let dropped = keys.iter().filter(|key| source_get_count(&env, key) == 1).count();
-    assert_eq!(dropped, queue_full, "only overflowed keys remain without a background GET");
+    assert!(
+        dropped > 0,
+        "the overflowed keys are the ones with no backfill GET, but every key got one"
+    );
     Ok(())
 }

crates/e2e_test/src/on_demand_migration/list_through_test.rs

@@ -265,13 +265,16 @@ async fn list_through_rejects_a_tampered_continuation_token() -> TestResult {
     let decoded = String::from_utf8(base64_simd::STANDARD.decode_to_vec(token.as_bytes())?)?;
     assert!(decoded.contains("\"t\":\"odm-list\""), "the merged token is an envelope: {decoded}");
 
-    let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":3").as_bytes());
-    assert_ne!(tampered, token, "the test must change the token version");
-    let query = serde_urlencoded::to_string([("continuation-token", tampered.as_str())])?;
-    let rejected = env.raw_list_objects_v2(bucket, &query).await?;
-    let error_body = String::from_utf8_lossy(&rejected.body);
-    assert_eq!(rejected.status, 400, "a bumped token version is a client error: {}", error_body);
-    assert!(error_body.contains("<Code>InvalidArgument</Code>"), "{error_body}");
+    let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":2").as_bytes());
+    let rejected = env
+        .raw_list_objects_v2(bucket, &format!("continuation-token={tampered}"))
+        .await?;
+    assert_eq!(
+        rejected.status,
+        400,
+        "a bumped token version is a client error: {}",
+        String::from_utf8_lossy(&rejected.body)
+    );
     Ok(())
 }
 

crates/ecstore/src/api/mod.rs

@@ -69,13 +69,6 @@ pub mod bucket {
             };
         }
 
-        pub mod recovery_control {
-            pub use crate::bucket::lifecycle::recovery_control::{
-                IlmRecoveryClassification, IlmRecoveryControlPage, IlmRecoveryControlView, IlmRecoveryProtocol,
-                inspect_recovery_control, list_recovery_controls,
-            };
-        }
-
         pub mod transition_transaction {
             pub use crate::bucket::lifecycle::transition_transaction::{
                 TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,

crates/ecstore/src/bucket/lifecycle/durable_namespace.rs

@@ -22,7 +22,7 @@ use super::{
     bucket_lifecycle_ops::{
         ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token,
     },
-    manual_transition_job, recovery_control, tier_delete_journal, transition_transaction,
+    manual_transition_job, tier_delete_journal, transition_transaction,
 };
 use crate::error::{Error, Result};
 use crate::services::tier::tier_probe_intent;
@@ -41,7 +41,6 @@ pub(crate) enum DurableIlmRecordKind {
     ManualTransitionScope,
     ManualTransitionTask,
     ManualTransitionWorkerResult,
-    RecoveryControl,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -106,14 +105,8 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
     max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE,
     kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
 };
-pub(crate) const RECOVERY_CONTROL_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
-    name: "recovery-control",
-    prefix: recovery_control::ILM_RECOVERY_CONTROL_PREFIX,
-    max_record_size: recovery_control::MAX_ILM_RECOVERY_CONTROL_SIZE,
-    kind: DurableIlmRecordKind::RecoveryControl,
-};
 
-pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
+pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
     TIER_DELETE_JOURNAL_NAMESPACE,
     TIER_DELETE_JOURNAL_V6_NAMESPACE,
     TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
@@ -123,7 +116,6 @@ pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
     MANUAL_TRANSITION_SCOPE_NAMESPACE,
     MANUAL_TRANSITION_TASK_NAMESPACE,
     MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
-    RECOVERY_CONTROL_NAMESPACE,
 ];
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -249,18 +241,6 @@ pub(crate) enum DurableIlmRecordCheckpoint {
     ManualTransitionWorkerResult {
         content_sha256: String,
     },
-    RecoveryControl {
-        content_sha256: String,
-        identity_sha256: String,
-        source_generation_sha256: String,
-        first_seen_at_unix_nanos: i64,
-        revision: u64,
-        classification: recovery_control::IlmRecoveryClassification,
-        attempt_count: u64,
-        consecutive_failure_count: u32,
-        #[serde(default, skip_serializing_if = "Option::is_none")]
-        owner_fence_sha256: Option<String>,
-    },
 }
 
 impl DurableIlmRecordCheckpoint {
@@ -274,8 +254,7 @@ impl DurableIlmRecordCheckpoint {
             | Self::ManualTransitionJob { content_sha256, .. }
             | Self::ManualTransitionScope { content_sha256, .. }
             | Self::ManualTransitionTask { content_sha256 }
-            | Self::ManualTransitionWorkerResult { content_sha256 }
-            | Self::RecoveryControl { content_sha256, .. } => content_sha256,
+            | Self::ManualTransitionWorkerResult { content_sha256 } => content_sha256,
         }
     }
 
@@ -549,51 +528,6 @@ impl DurableIlmRecordCheckpoint {
                     ..
                 },
             ) => previous_identity == next_identity && next_updated_at > previous_updated_at,
-            (
-                Self::RecoveryControl {
-                    identity_sha256: previous_identity,
-                    source_generation_sha256: previous_generation,
-                    first_seen_at_unix_nanos: previous_first_seen,
-                    revision: previous_revision,
-                    classification: previous_classification,
-                    attempt_count: previous_attempts,
-                    consecutive_failure_count: previous_failures,
-                    owner_fence_sha256: previous_owner,
-                    ..
-                },
-                Self::RecoveryControl {
-                    identity_sha256: next_identity,
-                    source_generation_sha256: next_generation,
-                    first_seen_at_unix_nanos: next_first_seen,
-                    revision: next_revision,
-                    classification: next_classification,
-                    attempt_count: next_attempts,
-                    consecutive_failure_count: next_failures,
-                    owner_fence_sha256: next_owner,
-                    ..
-                },
-            ) => {
-                let adjacent = previous_identity == next_identity
-                    && previous_first_seen == next_first_seen
-                    && previous_revision.checked_add(1) == Some(*next_revision);
-                let claim = next_owner.is_some()
-                    && *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
-                    && *next_classification == recovery_control::IlmRecoveryClassification::Retrying
-                    && previous_attempts.checked_add(1) == Some(*next_attempts)
-                    && previous_failures == next_failures;
-                let source_refresh = previous_owner.is_some()
-                    && previous_owner == next_owner
-                    && *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
-                    && *next_classification == recovery_control::IlmRecoveryClassification::Retrying
-                    && previous_attempts == next_attempts
-                    && previous_failures == next_failures
-                    && previous_generation != next_generation;
-                let completion = previous_owner.is_some()
-                    && next_owner.is_none()
-                    && previous_generation == next_generation
-                    && previous_attempts == next_attempts;
-                adjacent && (claim || source_refresh || completion)
-            }
             _ => false,
         };
 
@@ -619,14 +553,6 @@ impl DurableIlmRecordCheckpoint {
         {
             return false;
         }
-        if let Self::RecoveryControl { classification, .. } = terminal
-            && !matches!(
-                classification,
-                recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned
-            )
-        {
-            return false;
-        }
         if self == terminal || self.validate_successor(terminal).is_ok() {
             return true;
         }
@@ -726,32 +652,6 @@ impl DurableIlmRecordCheckpoint {
                         .is_some_and(|distance| tier_probe_state_reaches(*previous_state, *terminal_state, distance))
                     && (!previous_remote_version_known || previous_remote_version == terminal_remote_version)
             }
-            (
-                Self::RecoveryControl {
-                    identity_sha256: previous_identity,
-                    source_generation_sha256: previous_generation,
-                    first_seen_at_unix_nanos: previous_first_seen,
-                    revision: previous_revision,
-                    attempt_count: previous_attempts,
-                    ..
-                },
-                Self::RecoveryControl {
-                    identity_sha256: terminal_identity,
-                    source_generation_sha256: terminal_generation,
-                    first_seen_at_unix_nanos: terminal_first_seen,
-                    revision: terminal_revision,
-                    attempt_count: terminal_attempts,
-                    classification:
-                        recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned,
-                    ..
-                },
-            ) => {
-                previous_identity == terminal_identity
-                    && (previous_generation == terminal_generation || terminal_attempts > previous_attempts)
-                    && previous_first_seen == terminal_first_seen
-                    && terminal_revision > previous_revision
-                    && terminal_attempts >= previous_attempts
-            }
             _ => false,
         }
     }
@@ -1319,35 +1219,6 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
                 },
             )
         }
-        DurableIlmRecordKind::RecoveryControl => {
-            let (protocol, control_id) = recovery_control::recovery_control_id_from_record_object_name(path)
-                .map_err(|err| Error::other(err.to_string()))?;
-            let control =
-                recovery_control::IlmRecoveryControl::decode(&control_id, data).map_err(|err| Error::other(err.to_string()))?;
-            let canonical = recovery_control::recovery_control_record_object_name(protocol, &control_id)
-                .map_err(|err| Error::other(err.to_string()))?;
-            if canonical != path || control.identity.protocol != protocol {
-                return Err(Error::other("ILM recovery control path is not canonical"));
-            }
-            let identity_sha256 = checkpoint_hash(&control.identity)?;
-            let source_generation_sha256 = checkpoint_hash(&control.observed_source_generation)?;
-            let owner_fence_sha256 = control.owner.as_ref().map(checkpoint_hash).transpose()?;
-            (
-                "control_id",
-                control_id,
-                DurableIlmRecordCheckpoint::RecoveryControl {
-                    content_sha256,
-                    identity_sha256,
-                    source_generation_sha256,
-                    first_seen_at_unix_nanos: control.first_seen_at_unix_nanos,
-                    revision: control.revision,
-                    classification: control.classification,
-                    attempt_count: control.attempt_count,
-                    consecutive_failure_count: control.consecutive_failure_count,
-                    owner_fence_sha256,
-                },
-            )
-        }
         DurableIlmRecordKind::ManualTransitionJob => {
             let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
                 .map_err(|err| Error::other(err.to_string()))?;
@@ -1541,94 +1412,6 @@ mod tests {
             .checkpoint
     }
 
-    fn recovery_control_fixture() -> recovery_control::IlmRecoveryControl {
-        let source_path = "ilm/transition-transactions/records/12/34/1234567890abcdef1234567890abcdef.json";
-        let generation = recovery_control::IlmRecoverySourceGeneration::new(
-            transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
-            "source-etag",
-            "a".repeat(64),
-            vec![recovery_control::IlmRecoverySourceCopy {
-                authority: "pool-0/set-0".to_string(),
-                canonical_path: source_path.to_string(),
-                etag: "source-etag".to_string(),
-                encoded_len: 128,
-                content_sha256: "a".repeat(64),
-            }],
-        )
-        .expect("source generation should build");
-        recovery_control::IlmRecoveryControl::new(
-            recovery_control::IlmRecoveryControlIdentity {
-                protocol: recovery_control::IlmRecoveryProtocol::TransitionTransaction,
-                canonical_source_path: source_path.to_string(),
-                stable_operation_identity: "12345678-90ab-cdef-1234-567890abcdef".to_string(),
-                record_class: "transition_transaction_v1".to_string(),
-            },
-            generation,
-            recovery_control::IlmRecoveryClassification::Retrying,
-            1_000_000_000,
-            recovery_control::IlmRecoveryErrorCode::None,
-        )
-        .expect("recovery control should build")
-    }
-
-    fn recovery_control_checkpoint(control: &recovery_control::IlmRecoveryControl) -> DurableIlmRecordCheckpoint {
-        let control_id = control.identity.source_operation_digest().expect("control id should derive");
-        let path = recovery_control::recovery_control_record_object_name(control.identity.protocol, &control_id)
-            .expect("control path should build");
-        let encoded = control.encode().expect("control should encode");
-        let namespace = classify_durable_ilm_record(&path)
-            .expect("recovery control namespace should classify")
-            .expect("recovery control should be durable");
-        assert_eq!(namespace, &RECOVERY_CONTROL_NAMESPACE);
-        validate_durable_ilm_record(&path, &encoded)
-            .expect("recovery control should validate")
-            .checkpoint
-    }
-
-    #[test]
-    fn recovery_control_checkpoint_tracks_claim_retry_and_terminal_generations() {
-        let initial_control = recovery_control_fixture();
-        let initial = recovery_control_checkpoint(&initial_control);
-
-        let mut claimed_control = initial_control;
-        let mut advanced_generation = claimed_control.observed_source_generation.clone();
-        advanced_generation.source_schema = "rustfs-transition-transaction-v2".to_string();
-        claimed_control
-            .claim_for_source_generation("node-a", Uuid::new_v4(), 2_000_000_000, 300_000_000_000, advanced_generation)
-            .expect("control should claim");
-        let claimed = recovery_control_checkpoint(&claimed_control);
-        initial.validate_successor(&claimed).expect("claim should advance receipt");
-
-        let mut retry_control = claimed_control;
-        retry_control
-            .record_retryable_failure(3_000_000_000, recovery_control::IlmRecoveryErrorCode::BackendTimeout)
-            .expect("retry should persist");
-        let retry = recovery_control_checkpoint(&retry_control);
-        claimed.validate_successor(&retry).expect("retry should advance receipt");
-
-        let ready_at = retry_control
-            .next_attempt_at_unix_nanos
-            .expect("retry deadline should persist");
-        let mut terminal_control = retry_control;
-        terminal_control
-            .claim("node-b", Uuid::new_v4(), ready_at, 300_000_000_000)
-            .expect("retry should claim");
-        let reclaimed = recovery_control_checkpoint(&terminal_control);
-        retry.validate_successor(&reclaimed).expect("reclaim should advance receipt");
-        terminal_control
-            .finish_attempt(
-                recovery_control::IlmRecoveryClassification::Terminal,
-                recovery_control::IlmRecoveryErrorCode::None,
-            )
-            .expect("control should terminate");
-        let terminal = recovery_control_checkpoint(&terminal_control);
-        reclaimed
-            .validate_successor(&terminal)
-            .expect("terminal state should advance receipt");
-        assert!(initial.is_predecessor_of_terminal(&terminal));
-        assert!(!initial.is_predecessor_of_terminal(&retry));
-    }
-
     #[test]
     fn tier_probe_intent_checkpoint_tracks_exact_monotonic_generations() {
         let initial_intent = tier_probe_intent_fixture();

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -24,7 +24,6 @@ pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, g
 mod object_handlers_common;
 mod object_lock_boundary;
 pub use self::core as lifecycle;
-pub mod recovery_control;
 mod replication_sink;
 pub mod rule;
 mod runtime_boundary;

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -23,11 +23,6 @@ use uuid::Uuid;
 use crate::bucket::lifecycle::config_boundary;
 use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
-use crate::bucket::lifecycle::recovery_control::{
-    IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol,
-    ObservedIlmRecoveryControl, load_recovery_control, observe_recovery_source, recovery_control_record_object_name,
-    save_recovery_control_if_absent, save_recovery_control_if_current,
-};
 use crate::bucket::lifecycle::tier_sweeper::{
     delete_confirmed_transition_candidate_exact_with_lease_idempotent,
     delete_object_from_remote_tier_idempotent_with_manager_and_identity,
@@ -49,7 +44,6 @@ const EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY: &str = "lifecycle_transit
 pub const DEFAULT_TRANSITION_TRANSACTION_RECOVERY_LIMIT: usize = 1_000;
 const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(60);
 const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
-const TRANSITION_RECOVERY_CONTROL_LEASE_NANOS: i64 = 15 * 60 * 1_000_000_000;
 pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
 pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
 pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
@@ -743,8 +737,6 @@ pub enum TransitionTransactionRecoveryOutcome {
     RemoteCandidateDeleted,
     RecordDeleted,
     Retained,
-    RetainedAmbiguous(IlmRecoveryErrorCode),
-    OperatorRequired(IlmRecoveryErrorCode),
 }
 
 #[cfg(test)]
@@ -825,80 +817,6 @@ async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
     }
 }
 
-#[cfg(test)]
-#[derive(Default)]
-struct TransitionRecoveryTerminalBarrierState {
-    transaction_id: Uuid,
-    arrived: tokio::sync::Notify,
-    release: tokio::sync::Notify,
-}
-
-#[cfg(test)]
-pub(crate) struct TransitionRecoveryTerminalBarrier {
-    state: Arc<TransitionRecoveryTerminalBarrierState>,
-}
-
-#[cfg(test)]
-static TRANSITION_RECOVERY_TERMINAL_BARRIER: std::sync::OnceLock<
-    std::sync::Mutex<Option<Arc<TransitionRecoveryTerminalBarrierState>>>,
-> = std::sync::OnceLock::new();
-
-#[cfg(test)]
-impl TransitionRecoveryTerminalBarrier {
-    pub(crate) fn install(transaction_id: Uuid) -> Self {
-        let state = Arc::new(TransitionRecoveryTerminalBarrierState {
-            transaction_id,
-            ..Default::default()
-        });
-        let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER
-            .get_or_init(|| std::sync::Mutex::new(None))
-            .lock()
-            .expect("transition recovery terminal barrier mutex should not poison");
-        assert!(
-            slot.is_none(),
-            "transition recovery terminal barrier must be installed by one test at a time"
-        );
-        *slot = Some(Arc::clone(&state));
-        drop(slot);
-        Self { state }
-    }
-
-    pub(crate) async fn wait_until_paused(&self) {
-        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
-            .await
-            .expect("transition recovery should persist terminal control before source cleanup");
-    }
-}
-
-#[cfg(test)]
-impl Drop for TransitionRecoveryTerminalBarrier {
-    fn drop(&mut self) {
-        self.state.release.notify_one();
-        let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER
-            .get_or_init(|| std::sync::Mutex::new(None))
-            .lock()
-            .expect("transition recovery terminal barrier mutex should not poison");
-        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
-            *slot = None;
-        }
-    }
-}
-
-#[cfg(test)]
-async fn pause_after_transition_recovery_terminal(transaction_id: Uuid) {
-    let barrier = TRANSITION_RECOVERY_TERMINAL_BARRIER
-        .get_or_init(|| std::sync::Mutex::new(None))
-        .lock()
-        .expect("transition recovery terminal barrier mutex should not poison")
-        .as_ref()
-        .filter(|barrier| barrier.transaction_id == transaction_id)
-        .cloned();
-    if let Some(barrier) = barrier {
-        barrier.arrived.notify_one();
-        barrier.release.notified().await;
-    }
-}
-
 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
 #[serde(rename_all = "snake_case")]
 pub enum TransitionOperatorProbe {
@@ -1102,35 +1020,17 @@ fn transition_transaction_id_from_record_object_name(object: &str) -> Result<Uui
     let suffix = object
         .strip_prefix(&prefix)
         .ok_or(TransitionTransactionError::Corrupt("transaction record path has wrong prefix"))?;
-    let mut parts = suffix.split('/');
-    let shard_a = parts
+    let file_name = suffix
+        .rsplit('/')
         .next()
         .ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
-    let shard_b = parts
-        .next()
-        .ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
-    let file_name = parts
-        .next()
-        .ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
-    if parts.next().is_some() {
-        return Err(TransitionTransactionError::Corrupt("transaction record path is not canonical"));
-    }
     let transaction_key = file_name
         .strip_suffix(".json")
         .ok_or(TransitionTransactionError::Corrupt("transaction record path has wrong suffix"))?;
-    if transaction_key.len() != 32
-        || !transaction_key
-            .bytes()
-            .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
-        || shard_a != &transaction_key[..2]
-        || shard_b != &transaction_key[2..4]
-    {
+    if transaction_key.len() != 32 || !transaction_key.bytes().all(|byte| byte.is_ascii_hexdigit()) {
         return Err(TransitionTransactionError::Corrupt("transaction record path has invalid transaction id"));
     }
-    Uuid::parse_str(transaction_key)
-        .ok()
-        .filter(|transaction_id| !transaction_id.is_nil())
-        .ok_or(TransitionTransactionError::Corrupt("transaction record path has invalid uuid"))
+    Uuid::parse_str(transaction_key).map_err(|_| TransitionTransactionError::Corrupt("transaction record path has invalid uuid"))
 }
 
 pub async fn process_transition_transaction_record(
@@ -1155,27 +1055,6 @@ async fn process_transition_transaction_record_at(
 ) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
     let record_name =
         transition_transaction_record_object_name(observed.transaction_id).map_err(transition_transaction_store_error)?;
-    let now_unix_nanos =
-        i64::try_from(now_unix_nanos).map_err(|_| Error::other("transition transaction recovery timestamp does not fit i64"))?;
-    let recovery_control_identity = transition_recovery_control_identity(observed, &record_name);
-    let recovery_control_id = recovery_control_identity
-        .source_operation_digest()
-        .map_err(|err| Error::other(err.to_string()))?;
-    let control_record_name =
-        recovery_control_record_object_name(IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
-            .map_err(|err| Error::other(err.to_string()))?;
-    let control_lock = if transition_state_needs_recovery_control(observed, now_unix_nanos) {
-        Some(
-            api.new_ns_lock(RUSTFS_META_BUCKET, &format!("{control_record_name}.recovery-lock"))
-                .await?,
-        )
-    } else {
-        None
-    };
-    let _control_guard = match &control_lock {
-        Some(lock) => Some(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?),
-        None => None,
-    };
     // The synthetic key avoids nesting the recovery lock with the config
     // object's own I/O lock. Holding it across the bounded source proof and
     // remote DELETE elects one destructive recovery worker across nodes.
@@ -1194,400 +1073,55 @@ async fn process_transition_transaction_record_at(
         return Ok(TransitionTransactionRecoveryOutcome::Retained);
     }
 
-    let mut recovery_control = if transition_state_needs_recovery_control(&current, now_unix_nanos) {
-        if cleanup_terminal_transition_recovery_control(
-            api.clone(),
-            &current,
-            &record_name,
-            &recovery_control_identity,
-            &recovery_control_id,
-        )
-        .await?
-        {
-            return Ok(TransitionTransactionRecoveryOutcome::RecordDeleted);
-        }
-        match claim_transition_recovery_control(
-            api.clone(),
-            &current,
-            &record_name,
-            recovery_control_identity,
-            &recovery_control_id,
-            now_unix_nanos,
-        )
-        .await?
-        {
-            Some(control) => Some(control),
-            None => return Ok(TransitionTransactionRecoveryOutcome::Retained),
-        }
-    } else {
-        None
-    };
-
-    let recovery = match current.state {
+    match current.state {
         TransitionTransactionState::Uploaded => {
-            if transition_transaction_ownership_is_active(&current, i128::from(now_unix_nanos)) {
-                Ok(TransitionTransactionRecoveryOutcome::Retained)
-            } else {
-                let mut cleanup = current.clone();
-                cleanup
-                    .mark_cleanup_pending(
-                        current.fence(),
-                        TransitionCleanupProof {
-                            transaction_id: current.transaction_id,
-                            write_id: current.write_id,
-                            remote_object: current.remote_object.clone(),
-                            remote_version: current.remote_version.clone(),
-                            backend_fingerprint: current.backend_fingerprint,
-                            decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
-                        },
-                    )
-                    .map_err(transition_transaction_store_error)?;
-                #[cfg(test)]
-                pause_before_transition_recovery_claim(current.transaction_id).await;
-                match save_transition_transaction_record_if_current(api.clone(), &current, &cleanup).await {
-                    Ok(()) => recover_cleanup_pending(api.clone(), &cleanup).await,
-                    Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => {
-                        Ok(TransitionTransactionRecoveryOutcome::Retained)
-                    }
-                    Err(err) => Err(err),
-                }
+            if transition_transaction_ownership_is_active(&current, now_unix_nanos) {
+                return Ok(TransitionTransactionRecoveryOutcome::Retained);
+            }
+            let mut cleanup = current.clone();
+            cleanup
+                .mark_cleanup_pending(
+                    current.fence(),
+                    TransitionCleanupProof {
+                        transaction_id: current.transaction_id,
+                        write_id: current.write_id,
+                        remote_object: current.remote_object.clone(),
+                        remote_version: current.remote_version.clone(),
+                        backend_fingerprint: current.backend_fingerprint,
+                        decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
+                    },
+                )
+                .map_err(transition_transaction_store_error)?;
+            #[cfg(test)]
+            pause_before_transition_recovery_claim(current.transaction_id).await;
+            match save_transition_transaction_record_if_current(api.clone(), &current, &cleanup).await {
+                Ok(()) => recover_cleanup_pending(api, &cleanup).await,
+                Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => Ok(TransitionTransactionRecoveryOutcome::Retained),
+                Err(err) => Err(err),
             }
         }
-        TransitionTransactionState::CleanupPending => recover_cleanup_pending(api.clone(), &current).await,
+        TransitionTransactionState::CleanupPending => recover_cleanup_pending(api, &current).await,
         TransitionTransactionState::LocalCommitStarted => match local_commit_matches_transaction(api.clone(), &current).await {
-            Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
-            Ok(false) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(
-                IlmRecoveryErrorCode::LocalCommitAmbiguous,
-            )),
-            Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(
-                IlmRecoveryErrorCode::LocalCommitAmbiguous,
-            )),
+            Ok(true) => {
+                delete_transition_transaction_record(api, &current).await?;
+                Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
+            }
+            Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
+            Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::Retained),
             Err(err) => Err(err),
         },
         TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
+            delete_transition_transaction_record(api, &current).await?;
             Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
         }
         TransitionTransactionState::UploadOutcomeUnknown => {
-            if transition_transaction_ownership_is_active(&current, i128::from(now_unix_nanos)) {
+            if transition_transaction_ownership_is_active(&current, now_unix_nanos) {
                 Ok(TransitionTransactionRecoveryOutcome::Retained)
             } else {
-                recover_unknown_upload_outcome(api.clone(), &current).await
+                recover_unknown_upload_outcome(api, &current).await
             }
         }
-        TransitionTransactionState::UploadStarted => {
-            if transition_transaction_ownership_is_active(&current, i128::from(now_unix_nanos)) {
-                Ok(TransitionTransactionRecoveryOutcome::Retained)
-            } else {
-                Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
-                    IlmRecoveryErrorCode::RemoteVersionUnknown,
-                ))
-            }
-        }
-    };
-
-    if let Some(mut control) = recovery_control.take() {
-        let source_to_delete = if matches!(
-            recovery,
-            Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted
-                | TransitionTransactionRecoveryOutcome::RecordDeleted)
-        ) {
-            let refreshed =
-                refresh_transition_recovery_control_source(api.clone(), control, &record_name, current.transaction_id).await?;
-            control = refreshed.0;
-            refreshed.1
-        } else {
-            None
-        };
-        persist_transition_recovery_result(api.clone(), control, &recovery, now_unix_nanos).await?;
-        if let Some(source) = source_to_delete {
-            #[cfg(test)]
-            pause_after_transition_recovery_terminal(source.transaction_id).await;
-            delete_transition_transaction_record(api, &source).await?;
-        }
-    } else if matches!(
-        recovery,
-        Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted)
-    ) {
-        delete_transition_transaction_record(api, &current).await?;
-    }
-    recovery
-}
-
-fn transition_recovery_control_identity(transaction: &TransitionTransaction, record_name: &str) -> IlmRecoveryControlIdentity {
-    IlmRecoveryControlIdentity {
-        protocol: IlmRecoveryProtocol::TransitionTransaction,
-        canonical_source_path: record_name.to_string(),
-        stable_operation_identity: transaction.transaction_id.to_string(),
-        record_class: "transition_transaction_v1".to_string(),
-    }
-}
-
-#[cfg(test)]
-pub(crate) fn transition_recovery_control_id(transaction: &TransitionTransaction) -> Result<String> {
-    let record_name = transition_transaction_record_object_name(transaction.transaction_id)?;
-    transition_recovery_control_identity(transaction, &record_name)
-        .source_operation_digest()
-        .map_err(|_| TransitionTransactionError::Corrupt("transition recovery control identity is invalid"))
-}
-
-fn transition_state_needs_recovery_control(transaction: &TransitionTransaction, now_unix_nanos: i64) -> bool {
-    now_unix_nanos >= transaction.not_after_unix_nanos
-        && !matches!(
-            transaction.state,
-            TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed
-        )
-}
-
-async fn cleanup_terminal_transition_recovery_control(
-    api: Arc<ECStore>,
-    transaction: &TransitionTransaction,
-    record_name: &str,
-    identity: &IlmRecoveryControlIdentity,
-    control_id: &str,
-) -> EcstoreResult<bool> {
-    let observed = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
-        Ok(observed) => observed,
-        Err(Error::ConfigNotFound) => return Ok(false),
-        Err(err) => return Err(err),
-    };
-    if observed.control.classification != IlmRecoveryClassification::Terminal {
-        return Ok(false);
-    }
-    let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?;
-    let exact_source = source.is_consistent()
-        && source.generation == observed.control.observed_source_generation
-        && source.canonical_data.as_deref().is_some_and(|data| {
-            TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|decoded| decoded == *transaction)
-        });
-    if observed.control.identity != *identity || !exact_source {
-        return Ok(false);
-    }
-    delete_transition_transaction_record(api, transaction).await?;
-    Ok(true)
-}
-
-async fn claim_transition_recovery_control(
-    api: Arc<ECStore>,
-    transaction: &TransitionTransaction,
-    record_name: &str,
-    identity: IlmRecoveryControlIdentity,
-    control_id: &str,
-    now_unix_nanos: i64,
-) -> EcstoreResult<Option<ObservedIlmRecoveryControl>> {
-    let existing = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
-        Ok(control) => Some(control),
-        Err(Error::ConfigNotFound) => None,
-        Err(err) => return Err(err),
-    };
-    if let Some(observed) = existing.as_ref() {
-        if observed.control.identity != identity {
-            return Ok(None);
-        }
-        if observed
-            .control
-            .owner
-            .as_ref()
-            .is_some_and(|owner| owner.lease_expires_at_unix_nanos <= now_unix_nanos)
-        {
-            let mut expired = observed.control.clone();
-            expired
-                .record_expired_attempt(now_unix_nanos)
-                .map_err(|err| Error::other(err.to_string()))?;
-            save_recovery_control_if_current(api, observed, &expired).await?;
-            return Ok(None);
-        }
-        if !observed.control.should_attempt_at(now_unix_nanos) {
-            return Ok(None);
-        }
-    }
-
-    let source = match observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await {
-        Ok(source) => source,
-        Err(err) => {
-            if let Some(observed) = existing {
-                persist_transition_recovery_source_failure(api, observed, now_unix_nanos).await?;
-                return Ok(None);
-            }
-            return Err(err);
-        }
-    };
-    let source_matches = source.is_consistent()
-        && source.canonical_data.as_deref().is_some_and(|data| {
-            TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|observed| observed == *transaction)
-        });
-    let source_error = if source_matches {
-        IlmRecoveryErrorCode::None
-    } else if source.canonical_data.is_some() {
-        IlmRecoveryErrorCode::SourceGenerationChanged
-    } else {
-        IlmRecoveryErrorCode::SourceDivergent
-    };
-
-    let mut observed = match existing {
-        Some(control) => control,
-        None => {
-            let candidate = IlmRecoveryControl::new(
-                identity.clone(),
-                source.generation.clone(),
-                if source_matches {
-                    IlmRecoveryClassification::Retrying
-                } else {
-                    IlmRecoveryClassification::Corrupt
-                },
-                now_unix_nanos,
-                source_error,
-            )
-            .map_err(|err| Error::other(err.to_string()))?;
-            match save_recovery_control_if_absent(api.clone(), &candidate).await {
-                Ok(()) | Err(Error::PreconditionFailed) => {}
-                Err(err) => return Err(err),
-            }
-            load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?
-        }
-    };
-    if observed.control.identity != identity || !observed.control.should_attempt_at(now_unix_nanos) {
-        return Ok(None);
-    }
-
-    let mut claimed = observed.control.clone();
-    claimed
-        .claim_for_source_generation(
-            api.id.to_string(),
-            Uuid::new_v4(),
-            now_unix_nanos,
-            TRANSITION_RECOVERY_CONTROL_LEASE_NANOS,
-            source.generation,
-        )
-        .map_err(|err| Error::other(err.to_string()))?;
-    save_recovery_control_if_current(api.clone(), &observed, &claimed).await?;
-    observed = load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?;
-    if observed.control != claimed {
-        return Err(Error::PreconditionFailed);
-    }
-    if !source_matches {
-        let mut corrupt = observed.control.clone();
-        corrupt
-            .finish_attempt(IlmRecoveryClassification::Corrupt, source_error)
-            .map_err(|err| Error::other(err.to_string()))?;
-        save_recovery_control_if_current(api, &observed, &corrupt).await?;
-        return Ok(None);
-    }
-    Ok(Some(observed))
-}
-
-async fn persist_transition_recovery_source_failure(
-    api: Arc<ECStore>,
-    observed: ObservedIlmRecoveryControl,
-    now_unix_nanos: i64,
-) -> EcstoreResult<()> {
-    let mut claimed = observed.control.clone();
-    claimed
-        .claim(
-            api.id.to_string(),
-            Uuid::new_v4(),
-            now_unix_nanos,
-            TRANSITION_RECOVERY_CONTROL_LEASE_NANOS,
-        )
-        .map_err(|err| Error::other(err.to_string()))?;
-    save_recovery_control_if_current(api.clone(), &observed, &claimed).await?;
-    let claimed = load_recovery_control(
-        api.clone(),
-        IlmRecoveryProtocol::TransitionTransaction,
-        &claimed
-            .identity
-            .source_operation_digest()
-            .map_err(|err| Error::other(err.to_string()))?,
-    )
-    .await?;
-    let mut failed = claimed.control.clone();
-    failed
-        .record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceUnavailable)
-        .map_err(|err| Error::other(err.to_string()))?;
-    save_recovery_control_if_current(api, &claimed, &failed).await
-}
-
-async fn refresh_transition_recovery_control_source(
-    api: Arc<ECStore>,
-    mut observed: ObservedIlmRecoveryControl,
-    record_name: &str,
-    transaction_id: Uuid,
-) -> EcstoreResult<(ObservedIlmRecoveryControl, Option<TransitionTransaction>)> {
-    let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
-        Ok(transaction) => transaction,
-        Err(Error::ConfigNotFound) => return Ok((observed, None)),
-        Err(err) => return Err(err),
-    };
-    let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?;
-    let exact_source = source.is_consistent()
-        && source
-            .canonical_data
-            .as_deref()
-            .is_some_and(|data| TransitionTransaction::decode(transaction_id, data).is_ok_and(|decoded| decoded == transaction));
-    if !exact_source {
-        return Err(Error::PreconditionFailed);
-    }
-    if observed.control.observed_source_generation != source.generation {
-        let mut refreshed = observed.control.clone();
-        refreshed
-            .refresh_owned_source_generation(source.generation)
-            .map_err(|err| Error::other(err.to_string()))?;
-        save_recovery_control_if_current(api.clone(), &observed, &refreshed).await?;
-        observed = load_recovery_control(
-            api,
-            IlmRecoveryProtocol::TransitionTransaction,
-            &refreshed
-                .identity
-                .source_operation_digest()
-                .map_err(|err| Error::other(err.to_string()))?,
-        )
-        .await?;
-        if observed.control != refreshed {
-            return Err(Error::PreconditionFailed);
-        }
-    }
-    Ok((observed, Some(transaction)))
-}
-
-async fn persist_transition_recovery_result(
-    api: Arc<ECStore>,
-    observed: ObservedIlmRecoveryControl,
-    recovery: &EcstoreResult<TransitionTransactionRecoveryOutcome>,
-    now_unix_nanos: i64,
-) -> EcstoreResult<()> {
-    let mut next = observed.control.clone();
-    match recovery {
-        Ok(
-            TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted,
-        ) => next
-            .finish_attempt(IlmRecoveryClassification::Terminal, IlmRecoveryErrorCode::None)
-            .map_err(|err| Error::other(err.to_string()))?,
-        Ok(TransitionTransactionRecoveryOutcome::Retained) => next
-            .record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceGenerationChanged)
-            .map_err(|err| Error::other(err.to_string()))?,
-        Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(code)) => next
-            .finish_attempt(IlmRecoveryClassification::RetainedAmbiguous, *code)
-            .map_err(|err| Error::other(err.to_string()))?,
-        Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(code)) => next
-            .finish_attempt(IlmRecoveryClassification::OperatorRequired, *code)
-            .map_err(|err| Error::other(err.to_string()))?,
-        Err(err) => next
-            .record_retryable_failure(now_unix_nanos, transition_recovery_error_code(err))
-            .map_err(|err| Error::other(err.to_string()))?,
-    }
-    save_recovery_control_if_current(api, &observed, &next).await
-}
-
-fn transition_recovery_error_code(err: &Error) -> IlmRecoveryErrorCode {
-    match err {
-        Error::PreconditionFailed => IlmRecoveryErrorCode::CasConflict,
-        Error::ConfigNotFound
-        | Error::FileNotFound
-        | Error::FileVersionNotFound
-        | Error::ObjectNotFound(_, _)
-        | Error::VersionNotFound(_, _, _)
-        | Error::BucketNotFound(_) => IlmRecoveryErrorCode::SourceUnavailable,
-        Error::SlowDown => IlmRecoveryErrorCode::BackendThrottled,
-        _ => IlmRecoveryErrorCode::Unknown,
+        TransitionTransactionState::UploadStarted => Ok(TransitionTransactionRecoveryOutcome::Retained),
     }
 }
 
@@ -1600,7 +1134,10 @@ async fn recover_cleanup_pending(
     transaction: &TransitionTransaction,
 ) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
     match local_commit_matches_transaction(api.clone(), transaction).await {
-        Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
+        Ok(true) => {
+            delete_transition_transaction_record(api, transaction).await?;
+            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
+        }
         Ok(false) => delete_unreferenced_transition_candidate(api, transaction).await,
         Err(err) if transition_source_is_missing(&err) => delete_unreferenced_transition_candidate(api, transaction).await,
         Err(err) => Err(err),
@@ -1620,6 +1157,7 @@ async fn delete_unreferenced_transition_candidate(
         return Ok(TransitionTransactionRecoveryOutcome::Retained);
     }
     delete_transition_remote_candidate(api.clone(), &current).await?;
+    delete_transition_transaction_record(api, &current).await?;
     Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
 }
 
@@ -1640,26 +1178,24 @@ async fn recover_unknown_upload_outcome(
         .await
         .map_err(Error::other)?
     {
-        TransitionCandidateProbe::Missing => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
+        TransitionCandidateProbe::Missing => {
+            delete_transition_transaction_record(api, transaction).await?;
+            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
+        }
         TransitionCandidateProbe::UnversionedPresent => {
             cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
         }
         TransitionCandidateProbe::VersionedPresent(version_id)
             if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) =>
         {
-            Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
-                IlmRecoveryErrorCode::RemoteVersionUnknown,
-            ))
+            Ok(TransitionTransactionRecoveryOutcome::Retained)
         }
         TransitionCandidateProbe::VersionedPresent(version_id) => {
             cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
         }
-        TransitionCandidateProbe::Ambiguous => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
-            IlmRecoveryErrorCode::RemoteProbeAmbiguous,
-        )),
-        TransitionCandidateProbe::Unsupported => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
-            IlmRecoveryErrorCode::RemoteProbeUnsupported,
-        )),
+        TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => {
+            Ok(TransitionTransactionRecoveryOutcome::Retained)
+        }
     }
 }
 
@@ -1787,11 +1323,6 @@ async fn recover_transition_transaction_records_with_now(
             false,
         )
         .await?;
-    if list.is_truncated && list.next_continuation_token.is_none() {
-        return Err(Error::other(
-            "transition transaction recovery returned a truncated page without a continuation marker",
-        ));
-    }
 
     let mut stats = TransitionTransactionRecoveryStats {
         scanned: 0,
@@ -1850,11 +1381,7 @@ async fn recover_transition_transaction_records_with_now(
             ) => {
                 stats.recovered += 1;
             }
-            Ok(
-                TransitionTransactionRecoveryOutcome::Retained
-                | TransitionTransactionRecoveryOutcome::RetainedAmbiguous(_)
-                | TransitionTransactionRecoveryOutcome::OperatorRequired(_),
-            ) => {
+            Ok(TransitionTransactionRecoveryOutcome::Retained) => {
                 stats.retained += 1;
                 debug!(
                     event = EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY,
@@ -1982,74 +1509,11 @@ fn state_requires_known_remote_version(state: TransitionTransactionState) -> boo
 #[cfg(test)]
 mod tests {
     use std::collections::HashMap;
-    use std::sync::atomic::{AtomicBool, Ordering};
 
     use super::*;
 
     const BACKEND_FINGERPRINT: [u8; 32] = [7; 32];
 
-    struct RecoveryAttemptDropGuard(Arc<AtomicBool>);
-
-    impl Drop for RecoveryAttemptDropGuard {
-        fn drop(&mut self) {
-            self.0.store(true, Ordering::SeqCst);
-        }
-    }
-
-    async fn pending_recovery_attempt(started: Arc<tokio::sync::Notify>, dropped: Arc<AtomicBool>) -> EcstoreResult<()> {
-        let _drop_guard = RecoveryAttemptDropGuard(dropped);
-        started.notify_one();
-        std::future::pending().await
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn transition_recovery_timeout_and_cancellation_drop_inflight_attempts() {
-        let timeout_started = Arc::new(tokio::sync::Notify::new());
-        let timeout_dropped = Arc::new(AtomicBool::new(false));
-        let timeout_task = tokio::spawn({
-            let started = Arc::clone(&timeout_started);
-            let dropped = Arc::clone(&timeout_dropped);
-            async move {
-                await_transition_transaction_recovery(
-                    &CancellationToken::new(),
-                    TRANSITION_TRANSACTION_RECOVERY_TIMEOUT,
-                    pending_recovery_attempt(started, dropped),
-                )
-                .await
-            }
-        });
-        timeout_started.notified().await;
-        tokio::time::advance(TRANSITION_TRANSACTION_RECOVERY_TIMEOUT).await;
-        let timed_out = timeout_task.await.expect("timeout wrapper task should join");
-        assert!(matches!(timed_out, Some(Err(_))), "outer timeout should fail the recovery pass");
-        assert!(timeout_dropped.load(Ordering::SeqCst), "outer timeout must drop its in-flight attempt");
-
-        let cancel_token = CancellationToken::new();
-        let cancel_started = Arc::new(tokio::sync::Notify::new());
-        let cancel_dropped = Arc::new(AtomicBool::new(false));
-        let cancel_task = tokio::spawn({
-            let cancel_token = cancel_token.clone();
-            let started = Arc::clone(&cancel_started);
-            let dropped = Arc::clone(&cancel_dropped);
-            async move {
-                await_transition_transaction_recovery(
-                    &cancel_token,
-                    TRANSITION_TRANSACTION_RECOVERY_TIMEOUT,
-                    pending_recovery_attempt(started, dropped),
-                )
-                .await
-            }
-        });
-        cancel_started.notified().await;
-        cancel_token.cancel();
-        let cancelled = cancel_task.await.expect("cancellation wrapper task should join");
-        assert!(cancelled.is_none(), "outer cancellation should stop the recovery loop");
-        assert!(
-            cancel_dropped.load(Ordering::SeqCst),
-            "outer cancellation must drop its in-flight attempt"
-        );
-    }
-
     #[derive(Default)]
     struct MemoryTransactionStore {
         records: HashMap<Uuid, Vec<u8>>,
@@ -2504,19 +1968,5 @@ mod tests {
             transition_transaction_record_object_name(Uuid::nil()),
             Err(TransitionTransactionError::Corrupt("transaction_id is nil"))
         ));
-        assert_eq!(
-            transition_transaction_id_from_record_object_name(&object).expect("canonical record path should parse"),
-            transaction_id
-        );
-        for malformed in [
-            object.to_ascii_uppercase(),
-            object.replace("/aa/aa/", "/ff/aa/"),
-            object.replace("/aa/aa/", "/aa/aa/extra/"),
-        ] {
-            assert!(matches!(
-                transition_transaction_id_from_record_object_name(&malformed),
-                Err(TransitionTransactionError::Corrupt(_))
-            ));
-        }
     }
 }

crates/ecstore/src/store/init.rs

@@ -825,11 +825,6 @@ mod tests {
                 manual_transition_scope_record_object_name, manual_transition_task_object_name,
                 manual_transition_worker_result_object_name, manual_transition_worker_result_task_key,
             },
-            recovery_control::{
-                IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode,
-                IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, load_recovery_control, observe_recovery_source,
-                save_recovery_control_if_absent,
-            },
             tier_delete_journal::{
                 DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX,
                 TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard,
@@ -849,13 +844,12 @@ mod tests {
             },
             transition_transaction::{
                 TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
-                TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier,
-                TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
-                TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator,
-                finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
-                load_transition_transaction_record, recover_transition_transaction_records,
-                recover_transition_transaction_records_at, save_transition_transaction_record,
-                save_transition_transaction_record_if_current, transition_recovery_control_id,
+                TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRemoteVersion, TransitionSourceIdentity,
+                TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
+                delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
+                inspect_transition_transaction_for_operator, load_transition_transaction_record,
+                recover_transition_transaction_records, recover_transition_transaction_records_at,
+                save_transition_transaction_record, save_transition_transaction_record_if_current,
                 transition_transaction_record_object_name,
             },
             validate_durable_ilm_record,
@@ -19245,105 +19239,6 @@ mod tests {
         assert!(!Arc::ptr_eq(&ctx_a, &ctx_b), "the regression requires two distinct instance contexts");
     }
 
-    #[cfg(feature = "test-util")]
-    #[tokio::test]
-    #[serial_test::serial(storage_class_env)]
-    async fn transition_transaction_recovery_expires_abandoned_attempt_at_budget_bound() {
-        let temp_dir = tempfile::tempdir().expect("create temp store dir");
-        let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
-            temp_dir.path(),
-            "transition-transaction-expired-attempt-budget",
-            &[4],
-        ))
-        .await;
-        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-
-        let transaction = TransitionTransaction::new(TransitionTransactionInit {
-            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
-            transaction_id: uuid::Uuid::new_v4(),
-            owner_epoch: uuid::Uuid::new_v4(),
-            write_id: uuid::Uuid::new_v4(),
-            source: TransitionSourceIdentity {
-                bucket: "source-bucket".to_string(),
-                object: "source-object".to_string(),
-                version_id: Some(uuid::Uuid::new_v4()),
-                data_dir: uuid::Uuid::new_v4(),
-                mod_time_unix_nanos: 1_770_000_000_000_000_000,
-                size: 42,
-                etag: "source-etag".to_string(),
-                version_mode: TransitionSourceVersionMode::Versioned,
-            },
-            tier_name: "UNUSEDABANDONEDTIER".to_string(),
-            backend_fingerprint: [7; 32],
-            not_after_unix_nanos: 1,
-        })
-        .expect("transaction should build");
-        save_transition_transaction_record(store.clone(), &transaction)
-            .await
-            .expect("transaction record should persist");
-
-        let record_name =
-            transition_transaction_record_object_name(transaction.transaction_id).expect("transaction record name should derive");
-        let source = observe_recovery_source(
-            store.clone(),
-            &record_name,
-            crate::bucket::lifecycle::transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
-        )
-        .await
-        .expect("transaction source generation should be observable");
-        let mut control = IlmRecoveryControl::new(
-            IlmRecoveryControlIdentity {
-                protocol: IlmRecoveryProtocol::TransitionTransaction,
-                canonical_source_path: record_name,
-                stable_operation_identity: transaction.transaction_id.to_string(),
-                record_class: "transition_transaction_v1".to_string(),
-            },
-            source.generation,
-            IlmRecoveryClassification::Retrying,
-            2_000_000_000,
-            IlmRecoveryErrorCode::None,
-        )
-        .expect("recovery control should build");
-        let mut now = 3_000_000_000;
-        for _ in 1..MAX_RECOVERY_ATTEMPTS {
-            now = now.max(control.next_attempt_at_unix_nanos.unwrap_or(now));
-            control
-                .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
-                .expect("abandoned attempt should claim");
-            control
-                .record_expired_attempt(now + 1)
-                .expect("expired attempt should consume retry budget");
-            now += 2;
-        }
-        now = now.max(control.next_attempt_at_unix_nanos.expect("last retry should have a backoff"));
-        control
-            .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1)
-            .expect("final abandoned attempt should claim");
-        save_recovery_control_if_absent(store.clone(), &control)
-            .await
-            .expect("claimed recovery control should persist");
-
-        let stats = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(now + 1))
-            .await
-            .expect("recovery should account for the expired attempt");
-        assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
-
-        let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
-        let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
-            .await
-            .expect("expired recovery control should remain inspectable");
-        assert_eq!(persisted.control.classification, IlmRecoveryClassification::OperatorRequired);
-        assert_eq!(persisted.control.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS));
-        assert_eq!(persisted.control.consecutive_failure_count, MAX_RECOVERY_ATTEMPTS);
-        assert_eq!(persisted.control.last_error_code, IlmRecoveryErrorCode::AttemptLeaseExpired);
-        assert!(persisted.control.owner.is_none());
-        assert_eq!(
-            transition_transaction_record_count(store).await,
-            1,
-            "budget exhaustion must retain the source record"
-        );
-    }
-
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
@@ -19456,7 +19351,6 @@ mod tests {
             ),
         ];
         let mut expected_removes = Vec::new();
-        let mut recovery_control_ids = Vec::new();
         for (case, put_version, remote_version, source_mode) in cases {
             let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
                 deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
@@ -19494,8 +19388,6 @@ mod tests {
             save_transition_transaction_record(store.clone(), &transaction)
                 .await
                 .expect("transaction record should persist");
-            recovery_control_ids
-                .push(transition_recovery_control_id(&transaction).expect("transition recovery control id should derive"));
             expected_removes.push((transaction.remote_object, put_version));
         }
 
@@ -19511,14 +19403,6 @@ mod tests {
         assert_eq!(actual_removes, expected_removes, "recovery must preserve each remote version shape");
         assert_eq!(backend.exact_remove_count(), 2);
         assert_eq!(backend.object_count().await, 0);
-        for control_id in recovery_control_ids {
-            let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
-                .await
-                .expect("completed recovery control should remain inspectable");
-            assert_eq!(control.control.classification, IlmRecoveryClassification::Terminal);
-            assert_eq!(control.control.attempt_count, 1);
-            assert!(control.control.owner.is_none());
-        }
 
         let replay = recover_transition_transaction_records(store, 100, None)
             .await
@@ -19531,94 +19415,6 @@ mod tests {
         );
     }
 
-    #[cfg(feature = "test-util")]
-    #[tokio::test]
-    #[serial_test::serial(storage_class_env)]
-    async fn transition_transaction_recovery_resumes_source_cleanup_after_terminal_crash() {
-        let temp_dir = tempfile::tempdir().expect("create temp store dir");
-        let (ctx, store, _shutdown) =
-            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-terminal-crash", &[4]))
-                .await;
-        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
-
-        let tier_name = "TXTERMINALCRASH";
-        let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
-        let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
-            .await
-            .expect("tier lease should resolve")
-            .backend_identity();
-        let remote_version = uuid::Uuid::new_v4().to_string();
-        let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
-            deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
-            transaction_id: uuid::Uuid::new_v4(),
-            owner_epoch: uuid::Uuid::new_v4(),
-            write_id: uuid::Uuid::new_v4(),
-            source: TransitionSourceIdentity {
-                bucket: "source-bucket".to_string(),
-                object: "source-object".to_string(),
-                version_id: Some(uuid::Uuid::new_v4()),
-                data_dir: uuid::Uuid::new_v4(),
-                mod_time_unix_nanos: 1_770_000_000_000_000_000,
-                size: 42,
-                etag: "source-etag".to_string(),
-                version_mode: TransitionSourceVersionMode::Versioned,
-            },
-            tier_name: tier_name.to_string(),
-            backend_fingerprint: backend_identity,
-            not_after_unix_nanos: 1,
-        })
-        .expect("transaction should build");
-        transaction
-            .advance(
-                transaction.fence(),
-                TransitionTransactionState::Uploaded,
-                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
-            )
-            .expect("transaction should enter uploaded state");
-        backend.set_put_remote_version(Some(remote_version)).await;
-        let candidate = bytes::Bytes::from_static(b"terminal crash candidate");
-        backend
-            .put(
-                &transaction.remote_object,
-                ReaderImpl::Body(candidate.clone()),
-                i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
-            )
-            .await
-            .expect("mock backend should accept candidate");
-        save_transition_transaction_record(store.clone(), &transaction)
-            .await
-            .expect("transaction record should persist");
-        let control_id = transition_recovery_control_id(&transaction).expect("control id should derive");
-
-        let barrier = TransitionRecoveryTerminalBarrier::install(transaction.transaction_id);
-        let recovery_store = store.clone();
-        let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await });
-        barrier.wait_until_paused().await;
-        let terminal = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id)
-            .await
-            .expect("terminal control should persist before source cleanup");
-        assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
-        assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
-        assert_eq!(backend.object_count().await, 0);
-        assert_eq!(backend.exact_remove_count(), 1);
-
-        recovery.abort();
-        assert!(
-            recovery
-                .await
-                .expect_err("recovery should be cancelled at the crash boundary")
-                .is_cancelled()
-        );
-        drop(barrier);
-
-        let replay = recover_transition_transaction_records(store.clone(), 100, None)
-            .await
-            .expect("terminal control should resume source cleanup without another remote delete");
-        assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 1, 0, 0));
-        assert_eq!(transition_transaction_record_count(store).await, 0);
-        assert_eq!(backend.exact_remove_count(), 1, "terminal replay must not repeat the remote delete");
-    }
-
     #[cfg(feature = "test-util")]
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
@@ -19676,8 +19472,6 @@ mod tests {
         save_transition_transaction_record(store.clone(), &uploaded)
             .await
             .expect("transaction record should persist");
-        let recovery_control_id =
-            transition_recovery_control_id(&uploaded).expect("transition recovery control id should derive");
 
         let barrier = TransitionRecoveryClaimBarrier::install(uploaded.transaction_id);
         let recovery_store = store.clone();
@@ -19699,17 +19493,11 @@ mod tests {
             .expect("recovery should treat the lost CAS as a retained transaction");
         assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0));
         assert_eq!(
-            load_transition_transaction_record(store.clone(), uploaded.transaction_id)
+            load_transition_transaction_record(store, uploaded.transaction_id)
                 .await
                 .expect("newer transaction revision must remain"),
             active
         );
-        let control = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
-            .await
-            .expect("lost source CAS should retain a retryable recovery control");
-        assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
-        assert_eq!(control.control.consecutive_failure_count, 1);
-        assert_eq!(control.control.last_error_code, IlmRecoveryErrorCode::SourceGenerationChanged);
         assert_eq!(backend.object_count().await, 1, "a stale recovery must not delete the candidate");
         assert_eq!(backend.remove_count().await, 0);
     }
@@ -20011,15 +19799,27 @@ mod tests {
             not_after_unix_nanos: 1_780_000_000_000_000_000,
         })
         .expect("transaction should build");
-        transaction
+        let uploaded_fence = transaction
             .advance(
                 transaction.fence(),
                 TransitionTransactionState::Uploaded,
-                Some(TransitionRemoteVersion::versioned(remote_version.clone())),
+                Some(TransitionRemoteVersion::versioned(remote_version)),
             )
             .expect("transaction should enter uploaded state");
+        transaction
+            .mark_cleanup_pending(
+                uploaded_fence,
+                TransitionCleanupProof {
+                    transaction_id: transaction.transaction_id,
+                    write_id: transaction.write_id,
+                    remote_object: transaction.remote_object.clone(),
+                    remote_version: transaction.remote_version.clone(),
+                    backend_fingerprint: transaction.backend_fingerprint,
+                    decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
+                },
+            )
+            .expect("transaction should enter cleanup pending state");
         let candidate = bytes::Bytes::from_static(b"cleanup pending candidate retained after failure");
-        backend.set_put_remote_version(Some(remote_version)).await;
         backend
             .put(
                 &transaction.remote_object,
@@ -20031,8 +19831,6 @@ mod tests {
         save_transition_transaction_record(store.clone(), &transaction)
             .await
             .expect("transaction record should persist");
-        let recovery_control_id =
-            transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
 
         backend.set_remove_failure(true);
         let stats = recover_transition_transaction_records(store.clone(), 100, None)
@@ -20048,42 +19846,6 @@ mod tests {
         assert_eq!(backend.remove_versions().await, Vec::<(String, String)>::new());
         assert_eq!(backend.exact_remove_count(), 1);
         assert_eq!(backend.object_count().await, 1);
-        let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
-            .await
-            .expect("failed recovery control should persist");
-        assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying);
-        assert_eq!(control.control.attempt_count, 1);
-        assert_eq!(control.control.consecutive_failure_count, 1);
-        assert!(
-            control
-                .control
-                .next_attempt_at_unix_nanos
-                .is_some_and(|next| next > OffsetDateTime::now_utc().unix_timestamp_nanos() as i64)
-        );
-
-        backend.set_remove_failure(false);
-        let replay = recover_transition_transaction_records(store.clone(), 100, None)
-            .await
-            .expect("recovery before the persisted deadline should be skipped");
-        assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 0, 1, 0));
-        assert_eq!(backend.exact_remove_count(), 1, "persisted backoff must prevent an immediate retry");
-
-        let retry_at = control
-            .control
-            .next_attempt_at_unix_nanos
-            .expect("retry deadline should persist");
-        let retried = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(retry_at) + 1)
-            .await
-            .expect("recovery at the persisted deadline should retry the advanced source generation");
-        assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (1, 1, 0, 0));
-        assert_eq!(backend.exact_remove_count(), 2);
-        assert_eq!(backend.object_count().await, 0);
-        assert_eq!(transition_transaction_record_count(store.clone()).await, 0);
-        let terminal = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
-            .await
-            .expect("completed retry control should remain inspectable");
-        assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal);
-        assert_eq!(terminal.control.attempt_count, 2);
     }
 
     #[cfg(feature = "test-util")]
@@ -20384,10 +20146,6 @@ mod tests {
         local_commit_started
             .advance(local_commit_started.fence(), TransitionTransactionState::LocalCommitStarted, None)
             .expect("transaction should enter local commit state");
-        let upload_started_control_id =
-            transition_recovery_control_id(&upload_started).expect("upload-started control id should derive");
-        let local_commit_control_id =
-            transition_recovery_control_id(&local_commit_started).expect("local-commit control id should derive");
 
         backend.set_put_remote_version(Some(remote_version)).await;
         for transaction in [&upload_started, &local_commit_started] {
@@ -20418,19 +20176,6 @@ mod tests {
         assert_eq!(backend.object_count().await, 2, "recovery must not delete an unproven remote candidate");
         assert_eq!(backend.remove_count().await, 0);
         assert_eq!(backend.exact_remove_count(), 0);
-        let upload_started_control =
-            load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id)
-                .await
-                .expect("upload-started control should persist");
-        assert_eq!(
-            upload_started_control.control.classification,
-            IlmRecoveryClassification::RetainedAmbiguous
-        );
-        let local_commit_control =
-            load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id)
-                .await
-                .expect("local-commit control should persist");
-        assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired);
     }
 
     #[cfg(feature = "test-util")]
@@ -20781,12 +20526,6 @@ mod tests {
             "an unsupported provider probe must retain the unknown upload"
         );
         assert_eq!(transition_transaction_record_count(store.clone()).await, 1);
-        let recovery_control_id =
-            transition_recovery_control_id(&transaction).expect("transition recovery control id should derive");
-        let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
-            .await
-            .expect("unsupported probe control should persist");
-        assert_eq!(control.control.classification, IlmRecoveryClassification::RetainedAmbiguous);
         assert!(
             backend.contains(&transaction.remote_object).await,
             "unsupported recovery must not delete the candidate"

crates/scanner/tests/lifecycle_integration_test.rs

@@ -1020,7 +1020,6 @@ mod serial_tests {
         }
 
         let (_disk_paths, ecstore) = setup_isolated_test_env(false).await;
-        let expired_recovery_time = i128::from(i64::MAX / 2);
 
         for case in [
             CleanupCase::Persisted,
@@ -1118,7 +1117,7 @@ mod serial_tests {
                         .await
                         .expect("active unknown ownership must remain fenced after the transaction store was offline");
                     assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 1, 0));
-                    let recovered = recover_transition_transaction_records_at(ecstore.clone(), 100, None, expired_recovery_time)
+                    let recovered = recover_transition_transaction_records_at(ecstore.clone(), 100, None, i128::MAX)
                         .await
                         .expect("expired unknown ownership may use the provider's missing proof");
                     assert_eq!(
@@ -1167,7 +1166,7 @@ mod serial_tests {
                     assert_eq!(retained.recovered, 0);
                     assert_eq!(retained.retained + retained.failed, 1);
                     backend.set_remove_failure(false);
-                    let recovered = recover_transition_transaction_records_at(ecstore.clone(), 100, None, expired_recovery_time)
+                    let recovered = recover_transition_transaction_records_at(ecstore.clone(), 100, None, i128::MAX)
                         .await
                         .expect("expired recovery should delete the candidate after the backend becomes available");
                     assert_eq!(

docs/operations/scanner-benchmark-runbook.md

@@ -34,6 +34,142 @@ The `scanner` and `heal` subsystems are served by `GetConfigKVHandler` (`rustfs/
 
 ## Test Matrix
 
+### Formal Scanner/Heal ABBA
+
+The `--abba` mode runs five independent scenario cells: `cold-hot`, `fresh-hot`,
+`multi-hot-new`, `running-heal`, and `mrf-replay`. Each scenario runs at least
+three A1/B1/B2/A2 groups for both baseline/candidate with background work on,
+and candidate-only background off/on. A measured leg lasts at least 900
+seconds; the minimum matrix contains 120 legs (30 hours before setup/oracles).
+The existing `performance-ab.yml` supplies the pattern for immutable build
+provenance and failure propagation, but its short Warp workload is not this
+scanner gate. No scheduled workflow starts this matrix automatically.
+
+```bash
+scripts/run_scanner_validation_harness.sh --abba \
+  --manifest scanner-abba.json --adapter /path/to/isolated-deployment-adapter \
+  --out-dir /path/to/new-artifacts --data-root /path/to/new-test-data
+```
+
+Both roots must be new and non-overlapping. Every leg receives a unique data
+directory. The runner checks disk capacity before each leg, never removes data,
+and stops the adapter after success or failure. Retain raw artifacts and inspect
+task ownership before removing any test data. The operator must reserve the
+target machines and map the assigned directory to separate data paths on every
+node; the runner cannot prove remote isolation from local path names.
+
+The manifest has the following JSON contract (all fields are required):
+
+| Field | Value |
+|---|---|
+| `schema`, `evidence` | `1`, and `measured` or `synthetic`. |
+| `rounds`, `duration_seconds`, `min_free_bytes` | 3..10 groups, 900..86400 seconds for measured runs, and the independently estimated free-space reservation in bytes. Synthetic runs may use 1 second. |
+| `baseline`, `candidate` | Each contains executable `binary`, full 40-character `revision`, and verified `sha256`. The runner rehashes binaries before every leg. |
+| `fixed` | `config_sha256`, `dataset_sha256`, `release_flags`, `durability`, `disk_type`, `cache_state`, `load_command`, `resource_isolation`, `topology` (`EC8+4`), and positive `offered_load_ops`. Hashes use 64 lowercase hexadecimal characters. |
+| `oracles` | A map with all five scenario names. Each value contains positive integer `objects`, `versions`, `bytes`, and `sha256` of the independently prepared canonical object/version/content manifest. |
+| `expected_healed_objects` | A map with all five scenario names and independently seeded repair counts. Running-heal and MRF-replay require a positive count. |
+
+Record exact build flags and effective durability settings, not just defaults.
+Use deterministic workload seeds so every isolated leg has the same expected
+object/version/content result. Fix the foreground arrival rate (offered load),
+cache preparation procedure, configuration, and hardware across every leg.
+Do not include credentials in the manifest, adapter output, or saved commands;
+the collector reads `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` from its environment.
+
+#### Deployment Adapter Contract
+
+The runner invokes an executable as `adapter ACTION request.json response.json`
+with no shell evaluation. Actions are separate processes: `prepare`, `measure`,
+`oracle`, and `stop`. Every action must return zero and write a JSON object of
+at most 1 MiB. Logs are kept separately and require an operator-managed disk
+quota. Missing output, timeout, nonzero exit, unknown/missing metrics, zero
+samples, and request errors fail the run. Adapters must terminate their own
+children on failure and `stop` must be idempotent even after partial preparation.
+
+The request contains the fixed manifest fields, selected build, scenario, round,
+leg, comparison (`build` or `background`), background mode (`on` or `off`),
+duration, unique `data_dir`, expected object oracle, and expected repair count.
+Adapter responsibilities:
+
+1. `prepare` deploys the selected binary into an authorized isolated topology,
+   checks actual binary/config/durability, initializes deterministic scenario
+   data and the requested cache state, and returns `{"ready": true}`. For measured
+   runs it also returns `collector` with exactly `alias`, `endpoint`, and
+   comma-separated `metrics_endpoints`; the runner starts the existing scanner
+   collector at 60-second cadence while `measure` runs.
+2. `measure` maintains the fixed offered load for the entire requested duration.
+   `cold-hot` retains cold buckets while mutating a hot bucket; `fresh-hot`
+   creates a bucket after scanner startup; `multi-hot-new` combines several hot
+   buckets with a newly created bucket; `running-heal` applies foreground load
+   during active repair; `mrf-replay` replays independently seeded durable repair
+   work. Capture same-window status for bucket-freshness issue #7108. Actual
+   fault injection and dataset generation belong to the reviewed adapter.
+3. `oracle` independently enumerates all objects and versions, reads and checks
+   their complete bytes, and verifies repairs. Return `complete: true`, integer
+   `errors: 0`, and `actual` matching the manifest's expected oracle. Never copy
+   expected values into a measured oracle or infer completion from empty queues.
+4. `stop` stops task-owned workload/server processes and returns `stopped: true`.
+   Preserve data and artifacts for diagnosis. An adapter may restore previous
+   settings but must not delete arbitrary paths or stop unrelated deployments.
+
+The `measure` response echoes the observed `evidence`, `fixed`, `build`,
+`data_dir`, and `background`, plus `sample_count` (1..3600), `elapsed_seconds`,
+and `metrics`. All metrics must be finite nonnegative numbers: `p99_ms`,
+`throughput_ops`, `rss_bytes`, `cpu_seconds`, `iops`, `rpc_count`,
+`cache_clone_bytes`, `encode_bytes`, `save_bytes`, `oldest_age_seconds`,
+`walk_objects`, `cold_walk_objects`, `healed_objects`, `errors`, and `requests`.
+Requests, throughput, and p99 must be positive; errors must be zero. Repair
+counts must match the manifest when background work is on. Keep underlying
+request samples, counter reset checks, profiler captures, and per-node telemetry
+in the cell artifact directory; aggregate values alone do not establish their
+measurement provenance. Missing production instrumentation is a pending gate,
+not permission to report a fabricated zero.
+
+For P2, `measure.convergence` contains booleans `writes_stopped`,
+`last_mutation_observed`, `first_complete_publication`; numeric
+`last_mutation_time`, `last_mutation_observed_time`, `writes_stopped_time`, `window_start`, `window_end`,
+`budget_available_seconds`, `walk_objects`, and `full_walk_objects`. Times use
+one monotonic clock. The window starts after writes stop and the final mutation
+is observed, and ends at the first complete publication. The reference is an
+independent full walk of the same static namespace. Record available budget
+seconds to interpret elapsed time. During continuing writes, omit this proof
+and report useful-work ratio and justified invalidation/re-scan work separately;
+the runner reports P2 pending and does not impose a fixed cumulative walk bound.
+
+The nightly heal workflow clones **`rustfs/auto-testing`** separately and invokes
+`auto-testing/rustfs_heal_test.sh`; that script is not a local `scripts/test`
+entry point. If an adapter uses it, record and verify the external checkout's
+owner and full commit before use. The current workflow clones the default branch,
+so its contents must not be attributed to a RustFS source SHA.
+
+#### Evidence Gates
+
+`report.json` records each group's verdict and the raw responses remain in their
+cell directories. Candidate/build p99 regression must be at most 5% and
+throughput loss at most 3%; candidate background on/off limits are 10% and 5%.
+P1 requires cold-hot walk reduction of at least the baseline cold-walk share
+times 80%, rather than a fixed 80% reduction for every workload. P2 requires
+candidate post-stop work at most 1.2 times the independent full-walk reference.
+Missing candidate convergence proof yields `inconclusive`. A2/A1 or B2/B1 p99
+or throughput drift above 5% also yields `inconclusive`, with exit code 3.
+Correctness errors and non-noisy performance regressions exit 1. Every group
+must pass; a favorable median cannot hide a failing group.
+
+Synthetic success is explicitly `synthetic_validated`, with `performance:
+pending`. It validates orchestration and gate logic only. It proves no runtime,
+distributed, crash, mixed-version, or performance behavior and cannot close the
+performance acceptance gate. Run the fake-adapter self-tests with:
+
+```bash
+scripts/test_scanner_validation_harness.sh
+```
+
+They cover the complete 120-cell schedule, data isolation, missing builds and
+oracles, zero samples/requests, swallowed request errors, offered-load drift,
+incomplete repairs, missing metrics, noise, and P1/P2/p99 regressions. A real
+deployment adapter and actual ABBA artifacts remain required before any measured
+performance or release claim.
+
 Collect at least two runs on the same RustFS commit and the same workload. Keep hardware, commit, object count, object size, bucket count, scanner-enabled state, and foreground workload constant between runs.
 
 | Run | Purpose | Example scanner settings |

rustfs/src/admin/handlers/ilm_transition.rs

@@ -18,20 +18,18 @@ use crate::admin::runtime_sources::object_store_from_extensions;
 use crate::admin::storage_api::bucket::is_reserved_or_invalid_bucket;
 use crate::admin::storage_api::error::StorageError;
 use crate::admin::storage_api::lifecycle::{
-    IlmRecoveryClassification, IlmRecoveryProtocol, ManualTransitionCancelCheck, ManualTransitionJobRecord,
-    ManualTransitionJobState, ManualTransitionProgressSink, ManualTransitionQueueSnapshot, ManualTransitionRunOptions,
-    ManualTransitionRunReport, ManualTransitionScopeAdmission, ManualTransitionScopeAdmissionClaim,
-    TransitionOperatorDeleteResult, TransitionOperatorError, claim_manual_transition_scope_admission,
-    delete_manual_transition_scope_admission_if_current, delete_transition_candidate_for_operator,
-    enqueue_transition_for_existing_objects_scoped, finalize_missing_transition_transaction_for_operator,
-    inspect_recovery_control, inspect_transition_transaction_for_operator, list_recovery_controls,
+    ManualTransitionCancelCheck, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink,
+    ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission,
+    ManualTransitionScopeAdmissionClaim, TransitionOperatorDeleteResult, TransitionOperatorError,
+    claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
+    delete_transition_candidate_for_operator, enqueue_transition_for_existing_objects_scoped,
+    finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
     load_manual_transition_job_record, load_manual_transition_scope_admission, manual_transition_job_lease_expired,
     manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired,
     persist_manual_transition_job_progress_if_owned, renew_manual_transition_job_lease_if_owned,
     request_manual_transition_job_cancel, save_manual_transition_job_record, update_manual_transition_job_record,
 };
 use crate::admin::storage_api::runtime::ECStore;
-use crate::admin::storage_api::s3::{S3ErrorCode as AdminS3ErrorCode, error as admin_s3_error};
 use crate::admin::utils::json_response;
 use crate::server::{ADMIN_PREFIX, RemoteAddr};
 use http::HeaderMap;
@@ -232,48 +230,9 @@ pub fn register_ilm_transition_route(r: &mut S3Router<AdminOperation>) -> std::i
         format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(),
         AdminOperation(&TransitionReconcileApplyHandler {}),
     )?;
-    r.insert(
-        Method::GET,
-        format!("{ADMIN_PREFIX}/v3/ilm/recovery/records").as_str(),
-        AdminOperation(&IlmRecoveryControlListHandler {}),
-    )?;
-    r.insert(
-        Method::GET,
-        format!("{ADMIN_PREFIX}/v3/ilm/recovery/records/{{control_id}}").as_str(),
-        AdminOperation(&IlmRecoveryControlInspectHandler {}),
-    )?;
     Ok(())
 }
 
-#[derive(Debug, Deserialize)]
-#[serde(deny_unknown_fields)]
-struct IlmRecoveryControlListQuery {
-    protocol: IlmRecoveryProtocol,
-    #[serde(default)]
-    classification: Option<IlmRecoveryClassification>,
-    #[serde(default = "default_recovery_control_list_limit")]
-    limit: usize,
-    #[serde(default)]
-    marker: Option<String>,
-}
-
-const fn default_recovery_control_list_limit() -> usize {
-    100
-}
-
-fn parse_recovery_control_list_query(query: Option<&str>) -> S3Result<IlmRecoveryControlListQuery> {
-    let query = query.ok_or_else(|| admin_s3_error(AdminS3ErrorCode::InvalidRequest, "protocol is required"))?;
-    let parsed: IlmRecoveryControlListQuery = serde_urlencoded::from_bytes(query.as_bytes())
-        .map_err(|_| admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid ILM recovery control query"))?;
-    if !(1..=1_000).contains(&parsed.limit) {
-        return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "limit must be between 1 and 1000"));
-    }
-    if parsed.marker.as_ref().is_some_and(|marker| marker.is_empty()) {
-        return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "marker must not be empty"));
-    }
-    Ok(parsed)
-}
-
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum ManualTransitionRunMode {
     EnqueueOnly,
@@ -464,26 +423,6 @@ fn transition_transaction_id_from_params(params: &Params<'_, '_>) -> S3Result<Uu
         .map_err(|_| s3_error!(InvalidArgument, "invalid transition transaction id"))
 }
 
-fn recovery_control_id_from_params(params: &Params<'_, '_>) -> S3Result<String> {
-    let control_id = params.get("control_id").unwrap_or("");
-    if control_id.len() != 64
-        || !control_id
-            .bytes()
-            .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
-    {
-        return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid ILM recovery control id"));
-    }
-    Ok(control_id.to_string())
-}
-
-fn map_recovery_control_error(err: StorageError) -> S3Error {
-    if err == StorageError::ConfigNotFound {
-        admin_s3_error(AdminS3ErrorCode::NoSuchKey, "ILM recovery control not found")
-    } else {
-        admin_s3_error(AdminS3ErrorCode::InternalError, "ILM recovery control request failed")
-    }
-}
-
 fn map_transition_operator_error(err: TransitionOperatorError) -> S3Error {
     match err {
         TransitionOperatorError::NotFound => s3_error!(NoSuchKey, "transition transaction not found"),
@@ -1092,40 +1031,6 @@ impl Operation for TransitionReconcileInspectHandler {
     }
 }
 
-pub struct IlmRecoveryControlListHandler {}
-
-#[async_trait::async_trait]
-impl Operation for IlmRecoveryControlListHandler {
-    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?;
-        let query = parse_recovery_control_list_query(req.uri.query())?;
-        let Some(store) = object_store_from_extensions(&req.extensions) else {
-            return Err(admin_s3_error(AdminS3ErrorCode::InternalError, "object store is not initialized"));
-        };
-        let page = list_recovery_controls(store, query.protocol, query.classification, query.limit, query.marker)
-            .await
-            .map_err(map_recovery_control_error)?;
-        json_response(StatusCode::OK, &page)
-    }
-}
-
-pub struct IlmRecoveryControlInspectHandler {}
-
-#[async_trait::async_trait]
-impl Operation for IlmRecoveryControlInspectHandler {
-    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?;
-        let control_id = recovery_control_id_from_params(&params)?;
-        let Some(store) = object_store_from_extensions(&req.extensions) else {
-            return Err(admin_s3_error(AdminS3ErrorCode::InternalError, "object store is not initialized"));
-        };
-        let control = inspect_recovery_control(store, &control_id)
-            .await
-            .map_err(map_recovery_control_error)?;
-        json_response(StatusCode::OK, &control)
-    }
-}
-
 pub struct TransitionReconcileApplyHandler {}
 
 #[async_trait::async_trait]
@@ -1199,49 +1104,6 @@ mod tests {
         f(&matched.params)
     }
 
-    fn with_recovery_control_params<T>(path: &str, f: impl FnOnce(&Params<'_, '_>) -> T) -> T {
-        let mut router = Router::new();
-        router
-            .insert("/rustfs/admin/v3/ilm/recovery/records/{control_id}", ())
-            .expect("route should insert");
-        let matched = router.at(path).expect("route should match");
-        f(&matched.params)
-    }
-
-    #[test]
-    fn recovery_control_query_is_bounded_and_strict() {
-        let query = parse_recovery_control_list_query(Some("protocol=transition_transaction"))
-            .expect("minimal recovery query should parse");
-        assert_eq!(query.protocol, IlmRecoveryProtocol::TransitionTransaction);
-        assert_eq!(query.classification, None);
-        assert_eq!(query.limit, 100);
-
-        let filtered = parse_recovery_control_list_query(Some(
-            "protocol=tier_delete_journal&classification=retained_ambiguous&limit=1000&marker=opaque",
-        ))
-        .expect("bounded filtered query should parse");
-        assert_eq!(filtered.protocol, IlmRecoveryProtocol::TierDeleteJournal);
-        assert_eq!(filtered.classification, Some(IlmRecoveryClassification::RetainedAmbiguous));
-        assert_eq!(filtered.limit, 1000);
-        assert!(parse_recovery_control_list_query(None).is_err());
-        assert!(parse_recovery_control_list_query(Some("protocol=transition_transaction&limit=0")).is_err());
-        assert!(parse_recovery_control_list_query(Some("protocol=transition_transaction&limit=1001")).is_err());
-        assert!(parse_recovery_control_list_query(Some("protocol=unknown")).is_err());
-        assert!(parse_recovery_control_list_query(Some("protocol=transition_transaction&extra=true")).is_err());
-    }
-
-    #[test]
-    fn recovery_control_id_is_canonical_lowercase_sha256() {
-        let id = "ab".repeat(32);
-        with_recovery_control_params(&format!("/rustfs/admin/v3/ilm/recovery/records/{id}"), |params| {
-            assert_eq!(recovery_control_id_from_params(params).expect("control id should parse"), id);
-        });
-        let uppercase = "AB".repeat(32);
-        with_recovery_control_params(&format!("/rustfs/admin/v3/ilm/recovery/records/{uppercase}"), |params| {
-            assert!(recovery_control_id_from_params(params).is_err())
-        });
-    }
-
     fn manual_transition_job_request(method: Method, path: &'static str) -> S3Request<Body> {
         S3Request {
             input: Body::empty(),

rustfs/src/admin/storage_api.rs

@@ -232,9 +232,6 @@ pub(crate) mod lifecycle {
     pub(crate) type ManualTransitionRunOptions =
         super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunOptions;
     pub(crate) type ManualTransitionRunReport = super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunReport;
-    pub(crate) use super::ecstore_bucket::lifecycle::recovery_control::{
-        IlmRecoveryClassification, IlmRecoveryProtocol, inspect_recovery_control, list_recovery_controls,
-    };
     pub(crate) use super::ecstore_bucket::lifecycle::transition_transaction::{
         TransitionOperatorDeleteResult, TransitionOperatorError, delete_transition_candidate_for_operator,
         finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,

scripts/README.md

@@ -54,6 +54,8 @@ their issue closes.
 | `probe.sh` | dev-tool | Probe-style e2e run | `make probe-e2e` |
 | `run_scanner_validation_harness.sh` | dev-tool | Scanner validation harness | `docs/operations/scanner-benchmark-runbook.md` |
 | `test_scanner_validation_harness.sh` | dev-tool | Self-test for the scanner validation harness | — |
+| `scanner_abba.py` | dev-tool | Scanner/heal ABBA orchestration and evidence gates via `run_scanner_validation_harness.sh --abba` | `docs/operations/scanner-benchmark-runbook.md` |
+| `test_scanner_abba.py` | dev-tool | Synthetic ABBA adapter and failure-path tests | `test_scanner_validation_harness.sh` |
 | `test_build_rustfs_options.sh` | dev-tool | Shell test for rustfs build-option wiring | `make test` (script-tests) |
 | `test_entrypoint_credentials.sh` | dev-tool | Container entrypoint credential-handling test | `make test` (script-tests) |
 | `test_helm_chart_version.sh` | dev-tool | Test for `helm_chart_version.sh` | — |

scripts/run_scanner_validation_harness.sh

@@ -1,6 +1,12 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+if [[ "${1:-}" == "--abba" ]]; then
+  shift
+  SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+  exec "$SCRIPT_DIR/python_bin.sh" "$SCRIPT_DIR/scanner_abba.py" "$@"
+fi
+
 ALIAS=""
 ENDPOINT=""
 ACCESS_KEY="${RUSTFS_ACCESS_KEY:-}"
@@ -23,6 +29,7 @@ TELEMETRY_PIDS=()
 usage() {
   cat <<'USAGE'
 Usage:
+  scripts/run_scanner_validation_harness.sh --abba --help
   scripts/run_scanner_validation_harness.sh --alias <admin-alias> \
     --endpoint <url> [options]
 

scripts/scanner_abba.py

@@ -0,0 +1,384 @@
+#!/usr/bin/env python3
+"""Run isolated scanner/heal ABBA cells through a deployment-specific adapter."""
+
+import argparse
+from decimal import Decimal
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import shutil
+import signal
+import subprocess
+import sys
+import time
+
+SCENARIOS = ("cold-hot", "fresh-hot", "multi-hot-new", "running-heal", "mrf-replay")
+LEGS = ("A1", "B1", "B2", "A2")
+MAX_JSON_BYTES = 1024 * 1024
+METRICS = (
+    "p99_ms", "throughput_ops", "rss_bytes", "cpu_seconds", "iops", "rpc_count",
+    "cache_clone_bytes", "encode_bytes", "save_bytes", "oldest_age_seconds",
+    "walk_objects", "cold_walk_objects", "healed_objects", "errors", "requests",
+)
+REPEATABILITY_LIMIT = Decimal("0.05")
+P2_WORK_MULTIPLE_LIMIT = Decimal("1.2")
+
+
+def require(condition, message):
+    if not condition:
+        raise ValueError(message)
+
+
+def number(value, name, minimum=0):
+    require(type(value) in (float, int) and math.isfinite(value) and value >= minimum,
+            f"invalid {name}")
+    return value
+
+
+def decimal_number(value, name, minimum=0):
+    if isinstance(value, Decimal):
+        require(value.is_finite() and value >= Decimal(str(minimum)), f"invalid {name}")
+        return value
+    number(value, name, minimum)
+    return Decimal(str(value))
+
+
+def ratio(numerator, denominator, name):
+    denominator = decimal_number(denominator, f"{name} denominator")
+    require(denominator > 0, f"invalid {name} denominator")
+    return decimal_number(numerator, name) / denominator
+
+
+def relative_change(current, baseline, name):
+    return ratio(current, baseline, name) - Decimal("1")
+
+
+def repeatability_change(first, second, name):
+    first = decimal_number(first, name)
+    second = decimal_number(second, name)
+    if first == 0 and second == 0:
+        return Decimal("0")
+    if first == 0 or second == 0:
+        return Decimal("Infinity")
+    return abs(second / first - Decimal("1"))
+
+
+def report_number(value):
+    return None if value.is_infinite() else float(value)
+
+
+def digest(path):
+    with Path(path).open("rb") as stream:
+        return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def read_json(path):
+    require(path.stat().st_size <= MAX_JSON_BYTES, f"oversized JSON: {path.name}")
+    with path.open() as stream:
+        value = json.load(stream)
+    require(isinstance(value, dict), f"expected JSON object: {path.name}")
+    return value
+
+
+def write_json(path, value):
+    data = json.dumps(value, indent=2, allow_nan=False) + "\n"
+    require(len(data.encode()) <= MAX_JSON_BYTES, "oversized result")
+    path.write_text(data)
+
+
+def sha(value):
+    return isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value)
+
+
+def validate_manifest(manifest):
+    require(manifest.get("schema") == 1, "unsupported manifest schema")
+    require(manifest.get("evidence") in ("synthetic", "measured"), "missing evidence type")
+    fixed = manifest["fixed"]
+    for key in ("config_sha256", "dataset_sha256"):
+        require(sha(fixed.get(key)), f"invalid fixed.{key}")
+    for key in ("release_flags", "durability", "disk_type", "cache_state", "load_command", "resource_isolation"):
+        require(isinstance(fixed.get(key), str) and fixed[key].strip(), f"missing fixed.{key}")
+    require(fixed.get("topology") == "EC8+4", "formal matrix requires EC8+4")
+    number(fixed.get("offered_load_ops"), "offered load", 1)
+    require(type(manifest.get("rounds")) is int and 3 <= manifest["rounds"] <= 10,
+            "rounds must be 3..10")
+    minimum = 900 if manifest["evidence"] == "measured" else 1
+    require(type(manifest.get("duration_seconds")) is int and
+            minimum <= manifest["duration_seconds"] <= 86400, "invalid duration_seconds")
+    number(manifest.get("min_free_bytes"), "min_free_bytes", 1)
+    for phase in ("baseline", "candidate"):
+        build = manifest[phase]
+        path = Path(build["binary"]).resolve(strict=True)
+        require(path.is_file() and os.access(path, os.X_OK), f"missing executable {phase} build")
+        require(sha(build.get("sha256")) and digest(path) == build["sha256"], f"{phase} binary hash mismatch")
+        require(isinstance(build.get("revision"), str) and len(build["revision"]) == 40 and
+                all(c in "0123456789abcdef" for c in build["revision"]), f"invalid {phase} revision")
+        build["binary"] = str(path)
+    for scenario in SCENARIOS:
+        expected = manifest["oracles"][scenario]
+        for key in ("objects", "versions", "bytes"):
+            require(type(expected.get(key)) is int and expected[key] > 0, f"missing {scenario} oracle {key}")
+        require(sha(expected.get("sha256")), f"missing {scenario} content/version digest")
+        number(manifest["expected_healed_objects"].get(scenario), f"{scenario} expected repairs")
+        if scenario in ("running-heal", "mrf-replay"):
+            require(manifest["expected_healed_objects"][scenario] > 0, f"{scenario} requires repairs")
+
+
+def invoke(adapter, action, request, timeout):
+    """The adapter writes bounded JSON separately; stderr/stdout remain raw evidence."""
+    output = request.parent / f"{action}.json"
+    with (request.parent / f"{action}.log").open("wb") as log:
+        process = subprocess.Popen([str(adapter), action, str(request), str(output)],
+                                   stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
+        try:
+            returncode = process.wait(timeout=timeout)
+            if returncode:
+                raise subprocess.CalledProcessError(returncode, [str(adapter), action])
+        finally:
+            if process.poll() != 0:
+                try:
+                    os.killpg(process.pid, signal.SIGTERM)
+                except ProcessLookupError:
+                    pass
+                try:
+                    process.wait(timeout=10)
+                except subprocess.TimeoutExpired:
+                    os.killpg(process.pid, signal.SIGKILL)
+                    process.wait()
+    return read_json(output)
+
+
+def validate_result(result, request, expected):
+    require(result.get("evidence") == request["evidence"], "adapter evidence type mismatch")
+    require(result.get("fixed") == request["fixed"], "offered load/config/cache/durability drift")
+    require(result.get("build") == request["build"], "deployed build provenance mismatch")
+    require(result.get("data_dir") == request["data_dir"], "adapter data isolation mismatch")
+    require(result.get("background") == request["background"], "background mode mismatch")
+    require(type(result.get("sample_count")) is int and 1 <= result["sample_count"] <= 3600,
+            "sample_count must be 1..3600")
+    number(result.get("elapsed_seconds"), "elapsed_seconds", request["duration_seconds"])
+    metrics = result["metrics"]
+    for key in METRICS:
+        number(metrics.get(key), key)
+    for key in ("requests", "p99_ms", "throughput_ops"):
+        require(metrics[key] > 0, f"zero {key}")
+    require(metrics["errors"] == 0, "workload request errors")
+    require(metrics["cold_walk_objects"] <= metrics["walk_objects"], "cold walk exceeds total walk")
+    require(result.get("oracle") == expected, "object/version/byte oracle mismatch")
+    if request["background"] == "on":
+        require(metrics["walk_objects"] > 0, "zero background walk")
+        require(metrics["healed_objects"] == request["expected_healed_objects"], "incomplete repair oracle")
+        if request["scenario"] in ("running-heal", "mrf-replay"):
+            require(metrics["healed_objects"] > 0, "zero completed repairs")
+    return result
+
+
+def convergence(result):
+    window = result.get("convergence")
+    if not window or window.get("writes_stopped") is not True or window.get("last_mutation_observed") is not True or window.get("first_complete_publication") is not True:
+        return None
+    for key in ("last_mutation_time", "last_mutation_observed_time", "writes_stopped_time", "window_start", "window_end", "walk_objects", "full_walk_objects", "budget_available_seconds"):
+        number(window.get(key), f"convergence.{key}")
+    require(window["last_mutation_time"] <= window["writes_stopped_time"] <= window["window_start"] < window["window_end"],
+            "invalid post-mutation convergence window")
+    require(window["last_mutation_time"] <= window["last_mutation_observed_time"] <= window["window_start"],
+            "convergence started before last mutation was observed")
+    require(window["full_walk_objects"] > 0, "zero full walk reference")
+    require(0 < window["budget_available_seconds"] <= window["window_end"] - window["window_start"],
+            "invalid convergence budget window")
+    return window["walk_objects"] / window["full_walk_objects"]
+
+
+def evaluate(cells):
+    comparisons = []
+    inconclusive = False
+    failed = False
+    for offset in range(0, len(cells), 4):
+        group = cells[offset:offset + 4]
+        require([cell["leg"] for cell in group] == list(LEGS), "incomplete ABBA group")
+        a1, b1, b2, a2 = (cell["result"]["metrics"] for cell in group)
+        control = group[0]["comparison"] == "background"
+        drift = max(abs(relative_change(a2[k], a1[k], k)) for k in ("p99_ms", "throughput_ops"))
+        repeat_drift = max(abs(relative_change(b2[k], b1[k], k)) for k in ("p99_ms", "throughput_ops"))
+        noise = max(drift, repeat_drift) > REPEATABILITY_LIMIT
+        a = {key: (decimal_number(a1[key], key) + decimal_number(a2[key], key)) / Decimal("2") for key in METRICS}
+        b = {key: (decimal_number(b1[key], key) + decimal_number(b2[key], key)) / Decimal("2") for key in METRICS}
+        p99 = relative_change(b["p99_ms"], a["p99_ms"], "p99_ms")
+        throughput = relative_change(b["throughput_ops"], a["throughput_ops"], "throughput_ops")
+        thresholds = {"p99_regression": Decimal("0.10") if control else Decimal("0.05"),
+                      "throughput_loss": Decimal("0.05") if control else Decimal("0.03")}
+        passed = p99 <= thresholds["p99_regression"] and throughput >= -thresholds["throughput_loss"]
+        p1 = None
+        work_drift = None
+        if not control:
+            if group[0]["scenario"] == "cold-hot":
+                require(a["cold_walk_objects"] > 0, "cold-hot baseline has no cold walk samples")
+            work_drift = max(repeatability_change(a1[key], a2[key], key) for key in ("walk_objects", "cold_walk_objects"))
+            work_drift = max(work_drift, *(repeatability_change(b1[key], b2[key], key) for key in ("walk_objects", "cold_walk_objects")))
+            noise |= work_drift > REPEATABILITY_LIMIT
+            required = ratio(a["cold_walk_objects"], a["walk_objects"], "cold walk baseline") * Decimal("0.80")
+            reduction = Decimal("1") - ratio(b["walk_objects"], a["walk_objects"], "walk reduction")
+            p1 = {"required_reduction": float(required), "observed_reduction": float(reduction),
+                  "repeatability_drift": report_number(work_drift)}
+            if group[0]["scenario"] == "cold-hot":
+                # Compare counts before division can round repeating decimal ratios.
+                passed &= a["walk_objects"] - b["walk_objects"] >= a["cold_walk_objects"] * Decimal("0.80")
+        p2 = [convergence(cell["result"]) if cell["background"] == "on" else None for cell in group]
+        candidate_p2 = [value for cell, value in zip(group, p2) if cell["leg"].startswith("B")]
+        p2_pending = any(value is None for value in candidate_p2)
+        passed &= all(ratio(value, 1, "p2 work multiple") <= P2_WORK_MULTIPLE_LIMIT for value in candidate_p2 if value is not None)
+        inconclusive |= noise or p2_pending
+        if not noise and not passed:
+            failed = True
+        comparisons.append({"scenario": group[0]["scenario"], "comparison": group[0]["comparison"],
+                            "round": group[0]["round"], "status": "inconclusive" if noise else ("fail" if not passed else "inconclusive" if p2_pending else "pass"),
+                            "a2_a1_drift": report_number(drift), "b2_b1_drift": report_number(repeat_drift),
+                            "p99_regression": float(p99), "throughput_change": float(throughput),
+                            "thresholds": {key: float(value) for key, value in thresholds.items()},
+                            "p1": p1, "p2_max_work_multiple": float(P2_WORK_MULTIPLE_LIMIT),
+                            "p2_post_stop_work_multiples": p2})
+    return ("fail" if failed else "inconclusive" if inconclusive else "pass"), comparisons
+
+
+def collect_live(prepared, request, request_path, adapter):
+    collector = Path(__file__).with_name("run_scanner_validation_harness.sh")
+    # Only allow connection fields here; the runner owns cadence and output paths.
+    connection = prepared["collector"]
+    require(set(connection) == {"alias", "endpoint", "metrics_endpoints"}, "invalid collector connection")
+    require(all(isinstance(value, str) and value for value in connection.values()), "missing collector endpoint")
+    output = request_path.parent / "telemetry"
+    args = ["bash", str(collector), "--alias", connection["alias"], "--endpoint", connection["endpoint"],
+            "--metrics-endpoints", connection["metrics_endpoints"], "--deployment", "distributed",
+            "--samples", str(request["duration_seconds"] // 60 + 1), "--interval-secs", "60",
+            "--out-dir", str(output)]
+    with (request_path.parent / "collector.log").open("wb") as log:
+        process = subprocess.Popen(args, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
+        try:
+            started = time.monotonic()
+            result = invoke(adapter, "measure", request_path, request["duration_seconds"] + 300)
+            require(time.monotonic() - started >= request["duration_seconds"], "measurement ended before required window")
+            require(process.wait(timeout=120) == 0, "scanner collector failed")
+            require(output.joinpath("scanner-summary.csv").stat().st_size > 0, "missing collector samples")
+            samples = list((output / "status").glob("scanner-status.*.json"))
+            require(len(samples) == request["duration_seconds"] // 60 + 1, "missing scanner samples")
+            for sample in samples:
+                status = read_json(sample)
+                require(isinstance(status.get("metrics"), dict) and status["metrics"], "invalid scanner status response")
+            heals = list((output / "heal").glob("background-heal-status.*.json"))
+            require(bool(heals), "missing heal samples")
+            for sample in heals:
+                status = read_json(sample)
+                require(isinstance(status.get("healOperations"), dict) and status["healOperations"], "invalid heal status response")
+            metrics = list((output / "metrics").glob("admin-metrics.*.ndjson"))
+            endpoints = [endpoint for endpoint in connection["metrics_endpoints"].split(",") if endpoint]
+            require(metrics and len(metrics) == len(endpoints) * len(samples), "missing distributed metrics samples")
+            for sample in metrics:
+                # The collector requests n=1, so each file contains one final JSON record.
+                status = read_json(sample)
+                require(status.get("errors") == [], "distributed metrics errors")
+                require(status.get("final") is True, "incomplete distributed metrics")
+                hosts = status.get("by_host")
+                require(isinstance(hosts, dict) and hosts, "missing by-host metrics")
+                for host in hosts.values():
+                    require(isinstance(host, dict) and isinstance(host.get("scanner"), dict) and host["scanner"],
+                            "missing per-host scanner metrics")
+            return result
+        finally:
+            # Stop telemetry children as well when measurement fails or times out.
+            try:
+                os.killpg(process.pid, signal.SIGTERM)
+            except ProcessLookupError:
+                pass
+            try:
+                process.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                os.killpg(process.pid, signal.SIGKILL)
+                process.wait()
+
+
+def run(manifest, adapter, output, data_root):
+    validate_manifest(manifest)
+    require(adapter.is_file() and os.access(adapter, os.X_OK), "missing executable adapter")
+    require(not output.exists() and not data_root.exists(), "output/data root must be new; existing data is preserved")
+    require(output != data_root and output not in data_root.parents and data_root not in output.parents,
+            "output and data roots must not overlap")
+    output.mkdir(parents=True)
+    data_root.mkdir(parents=True)
+    require(shutil.disk_usage(data_root).free >= manifest["min_free_bytes"], "insufficient free disk space")
+    manifest["adapter_sha256"] = digest(adapter)
+    manifest["collector_sha256"] = digest(Path(__file__).with_name("run_scanner_validation_harness.sh"))
+    write_json(output / "manifest.json", manifest)
+    cells = []
+    write_json(output / "report.json", {"status": "incomplete", "performance": "pending"})
+    try:
+        for scenario in SCENARIOS:
+            for comparison in ("build", "background"):
+                for round_id in range(1, manifest["rounds"] + 1):
+                    for leg in LEGS:
+                        phase = "baseline" if comparison == "build" and leg.startswith("A") else "candidate"
+                        background = "off" if comparison == "background" and leg.startswith("A") else "on"
+                        name = f"{scenario}-{comparison}-{round_id}-{leg}"
+                        cell_dir = output / name
+                        cell_dir.mkdir()
+                        data_dir = data_root / name
+                        data_dir.mkdir()
+                        request = {"schema": 1, "scenario": scenario, "comparison": comparison, "round": round_id,
+                                   "leg": leg, "background": background, "build": manifest[phase],
+                                   "evidence": manifest["evidence"], "fixed": manifest["fixed"],
+                                   "duration_seconds": manifest["duration_seconds"], "data_dir": str(data_dir),
+                                   "expected_healed_objects": manifest["expected_healed_objects"][scenario],
+                                   "expected_oracle": manifest["oracles"][scenario]}
+                        require(digest(Path(request["build"]["binary"])) == request["build"]["sha256"], "binary changed during run")
+                        require(digest(adapter) == manifest["adapter_sha256"], "adapter changed during run")
+                        require(shutil.disk_usage(data_root).free >= manifest["min_free_bytes"], "insufficient free disk space")
+                        request_path = cell_dir / "request.json"
+                        write_json(request_path, request)
+                        print(name, flush=True)
+                        try:
+                            prepared = invoke(adapter, "prepare", request_path, 300)
+                            require(prepared.get("ready") is True, "deployment not ready")
+                            if manifest["evidence"] == "measured":
+                                result = collect_live(prepared, request, request_path, adapter)
+                            else:
+                                result = invoke(adapter, "measure", request_path, 300)
+                            # An independent operation must enumerate all object versions and bytes.
+                            oracle = invoke(adapter, "oracle", request_path, 300)
+                            require(oracle.get("complete") is True and oracle.get("errors") == 0, "correctness oracle failed")
+                            require(type(oracle.get("errors")) is int, "invalid oracle error count")
+                            result["oracle"] = oracle["actual"]
+                            validate_result(result, request, request["expected_oracle"])
+                            cells.append({**request, "result": result})
+                        finally:
+                            stopped = invoke(adapter, "stop", request_path, 300)
+                            require(stopped.get("stopped") is True, "adapter failed to stop deployment")
+        status, comparisons = evaluate(cells)
+        synthetic = manifest["evidence"] == "synthetic"
+        report = {"status": "synthetic_validated" if synthetic and status == "pass" else status,
+                  "evidence": manifest["evidence"], "performance": "pending" if synthetic else status,
+                  "cells": len(cells), "comparisons": comparisons}
+        write_json(output / "report.json", report)
+        return 0 if status == "pass" else 3 if status == "inconclusive" else 1
+    except (ValueError, KeyError, OSError, subprocess.SubprocessError) as error:
+        write_json(output / "report.json", {"status": "failed", "performance": "pending",
+                                            "completed_cells": len(cells), "error": str(error)})
+        raise
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--adapter", type=Path, required=True)
+    parser.add_argument("--out-dir", type=Path, required=True)
+    parser.add_argument("--data-root", type=Path, required=True)
+    args = parser.parse_args()
+    try:
+        return run(read_json(args.manifest), args.adapter.resolve(), args.out_dir.resolve(), args.data_root.resolve())
+    except (ValueError, KeyError, OSError, subprocess.SubprocessError) as error:
+        print(f"ERROR: {error}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/test_scanner_abba.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+"""Synthetic adapter and failure-propagation tests; never start a RustFS server."""
+
+import contextlib
+import copy
+import io
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import Mock, patch
+
+import scanner_abba as harness
+
+
+def fake_adapter():
+    action, request_path, output_path = sys.argv[1:]
+    request = harness.read_json(Path(request_path))
+    fault = os.environ.get("SCANNER_ABBA_TEST_FAULT", "")
+    if action == "prepare":
+        result = {"ready": True}
+    elif action == "stop":
+        result = {"stopped": True}
+    elif action == "oracle":
+        if fault == "oracle-exit":
+            return 42
+        if fault == "missing-oracle":
+            return 0
+        result = {"complete": True, "errors": 0, "actual": request["expected_oracle"]}
+        if fault == "oracle-mismatch":
+            result["actual"]["bytes"] += 1
+    else:
+        if fault == "measure-exit":
+            return 42
+        result = {key: request[key] for key in ("evidence", "fixed", "build", "data_dir", "background")}
+        result.update({"sample_count": 10, "elapsed_seconds": request["duration_seconds"],
+                       "metrics": dict.fromkeys(harness.METRICS, 10)})
+        baseline = request["comparison"] == "build" and request["leg"].startswith("A")
+        result["metrics"].update(p99_ms=10, throughput_ops=100, errors=0, requests=100,
+                                 walk_objects=100 if baseline else 20, cold_walk_objects=100 if baseline else 0,
+                                 healed_objects=request["expected_healed_objects"])
+        result["convergence"] = {"writes_stopped": True, "last_mutation_observed": True,
+                                 "first_complete_publication": True, "last_mutation_time": 1,
+                                 "last_mutation_observed_time": 2,
+                                 "writes_stopped_time": 2, "window_start": 2, "window_end": 3,
+                                 "budget_available_seconds": 1, "walk_objects": 110, "full_walk_objects": 100}
+        if fault == "zero-samples":
+            result["sample_count"] = 0
+        elif fault == "request-errors":
+            result["metrics"]["errors"] = 1
+        elif fault == "load-drift":
+            result["fixed"]["offered_load_ops"] += 1
+        elif fault == "noise" and request["leg"] == "A2":
+            result["metrics"]["p99_ms"] = 20
+        elif fault == "zero-requests":
+            result["metrics"]["requests"] = 0
+        elif fault == "no-publication":
+            result["convergence"]["first_complete_publication"] = False
+        elif fault == "p2-regression":
+            result["convergence"]["walk_objects"] = 121
+        elif fault == "latency-regression" and request["leg"].startswith("B"):
+            result["metrics"]["p99_ms"] = 12
+        elif fault == "exact-thresholds" and request["leg"].startswith("B"):
+            result["metrics"].update(p99_ms=10.5, throughput_ops=97)
+        elif fault == "just-over-threshold" and request["comparison"] == "build" and request["leg"].startswith("B"):
+            result["metrics"]["p99_ms"] = 10.500001
+        elif fault == "p1-regression" and not baseline:
+            result["metrics"]["walk_objects"] = 30
+        elif fault in ("p1-exact-fraction", "p1-over-fraction"):
+            result["metrics"].update(walk_objects=9 if baseline else 5 + (fault == "p1-over-fraction"),
+                                     cold_walk_objects=5 if baseline else 0)
+        elif fault == "unstable-p1-control" and request["comparison"] == "build":
+            if request["leg"] == "A1":
+                result["metrics"].update(walk_objects=1000, cold_walk_objects=1000)
+            elif request["leg"] == "A2":
+                result["metrics"].update(walk_objects=10, cold_walk_objects=10)
+            elif request["leg"].startswith("B"):
+                result["metrics"].update(walk_objects=100, cold_walk_objects=0)
+        elif fault == "missing-metric":
+            del result["metrics"]["save_bytes"]
+        elif fault == "incomplete-repair":
+            result["metrics"]["healed_objects"] = 0
+    harness.write_json(Path(output_path), result)
+    return 0
+
+
+class ScannerAbbaTest(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.binary = Path(sys.executable).resolve()
+        self.adapter = Path(__file__).resolve()
+        self.manifest = {
+            "schema": 1, "evidence": "synthetic", "rounds": 3, "duration_seconds": 1, "min_free_bytes": 1,
+            "fixed": {"config_sha256": "1" * 64, "dataset_sha256": "2" * 64,
+                      "release_flags": "--release", "durability": "drive-sync=on",
+                      "disk_type": "synthetic", "cache_state": "cold", "load_command": "fake",
+                      "topology": "EC8+4", "offered_load_ops": 100, "resource_isolation": "synthetic"},
+            "oracles": {s: {"objects": 10, "versions": 20, "bytes": 30, "sha256": "3" * 64} for s in harness.SCENARIOS},
+            "expected_healed_objects": {s: 10 for s in harness.SCENARIOS},
+        }
+        build = {"binary": str(self.binary), "sha256": harness.digest(self.binary), "revision": "a" * 40}
+        self.manifest.update(baseline=build.copy(), candidate=build.copy())
+
+    def run_harness(self, fault=""):
+        with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": fault}), contextlib.redirect_stdout(io.StringIO()):
+            return harness.run(copy.deepcopy(self.manifest), self.adapter, self.root / "out", self.root / "data")
+
+    def test_complete_synthetic_matrix_is_not_performance_evidence(self):
+        self.assertEqual(self.run_harness(), 0)
+        report = harness.read_json(self.root / "out/report.json")
+        self.assertEqual((report["status"], report["performance"], report["cells"]), ("synthetic_validated", "pending", 120))
+        requests = [harness.read_json(path) for path in (self.root / "out").glob("*/request.json")]
+        self.assertEqual(len({r["data_dir"] for r in requests}), 120)
+        for scenario in harness.SCENARIOS:
+            for comparison in ("build", "background"):
+                for round_id in (1, 2, 3):
+                    legs = [r for r in requests if (r["scenario"], r["comparison"], r["round"]) == (scenario, comparison, round_id)]
+                    self.assertEqual({r["leg"] for r in legs}, set(harness.LEGS))
+        self.assertTrue(all(c["p2_max_work_multiple"] == 1.2 for c in report["comparisons"]))
+
+    def test_fail_closed_adapter_and_data_errors(self):
+        for fault in ("measure-exit", "oracle-exit", "missing-oracle", "oracle-mismatch", "zero-samples",
+                      "zero-requests", "request-errors", "load-drift", "missing-metric", "incomplete-repair"):
+            with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory:
+                self.root = Path(directory)
+                with self.assertRaises((ValueError, OSError, subprocess.SubprocessError)):
+                    self.run_harness(fault)
+                report = harness.read_json(self.root / "out/report.json")
+                self.assertEqual(report["status"], "failed")
+                self.assertTrue(list((self.root / "out").glob("*/stop.json")))
+
+    def test_noise_is_inconclusive_and_nonzero(self):
+        with patch.object(harness, "SCENARIOS", ("cold-hot",)):
+            self.assertEqual(self.run_harness("noise"), 3)
+        self.assertEqual(harness.read_json(self.root / "out/report.json")["status"], "inconclusive")
+
+    def test_missing_first_publication_is_inconclusive(self):
+        with patch.object(harness, "SCENARIOS", ("cold-hot",)):
+            self.assertEqual(self.run_harness("no-publication"), 3)
+
+    def test_performance_regressions_fail(self):
+        for fault in ("p1-regression", "p2-regression", "latency-regression"):
+            with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory:
+                self.root = Path(directory)
+                with patch.object(harness, "SCENARIOS", ("cold-hot",)):
+                    self.assertEqual(self.run_harness(fault), 1)
+
+    def test_exact_threshold_boundaries_pass(self):
+        with patch.object(harness, "SCENARIOS", ("cold-hot",)):
+            self.assertEqual(self.run_harness("exact-thresholds"), 0)
+
+    def test_just_over_threshold_fails(self):
+        with patch.object(harness, "SCENARIOS", ("cold-hot",)):
+            self.assertEqual(self.run_harness("just-over-threshold"), 1)
+
+    def test_p1_fractional_boundary(self):
+        for fault, expected in (("p1-exact-fraction", 0), ("p1-over-fraction", 1)):
+            with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory:
+                self.root = Path(directory)
+                with patch.object(harness, "SCENARIOS", ("cold-hot",)):
+                    self.assertEqual(self.run_harness(fault), expected)
+
+    def test_live_collector_rejects_missing_or_failed_node_metrics(self):
+        telemetry = self.root / "telemetry"
+        for name in ("status", "heal", "metrics"):
+            (telemetry / name).mkdir(parents=True)
+        (telemetry / "scanner-summary.csv").write_text("timestamp\n")
+        valid = {"errors": [], "final": True, "by_host": {"node-b:9000": {"scanner": {"objects": 10}}}}
+        for index in range(16):
+            harness.write_json(telemetry / f"status/scanner-status.{index}.json", {"metrics": {"objects": 10}})
+            for node in ("node-a", "node-b"):
+                harness.write_json(telemetry / f"heal/background-heal-status.{node}.{index}.json",
+                                   {"healOperations": {"queueLength": 0}})
+                harness.write_json(telemetry / f"metrics/admin-metrics.{node}.{index}.ndjson",
+                                   {**valid, "by_host": {f"{node}:9000": {"scanner": {"objects": 10}}}})
+        sample = telemetry / "metrics/admin-metrics.node-b.15.ndjson"
+        prepared = {"collector": {"alias": "test", "endpoint": "http://node-a:9000",
+                                  "metrics_endpoints": "http://node-a:9000,http://node-b:9000"}}
+        cases = (
+            ("valid", valid, None),
+            ("missing", None, "missing distributed metrics samples"),
+            ("empty", "", "Expecting value"),
+            ("http-error", {"Code": "AccessDenied"}, "distributed metrics errors"),
+            ("partial-error", {**valid, "errors": ["node unavailable"]}, "distributed metrics errors"),
+            ("unfinished", {**valid, "final": False}, "incomplete distributed metrics"),
+            ("missing-host", {**valid, "by_host": {}}, "missing by-host metrics"),
+            ("missing-scanner", {**valid, "by_host": {"node-a:9000": {}}}, "missing per-host scanner metrics"),
+            ("collector-exit", valid, "scanner collector failed"),
+        )
+        for name, payload, error in cases:
+            with self.subTest(fault=name):
+                if payload is None:
+                    sample.unlink()
+                elif isinstance(payload, str):
+                    sample.write_text(payload)
+                else:
+                    harness.write_json(sample, payload)
+                process = Mock(pid=123, wait=Mock(return_value=1 if name == "collector-exit" else 0))
+                with patch.object(harness.subprocess, "Popen", return_value=process), \
+                        patch.object(harness, "invoke", return_value={"sample_count": 10}), \
+                        patch.object(harness.time, "monotonic", side_effect=(0, 900)), \
+                        patch.object(harness.os, "killpg"):
+                    if error:
+                        with self.assertRaisesRegex(ValueError, error):
+                            harness.collect_live(prepared, {"duration_seconds": 900}, self.root / "request.json", self.adapter)
+                    else:
+                        self.assertEqual(harness.collect_live(prepared, {"duration_seconds": 900},
+                                                             self.root / "request.json", self.adapter), {"sample_count": 10})
+
+    def test_unstable_p1_work_control_is_inconclusive(self):
+        with patch.object(harness, "SCENARIOS", ("cold-hot",)):
+            self.assertEqual(self.run_harness("unstable-p1-control"), 3)
+        comparison = harness.read_json(self.root / "out/report.json")["comparisons"][0]
+        self.assertEqual(comparison["status"], "inconclusive")
+        self.assertGreater(comparison["p1"]["repeatability_drift"], 0.05)
+
+    def test_manifest_rejects_missing_build_or_oracle(self):
+        for section, key in (("baseline", "binary"), ("oracles", "cold-hot")):
+            manifest = copy.deepcopy(self.manifest)
+            del manifest[section][key]
+            with self.subTest(section=section), self.assertRaises((ValueError, KeyError)):
+                harness.validate_manifest(manifest)
+
+    def test_short_measured_window_and_fewer_rounds_rejected(self):
+        self.manifest["evidence"] = "measured"
+        with self.assertRaisesRegex(ValueError, "duration_seconds"):
+            harness.validate_manifest(self.manifest)
+        self.manifest["duration_seconds"] = 900
+        self.manifest["rounds"] = 2
+        with self.assertRaisesRegex(ValueError, "rounds"):
+            harness.validate_manifest(self.manifest)
+
+    def test_existing_data_preserved(self):
+        (self.root / "data").mkdir()
+        marker = self.root / "data/keep"
+        marker.write_text("existing")
+        with self.assertRaisesRegex(ValueError, "preserved"):
+            self.run_harness()
+        self.assertEqual(marker.read_text(), "existing")
+
+    def test_invalid_or_live_write_window_does_not_claim_p2(self):
+        self.assertIsNone(harness.convergence({"convergence": {"writes_stopped": False}}))
+        with self.assertRaises(ValueError):
+            harness.convergence({"convergence": {"writes_stopped": True, "last_mutation_observed": True,
+                                                 "first_complete_publication": True}})
+
+    def test_nan_and_oversized_samples_rejected(self):
+        with self.assertRaises(ValueError):
+            harness.number(float("nan"), "latency")
+        path = self.root / "oversized.json"
+        path.write_bytes(b" " * (harness.MAX_JSON_BYTES + 1))
+        with self.assertRaisesRegex(ValueError, "oversized"):
+            harness.read_json(path)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 4 and sys.argv[1] in ("prepare", "measure", "oracle", "stop"):
+        sys.exit(fake_adapter())
+    unittest.main()

scripts/test_scanner_validation_harness.sh

@@ -312,3 +312,5 @@ if PATH="$BIN_DIR:$PATH" "$SCRIPT" --secret-key rustfsadmin >"$secret_arg_log" 2
 fi
 
 grep -q -- 'unknown arg: --secret-key' "$secret_arg_log"
+
+"$ROOT_DIR/scripts/python_bin.sh" "$ROOT_DIR/scripts/test_scanner_abba.py"