Merge branch 'main' into codex/test-lifecycle-adversarial

.config/make/tests.mak

@@ -41,7 +41,6 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
-	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test

.github/pull_request_template.md

@@ -10,16 +10,16 @@ Use N/A when there is no related issue.
 
 ## Summary of Changes
 <!--
-Describe the concrete problem and resulting behavior. For a behavior change, name the input or state that triggers it and the expected outcome. Explain any new dependency or abstraction that the change needs.
+Briefly explain what changed and why reviewers should accept it.
+Focus on behavior, compatibility, and review-relevant context.
 -->
 
 ## Verification
 <!--
-Give 1–3 concrete pieces of evidence for the changed behavior: the test or command, its observed result, and the regression it catches. For a bug fix, record a failing-before/passing-after check or explain why it was unavailable.
+List the commands or checks you ran, for example:
+- `make pre-commit`
 
-Identify the tested commit and any local changes. When testing a prebuilt binary or external service, include its source/version and artifact identity; a successful run against a different build is not evidence for this change.
-
-List relevant checks not run and the remaining risk. Use the validation tier in AGENTS.md; do not run broader checks solely to fill this section. For documentation-only changes, list the applicable documentation checks. Use N/A only when verification is not applicable.
+Use N/A only when verification is not applicable.
 -->
 
 ## Impact

.github/workflows/ci.yml

@@ -168,7 +168,6 @@ jobs:
           python3 ./scripts/check_test_wiring.py --self-test
           python3 ./scripts/check_scheduled_validation_freshness.py --self-test
           python3 ./scripts/test_security_workflow.py
-          python3 ./scripts/test_nightly_candidate.py
           python3 ./scripts/check_test_wiring.py
 
       - name: Check no planning docs committed

.github/workflows/nightly-gnu.yml

@@ -166,9 +166,8 @@ jobs:
       # e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... .
       # Skipped when the R2 secrets are not configured (artifact-only mode).
       - name: Upload DEB to Cloudflare R2
-        id: publish
+        if: env.R2_ACCESS_KEY_ID != ''
         env:
-          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
           R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
@@ -183,70 +182,28 @@ jobs:
             exit 0
           fi
 
+          if ! command -v aws >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y -qq awscli
+          fi
+
           export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
-          SOURCE_SHA="$(git rev-parse HEAD)"
-          if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
-            echo "Checkout SHA does not match the nightly build run" >&2
-            exit 1
-          fi
-          DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
-          CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
-          CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
-
-          # Old AWS CLI models lack conditional PutObject support. Never fall
-          # back to an overwriting upload for a candidate.
-          AWS_CLI=aws
-          if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
-            sudo apt-get update
-            sudo apt-get install -y -qq python3-venv
-            AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")"
-            trap 'rm -rf "${AWS_CLI_DIR}"' EXIT
-            python3 -m venv "${AWS_CLI_DIR}"
-            "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
-            AWS_CLI="${AWS_CLI_DIR}/bin/aws"
-          fi
-          "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
-          "${AWS_CLI}" --version
-          "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \
-            --body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
-          PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)"
-          if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then
-            echo "Published candidate checksum does not match the built package" >&2
-            exit 1
-          fi
-
+          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
           R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/"
 
           echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}"
-          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
 
           # Stable "latest" alias so tests can fetch the newest nightly
           # without knowing today's date.
           echo "📤 Uploading latest alias"
-          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
+          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
             --endpoint-url "$R2_ENDPOINT" --only-show-errors
 
           echo "✅ R2 upload complete"
 
-          CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
-          jq -n --arg source_sha "${SOURCE_SHA}" \
-            --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
-            --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
-            '{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
-            > "${CANDIDATE_FILE}"
-          echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"
-
-      - name: Upload nightly candidate manifest
-        if: ${{ steps.publish.outputs.candidate_file != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ steps.publish.outputs.candidate_file }}
-          if-no-files-found: error
-
   # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
   #
   # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and

crates/lifecycle/src/core.rs

@@ -43,6 +43,10 @@ const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
     "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
 const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
 const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "'Days' for Expiration action must be a positive integer";
+const ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT: &str = "Expiration cannot specify both Days and Date";
+const ERR_LIFECYCLE_MULTIPLE_TRANSITIONS: &str = "Only one Transition action per lifecycle rule is supported";
+const ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS: &str =
+    "Only one NoncurrentVersionTransition action per lifecycle rule is supported";
 const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str =
     "'NoncurrentDays' for NoncurrentVersionExpiration action must be a positive integer";
 const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str =
@@ -361,6 +365,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
                 {
                     return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS));
                 }
+                if expiration.days.is_some() && expiration.date.is_some() {
+                    return Err(std::io::Error::new(
+                        std::io::ErrorKind::InvalidInput,
+                        ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT,
+                    ));
+                }
                 if let Some(expiration_date) = &expiration.date {
                     let date = OffsetDateTime::from(expiration_date.clone());
                     if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 {
@@ -394,11 +404,20 @@ impl Lifecycle for BucketLifecycleConfiguration {
                 }
             }
             if let Some(transitions) = &r.transitions {
+                if transitions.len() > 1 {
+                    return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, ERR_LIFECYCLE_MULTIPLE_TRANSITIONS));
+                }
                 for transition in transitions {
                     TransitionOps::validate(transition)?;
                 }
             }
             if let Some(noncurrent_transitions) = &r.noncurrent_version_transitions {
+                if noncurrent_transitions.len() > 1 {
+                    return Err(std::io::Error::new(
+                        std::io::ErrorKind::InvalidInput,
+                        ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS,
+                    ));
+                }
                 for transition in noncurrent_transitions {
                     NoncurrentVersionTransitionOps::validate(transition)?;
                 }
@@ -473,6 +492,8 @@ impl Lifecycle for BucketLifecycleConfiguration {
     }
 
     async fn eval(&self, obj: &ObjectOpts) -> Event {
+        // A single-object lookup cannot prove how many newer historical versions
+        // survive. Count-dependent actions wait for the complete-group evaluator.
         self.eval_inner(obj, OffsetDateTime::now_utc(), 0).await
     }
 
@@ -536,23 +557,8 @@ impl Lifecycle for BucketLifecycleConfiguration {
             return Event::default();
         };
 
-        if let Some(restore_expires) = obj.restore_expires
-            && restore_expires.unix_timestamp() != 0
-            && now.unix_timestamp() > restore_expires.unix_timestamp()
-        {
-            let mut action = IlmAction::DeleteRestoredAction;
-            if !obj.is_latest {
-                action = IlmAction::DeleteRestoredVersionAction;
-            }
-
-            events.push(Event {
-                action,
-                due: Some(now),
-                rule_id: "".into(),
-                noncurrent_days: 0,
-                newer_noncurrent_versions: 0,
-                storage_class: "".into(),
-            });
+        if let Some(event) = obj.restored_copy_expiry(now) {
+            events.push(event);
         }
 
         if let Some(ref lc_rules) = self.filter_rules(obj).await {
@@ -611,17 +617,12 @@ impl Lifecycle for BucketLifecycleConfiguration {
                     continue;
                 }
 
-                if !obj.is_latest
-                    && let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
-                    && let Some(retain_newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions
-                    && newer_noncurrent_versions < usize::try_from(retain_newer_noncurrent_versions).unwrap_or(usize::MAX)
-                {
-                    continue;
-                }
-
                 if !obj.is_latest
                     && let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
                     && let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
+                    && noncurrent_version_expiration
+                        .newer_noncurrent_versions
+                        .is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
                 {
                     if let Some(successor_mod_time) = obj.successor_mod_time {
                         let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
@@ -651,7 +652,11 @@ impl Lifecycle for BucketLifecycleConfiguration {
                     && let Some(noncurrent_version_transition) = rule
                         .noncurrent_version_transitions
                         .as_ref()
+                        .filter(|transitions| transitions.len() == 1)
                         .and_then(|transitions| transitions.first())
+                    && noncurrent_version_transition
+                        .newer_noncurrent_versions
+                        .is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
                     && let Some(storage_class) = noncurrent_version_transition.storage_class.as_ref()
                     && !storage_class.as_str().is_empty()
                     && !obj.delete_marker
@@ -735,7 +740,11 @@ impl Lifecycle for BucketLifecycleConfiguration {
                     }
 
                     if obj.transition_status != TRANSITION_COMPLETE
-                        && let Some(transition) = rule.transitions.as_ref().and_then(|transitions| transitions.first())
+                        && let Some(transition) = rule
+                            .transitions
+                            .as_ref()
+                            .filter(|transitions| transitions.len() == 1)
+                            .and_then(|transitions| transitions.first())
                         && let Some(storage_class) = transition.storage_class.as_ref()
                         && !storage_class.as_str().is_empty()
                     {
@@ -758,18 +767,15 @@ impl Lifecycle for BucketLifecycleConfiguration {
         }
 
         if !events.is_empty() {
-            // Select the winning event using a strict total order (MinIO semantics):
-            // the earliest `due` wins, and ties break toward delete-type actions. A
-            // missing `due` is treated as UNIX_EPOCH. This replaces a hand-written
-            // `sort_by` comparator that was not a strict weak ordering (it could return
-            // `Ordering::Less` for both `(a, b)` and `(b, a)`), which panics on the
-            // repository toolchain and did not deterministically pick the earliest event.
+            // Eligible expiration takes precedence over transition, even when a
+            // failed transition has an earlier deadline. Within each action class,
+            // prefer the earliest deadline using a deterministic total order.
             let event = events
                 .iter()
                 .min_by_key(|event| {
                     (
-                        event.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp(),
                         ilm_action_priority_rank(&event.action),
+                        event.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp(),
                     )
                 })
                 .cloned()
@@ -1042,6 +1048,27 @@ impl ObjectOpts {
     pub fn expired_object_deletemarker(&self) -> bool {
         self.delete_marker && self.is_latest && self.num_versions == 1
     }
+
+    pub(crate) fn restored_copy_expiry(&self, now: OffsetDateTime) -> Option<Event> {
+        let restore_expires = self.restore_expires?;
+        // Restore metadata alone does not prove that a durable remote copy exists.
+        if self.transition_status != TRANSITION_COMPLETE
+            || restore_expires.unix_timestamp() == 0
+            || now.unix_timestamp() <= restore_expires.unix_timestamp()
+        {
+            return None;
+        }
+        let action = if self.is_latest {
+            IlmAction::DeleteRestoredAction
+        } else {
+            IlmAction::DeleteRestoredVersionAction
+        };
+        expiration_action_has_valid_target(action, self.version_id, self.is_latest, self.delete_marker).then(|| Event {
+            action,
+            due: Some(now),
+            ..Default::default()
+        })
+    }
 }
 
 /// Returns whether an expiry action has enough identity to target the object
@@ -1064,11 +1091,8 @@ pub fn expiration_action_has_valid_target(
     }
 }
 
-/// Total-order rank for lifecycle actions used to break `due` ties.
-///
-/// Delete-type actions rank before every other action so that, when two events
-/// share the same `due`, a delete wins (MinIO semantics). The concrete numeric
-/// values only matter relative to each other.
+/// Eligible logical expiration takes precedence over transition and restore-copy
+/// cleanup. Deadlines break ties within an action class.
 fn ilm_action_priority_rank(action: &IlmAction) -> u8 {
     match action {
         IlmAction::DeleteAllVersionsAction
@@ -4159,6 +4183,392 @@ mod tests {
         assert_eq!(event.action, IlmAction::NoneAction);
     }
 
+    mod adversarial_regressions {
+        use super::*;
+        use s3s::dto::NoncurrentVersionExpiration;
+
+        fn run(test: impl std::future::Future<Output = ()>) {
+            with_default_ilm_process_time(|| {
+                tokio::runtime::Builder::new_current_thread()
+                    .build()
+                    .expect("lifecycle regression runtime should build")
+                    .block_on(test);
+            });
+        }
+
+        fn noncurrent_object() -> ObjectOpts {
+            ObjectOpts {
+                name: "logs/object".to_string(),
+                mod_time: Some(datetime!(2020-01-01 00:00:00 UTC)),
+                successor_mod_time: Some(datetime!(2020-01-02 00:00:00 UTC)),
+                version_id: Some(Uuid::from_u128(1)),
+                size: 1024 * 1024,
+                ..Default::default()
+            }
+        }
+
+        #[test]
+        #[serial]
+        fn noncurrent_transition_retains_the_requested_newer_versions() {
+            run(async {
+                let mut rule = enabled_rule(None, None, Some("retain-two-hot-versions"));
+                rule.filter = Some(LifecycleRuleFilter::default());
+                rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
+                    noncurrent_days: Some(1),
+                    newer_noncurrent_versions: Some(2),
+                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                }]);
+                let lc = Arc::new(BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                });
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("valid noncurrent transition policy");
+                let objects = (0..4)
+                    .map(|index| ObjectOpts {
+                        mod_time: Some(datetime!(2020-01-05 00:00:00 UTC) - Duration::days(index)),
+                        successor_mod_time: (index > 0).then_some(datetime!(2020-01-06 00:00:00 UTC) - Duration::days(index)),
+                        version_id: Some(Uuid::from_u128(u128::try_from(index + 1).expect("small version index"))),
+                        is_latest: index == 0,
+                        num_versions: 4,
+                        ..noncurrent_object()
+                    })
+                    .collect::<Vec<_>>();
+                let actions = crate::Evaluator::new(lc)
+                    .eval(&objects)
+                    .await
+                    .expect("complete version chain should evaluate")
+                    .into_iter()
+                    .map(|event| event.action)
+                    .collect::<Vec<_>>();
+                assert_eq!(
+                    actions,
+                    [
+                        IlmAction::NoneAction,
+                        IlmAction::NoneAction,
+                        IlmAction::NoneAction,
+                        IlmAction::TransitionVersionAction
+                    ],
+                    "the two newest noncurrent versions must remain in their current storage class"
+                );
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn noncurrent_transition_checks_count_age_and_single_object_context() {
+            run(async {
+                let mut rule = enabled_rule(None, None, Some("retain-two"));
+                rule.filter = Some(LifecycleRuleFilter::default());
+                rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
+                    noncurrent_days: Some(3),
+                    newer_noncurrent_versions: Some(2),
+                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                }]);
+                let mut lc = BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                };
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("valid counted transition");
+                let object = noncurrent_object();
+                let now = datetime!(2020-01-10 00:00:00 UTC);
+                for (newer, expected) in [
+                    (0, IlmAction::NoneAction),
+                    (1, IlmAction::NoneAction),
+                    (2, IlmAction::TransitionVersionAction),
+                    (3, IlmAction::TransitionVersionAction),
+                ] {
+                    assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}");
+                }
+                assert_eq!(
+                    lc.eval_inner(&object, datetime!(2020-01-04 00:00:00 UTC), 2).await.action,
+                    IlmAction::NoneAction,
+                    "the retention count does not replace the age condition"
+                );
+                assert_eq!(
+                    lc.eval(&object).await.action,
+                    IlmAction::NoneAction,
+                    "a single-object lookup must not assume a complete version history"
+                );
+                for retain in [None, Some(0), Some(-1), Some(i32::MAX)] {
+                    lc.rules[0]
+                        .noncurrent_version_transitions
+                        .as_mut()
+                        .expect("transition exists")[0]
+                        .newer_noncurrent_versions = retain;
+                    let expected = if matches!(retain, None | Some(0)) {
+                        IlmAction::TransitionVersionAction
+                    } else {
+                        IlmAction::NoneAction
+                    };
+                    assert_eq!(lc.eval_inner(&object, now, 2).await.action, expected, "retention: {retain:?}");
+                }
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn noncurrent_expiration_and_transition_have_independent_retention_counts() {
+            run(async {
+                let mut rule = enabled_rule(None, None, Some("independent-counts"));
+                rule.filter = Some(LifecycleRuleFilter::default());
+                rule.noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
+                    noncurrent_days: Some(90),
+                    newer_noncurrent_versions: Some(4),
+                });
+                rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
+                    noncurrent_days: Some(30),
+                    newer_noncurrent_versions: Some(2),
+                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                }]);
+                let lc = BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                };
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("valid independent retention limits");
+                let object = noncurrent_object();
+                let now = datetime!(2020-05-01 00:00:00 UTC);
+                for (newer, expected) in [
+                    (1, IlmAction::NoneAction),
+                    (2, IlmAction::TransitionVersionAction),
+                    (3, IlmAction::TransitionVersionAction),
+                    (4, IlmAction::DeleteVersionAction),
+                ] {
+                    assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}");
+                }
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn expiration_retention_does_not_skip_an_independent_transition() {
+            run(async {
+                let mut rule = enabled_rule(None, None, Some("transition-then-expire"));
+                rule.filter = Some(LifecycleRuleFilter::default());
+                rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
+                    noncurrent_days: Some(1),
+                    newer_noncurrent_versions: None,
+                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                }]);
+                let mut lc = BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                };
+                let object = noncurrent_object();
+                let now = datetime!(2020-01-10 00:00:00 UTC);
+                let transition_only = lc.eval_inner(&object, now, 0).await;
+                assert_eq!(transition_only.action, IlmAction::TransitionVersionAction);
+
+                lc.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
+                    noncurrent_days: Some(90),
+                    newer_noncurrent_versions: Some(2),
+                });
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("valid combined policy");
+                let combined = lc.eval_inner(&object, now, 0).await;
+                assert_eq!(combined.action, transition_only.action, "retention limits expiration, not transition");
+                assert_eq!(combined.storage_class, transition_only.storage_class);
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn current_transition_rejects_multiple_stages_in_any_order() {
+            run(async {
+                let mut rule = enabled_rule(None, None, Some("two-current-transitions"));
+                rule.transitions = Some(vec![
+                    Transition {
+                        date: Some(datetime!(2020-03-01 00:00:00 UTC).into()),
+                        days: None,
+                        storage_class: Some(TransitionStorageClass::from_static("COLD")),
+                    },
+                    Transition {
+                        date: Some(datetime!(2020-01-03 00:00:00 UTC).into()),
+                        days: None,
+                        storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                    },
+                ]);
+                let mut lc = BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                };
+                let object = ObjectOpts {
+                    is_latest: true,
+                    ..noncurrent_object()
+                };
+                let now = datetime!(2020-01-10 00:00:00 UTC);
+                for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] {
+                    lc.rules[0].status = ExpirationStatus::from_static(status);
+                    for _ in 0..2 {
+                        let err = lc
+                            .validate(&ObjectLockConfiguration::default())
+                            .await
+                            .expect_err("multiple transition stages must be rejected");
+                        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+                        assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_TRANSITIONS);
+                        assert_eq!(
+                            lc.eval_inner(&object, now, 0).await.action,
+                            IlmAction::NoneAction,
+                            "legacy multi-stage configurations must not silently execute their first stage"
+                        );
+                        lc.rules[0]
+                            .transitions
+                            .as_mut()
+                            .expect("transition array is present")
+                            .reverse();
+                    }
+                }
+                lc.rules[0]
+                    .transitions
+                    .as_mut()
+                    .expect("transition array is present")
+                    .remove(0);
+                lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED);
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("one stage is supported");
+                let event = lc.eval_inner(&object, now, 0).await;
+                assert_eq!(event.action, IlmAction::TransitionAction);
+                assert_eq!(event.storage_class, "WARM");
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn noncurrent_transition_rejects_multiple_stages_in_any_order() {
+            run(async {
+                let mut rule = enabled_rule(None, None, Some("two-noncurrent-transitions"));
+                rule.noncurrent_version_transitions = Some(vec![
+                    NoncurrentVersionTransition {
+                        noncurrent_days: Some(30),
+                        newer_noncurrent_versions: None,
+                        storage_class: Some(TransitionStorageClass::from_static("COLD")),
+                    },
+                    NoncurrentVersionTransition {
+                        noncurrent_days: Some(1),
+                        newer_noncurrent_versions: None,
+                        storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                    },
+                ]);
+                let mut lc = BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                };
+                let object = noncurrent_object();
+                let now = datetime!(2020-01-10 00:00:00 UTC);
+                for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] {
+                    lc.rules[0].status = ExpirationStatus::from_static(status);
+                    for _ in 0..2 {
+                        let err = lc
+                            .validate(&ObjectLockConfiguration::default())
+                            .await
+                            .expect_err("multiple noncurrent transition stages must be rejected");
+                        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+                        assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS);
+                        assert_eq!(
+                            lc.eval_inner(&object, now, 0).await.action,
+                            IlmAction::NoneAction,
+                            "legacy multi-stage configurations must not silently execute their first stage"
+                        );
+                        lc.rules[0]
+                            .noncurrent_version_transitions
+                            .as_mut()
+                            .expect("transition array is present")
+                            .reverse();
+                    }
+                }
+                lc.rules[0]
+                    .noncurrent_version_transitions
+                    .as_mut()
+                    .expect("transition array is present")
+                    .remove(0);
+                lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED);
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("one stage is supported");
+                let event = lc.eval_inner(&object, now, 0).await;
+                assert_eq!(event.action, IlmAction::TransitionVersionAction);
+                assert_eq!(event.storage_class, "WARM");
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn expiration_rejects_simultaneous_days_and_date() {
+            run(async {
+                let mut lc = BucketLifecycleConfiguration {
+                    rules: vec![enabled_rule(
+                        Some(LifecycleExpiration {
+                            days: Some(1),
+                            ..Default::default()
+                        }),
+                        None,
+                        Some("ambiguous-expiry"),
+                    )],
+                    expiry_updated_at: None,
+                };
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("a single Days expiration is valid");
+                lc.rules[0].expiration.as_mut().expect("expiration is present").date =
+                    Some(datetime!(2099-01-01 00:00:00 UTC).into());
+                let err = lc
+                    .validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect_err("Days and Date are mutually exclusive; accepting both silently overrides Days");
+                assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+                assert_eq!(err.to_string(), ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT);
+            });
+        }
+
+        #[test]
+        #[serial]
+        fn overdue_transition_does_not_starve_permanent_expiration() {
+            run(async {
+                let mut rule = enabled_rule(
+                    Some(LifecycleExpiration {
+                        days: Some(90),
+                        ..Default::default()
+                    }),
+                    None,
+                    Some("archive-then-delete"),
+                );
+                rule.transitions = Some(vec![Transition {
+                    days: Some(30),
+                    date: None,
+                    storage_class: Some(TransitionStorageClass::from_static("WARM")),
+                }]);
+                let lc = BucketLifecycleConfiguration {
+                    rules: vec![rule],
+                    expiry_updated_at: None,
+                };
+                lc.validate(&ObjectLockConfiguration::default())
+                    .await
+                    .expect("valid transition and expiration policy");
+                let object = ObjectOpts {
+                    is_latest: true,
+                    version_id: None,
+                    transition_status: TRANSITION_PENDING.to_string(),
+                    ..noncurrent_object()
+                };
+                let before_expiration = lc.eval_inner(&object, datetime!(2020-02-15 00:00:00 UTC), 0).await;
+                assert_eq!(before_expiration.action, IlmAction::TransitionAction);
+                let overdue = lc.eval_inner(&object, datetime!(2020-05-01 00:00:00 UTC), 0).await;
+                assert_eq!(
+                    overdue.action,
+                    IlmAction::DeleteAction,
+                    "an unavailable tier must not prevent permanent expiration indefinitely"
+                );
+            });
+        }
+    }
+
     /// Property-based tests for the rule evaluator (backlog#1148 ilm-14,
     /// follow-up to backlog#1030 / rustfs#4455).
     ///
@@ -4169,7 +4579,7 @@ mod tests {
     ///
     /// * `eval_inner` never panics and is deterministic for a fixed input;
     /// * the winning event matches an independently recomputed candidate set:
-    ///   earliest `due` wins, ties break toward delete-class actions (the
+    ///   eligible expiration wins over transition, then earliest `due` wins (the
     ///   `min_by_key` selection that replaced the rustfs#4455 comparator);
     /// * `expected_expiry_time` is monotonically non-decreasing in `days` and
     ///   always lands on the processing boundary, both at production defaults
@@ -4458,8 +4868,8 @@ mod tests {
         /// consider for a live current version under `selection`-shaped rules
         /// (expiration and first-transition only, no filters): expiration
         /// fires when `now >= due`, transition when `now > due` and the object
-        /// has not already transitioned. Selection semantics under test:
-        /// earliest due wins, ties prefer delete-class.
+        /// has not already transitioned. Eligible expiration wins over transition;
+        /// the earliest deadline wins within the selected action class.
         fn oracle_candidates(lc: &BucketLifecycleConfiguration, obj: &ObjectOpts, now: OffsetDateTime) -> Vec<Candidate> {
             let mod_time = obj.mod_time.expect("selection strategy always sets mod_time");
             let mut candidates = Vec::new();
@@ -4548,8 +4958,8 @@ mod tests {
             /// Differential test of winner selection (the rustfs#4455 fix):
             /// for a live current version under randomized expiration and
             /// transition rules, `eval_inner`'s winner must carry the
-            /// minimum `(due, rank)` of the independently recomputed
-            /// candidate set — earliest due wins, ties prefer delete-class —
+            /// earliest expiration from the independently recomputed candidate
+            /// set, or the earliest transition when no expiration is eligible,
             /// and must be `NoneAction` exactly when that set is empty.
             #[test]
             #[serial]
@@ -4578,7 +4988,13 @@ mod tests {
 
                 // Oracle and evaluator must observe the same (pinned) time env.
                 let (event, expected) = with_production_time_env(|| {
-                    let expected = oracle_candidates(&lc, &obj, now).into_iter().min();
+                    let candidates = oracle_candidates(&lc, &obj, now);
+                    let expected = candidates
+                        .iter()
+                        .filter(|(_, rank)| *rank == 0)
+                        .min()
+                        .copied()
+                        .or_else(|| candidates.into_iter().min());
                     let rt = tokio::runtime::Builder::new_current_thread()
                         .enable_all()
                         .build()

crates/lifecycle/src/evaluator.rs

@@ -116,13 +116,10 @@ impl Evaluator {
                             break 'top_loop;
                         }
                     }
-                    IlmAction::DeleteAction
-                    | IlmAction::DeleteRestoredAction
-                    | IlmAction::DeleteVersionAction
-                    | IlmAction::DeleteRestoredVersionAction
-                        if self.is_object_locked(obj) =>
-                    {
-                        event = Event::default();
+                    // Restore expiry removes only the temporary local copy; the
+                    // retained logical version and its remote data remain intact.
+                    IlmAction::DeleteAction | IlmAction::DeleteVersionAction if self.is_object_locked(obj) => {
+                        event = obj.restored_copy_expiry(now).unwrap_or_default();
                     }
                     _ => {}
                 }
@@ -206,6 +203,95 @@ mod tests {
 
     use super::*;
     use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType};
+
+    #[tokio::test]
+    async fn adversarial_restore_expiry_survives_legal_hold() {
+        let mut policy = (*latest_expiration_lifecycle()).clone();
+        policy.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::DISABLED);
+        let policy = Arc::new(policy);
+        policy
+            .validate(&lock_enabled_without_default_retention())
+            .await
+            .expect("valid disabled lifecycle rule");
+        let mut objects = [true, false].map(|is_latest| ObjectOpts {
+            is_latest,
+            num_versions: 2,
+            mod_time: Some(
+                OffsetDateTime::from_unix_timestamp(if is_latest { 1_200_000 } else { 1_000_000 })
+                    .expect("fixed version timestamp"),
+            ),
+            successor_mod_time: (!is_latest)
+                .then(|| OffsetDateTime::from_unix_timestamp(1_200_000).expect("fixed successor timestamp")),
+            transition_status: crate::TRANSITION_COMPLETE.to_string(),
+            restore_expires: Some(OffsetDateTime::from_unix_timestamp(2_000_000).expect("fixed expired restore timestamp")),
+            ..current_object_opts(ReplicationStatusType::Completed)
+        });
+        let evaluator = Evaluator::new(policy).with_lock_retention(Some(lock_enabled_without_default_retention()));
+        let expected = [IlmAction::DeleteRestoredAction, IlmAction::DeleteRestoredVersionAction];
+        let unlocked = evaluator
+            .eval(&objects)
+            .await
+            .expect("unlocked restored versions should evaluate");
+        assert_eq!(unlocked.iter().map(|event| event.action).collect::<Vec<_>>(), expected);
+
+        for object in &mut objects {
+            object
+                .user_defined
+                .insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string());
+        }
+        let locked = evaluator
+            .eval(&objects)
+            .await
+            .expect("locked restored versions should evaluate");
+        assert_eq!(
+            locked.iter().map(|event| event.action).collect::<Vec<_>>(),
+            expected,
+            "expiring a restored local copy preserves the retained logical version and remote object"
+        );
+
+        let mut expiring_policy = (*latest_expiration_lifecycle()).clone();
+        expiring_policy.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
+            noncurrent_days: Some(1),
+            newer_noncurrent_versions: None,
+        });
+        let expiring_evaluator =
+            Evaluator::new(Arc::new(expiring_policy)).with_lock_retention(Some(lock_enabled_without_default_retention()));
+        let locked = expiring_evaluator
+            .eval(&objects)
+            .await
+            .expect("locked expired versions should evaluate");
+        assert_eq!(
+            locked.iter().map(|event| event.action).collect::<Vec<_>>(),
+            expected,
+            "blocked logical expiration must still allow an eligible restore-copy cleanup"
+        );
+
+        for status in [ReplicationStatusType::Pending, ReplicationStatusType::Failed] {
+            for object in &mut objects {
+                object.replication_status = status.clone();
+            }
+            for evaluator in [&evaluator, &expiring_evaluator] {
+                let events = evaluator.eval(&objects).await.expect("pending replication should evaluate");
+                assert!(events.iter().all(|event| event.action == IlmAction::NoneAction));
+            }
+        }
+        for object in &mut objects {
+            object.replication_status = ReplicationStatusType::Completed;
+        }
+        for transition_status in ["", crate::TRANSITION_PENDING, "unknown"] {
+            for object in &mut objects {
+                object.transition_status = transition_status.to_string();
+            }
+            for evaluator in [&evaluator, &expiring_evaluator] {
+                let events = evaluator.eval(&objects).await.expect("incomplete transition should evaluate");
+                assert!(
+                    events.iter().all(|event| event.action == IlmAction::NoneAction),
+                    "restore metadata cannot authorize cleanup without a completed transition"
+                );
+            }
+        }
+    }
+
     fn expired_marker_lifecycle() -> Arc<BucketLifecycleConfiguration> {
         Arc::new(BucketLifecycleConfiguration {
             expiry_updated_at: None,

docs/operations/tier-ilm-debugging.md

@@ -22,6 +22,16 @@
 | `FileMeta` / `FileInfo` / version metadata | `crates/filemeta/src/` |
 | Dual-key internal metadata helpers (`insert_bytes` / `get_bytes`) | `crates/utils/src/http/metadata_compat.rs` |
 
+## Lifecycle rule limits and evaluation
+
+Each lifecycle rule supports at most one `Transition` and one `NoncurrentVersionTransition`. A version can make one initial transition; chaining additional tiers after it reaches `complete` is not supported. Splitting stages across overlapping rules does not enable a transition chain. `PutBucketLifecycleConfiguration` rejects multiple entries in either transition array with `InvalidArgument`, including in disabled rules. Existing stored multi-entry arrays are not executed; replace each with a single intended destination. Independent expiration actions in the rule remain eligible.
+
+`Expiration.Days` and `Expiration.Date` are mutually exclusive. A request containing both is rejected instead of silently selecting the date. When expiration and transition are both eligible, expiration takes precedence; a failed earlier transition does not keep an expired object indefinitely. Deadlines select the earliest action within the same action class.
+
+Noncurrent expiration and transition have independent `NewerNoncurrentVersions` limits. A transition with a positive limit waits for a complete version-group evaluation to establish that enough newer noncurrent versions remain. Single-object evaluation, including the current manual transition and immediate-enqueue paths, conservatively defers these counted transitions to the lifecycle scanner. An unmet expiration retention limit does not suppress a separately eligible transition.
+
+An expired restored local copy can be cleaned up under Object Lock because the retained logical version and remote data remain intact. Cleanup requires a completed transition and still waits for pending or failed replication. The storage layer revalidates the source identity and restore metadata before removing the local copy; restore headers alone do not authorize cleanup.
+
 ## Free-version recovery controls
 
 The dedicated free-version recovery loop is enabled by default and is independent of the data scanner and heal switches. Setting `RUSTFS_SCANNER_ENABLED=false` does not stop this repair loop. Set `RUSTFS_TIER_FREE_VERSION_RECOVERY_ENABLED=false` before process startup to disable only the dedicated persisted-marker walk. That setting does not disable lifecycle workers or prevent another scanner path from discovering a free version, and it can leave remote cleanup markers pending for longer, so use it as a break-glass pressure control rather than a cleanup mechanism.

scripts/test_nightly_candidate.py

@@ -1,208 +0,0 @@
-#!/usr/bin/env python3
-"""Exercise the nightly publication step without AWS, network or package builds."""
-
-import hashlib
-import json
-import os
-from pathlib import Path
-import subprocess
-import tempfile
-import unittest
-
-from check_test_wiring import yaml_block
-
-
-ROOT = Path(__file__).resolve().parents[1]
-WORKFLOW = ROOT / ".github/workflows/nightly-gnu.yml"
-
-
-class NightlyCandidateTests(unittest.TestCase):
-    def setUp(self):
-        self.temp = tempfile.TemporaryDirectory()
-        self.addCleanup(self.temp.cleanup)
-        self.root = Path(self.temp.name)
-        self.package = self.root / "rustfs-nightly-2026-09-06.deb"
-        self.package.write_bytes(b"built package bytes\x00\xff")
-        for command in (["git", "init", "-q"], ["git", "add", self.package.name],
-                        ["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "fixture"]):
-            subprocess.run(command, cwd=self.root, check=True, capture_output=True)
-        self.sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=self.root, text=True).strip()
-        self.digest = hashlib.sha256(self.package.read_bytes()).hexdigest()
-        self.output = self.root / "github-output"
-        self.store = self.root / "store"
-        self.shims = self.root / "fake-tools.sh"
-        self.shims.write_text(r'''aws() {
-  printf '%s\n' "$*" >> "$FAKE_AWS_LOG"
-  if [[ "$1" == --version ]]; then printf 'aws-cli/1.44.79 fixture\n'; return; fi
-  if [[ "$*" == *--generate-cli-skeleton* ]]; then
-    if [[ "$FAKE_MODE" == broken-install || "$FAKE_MODE" =~ ^(old-cli|bootstrap-failure|install-failure)$ && ! -e "$FAKE_INSTALLED" ]]; then
-      printf '{}\n'
-    else
-      printf '{"IfNoneMatch":""}\n'
-    fi
-    return
-  fi
-  if [[ "$1 $2" == 's3api put-object' ]]; then
-    [[ "$FAKE_MODE" != upload-failure ]] || return 42
-    shift 2
-    local key="" body="" condition=""
-    while [[ $# -gt 0 ]]; do
-      case "$1" in
-        --key) key="$2";;
-        --body) body="$2";;
-        --if-none-match) condition="$2";;
-      esac
-      shift 2
-    done
-    [[ -z "$condition" || "$condition" == '*' ]] || return 43
-    if [[ "$condition" == '*' && -e "$FAKE_STORE/$key" ]]; then return 44; fi
-    mkdir -p "$(dirname "$FAKE_STORE/$key")"
-    cp "$body" "$FAKE_STORE/$key"
-  elif [[ "$1 $2" == 's3 cp' ]]; then
-    [[ "$FAKE_MODE" != alias-failure ]] || return 45
-    local destination="${4#s3://test-bucket/}"
-    [[ "$destination" != */ ]] || destination+="$(basename "$3")"
-    mkdir -p "$(dirname "$FAKE_STORE/$destination")"
-    cp "$3" "$FAKE_STORE/$destination"
-  else
-    return 46
-  fi
-}
-curl() {
-  local url="${!#}"
-  printf '%s\n' "$url" >> "$FAKE_CURL_LOG"
-  [[ "$url" == https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/* ]] || return 22
-  [[ "$FAKE_MODE" != missing-public-url ]] || return 22
-  if [[ "$FAKE_MODE" == wrong-public-bytes ]]; then printf 'different package'; return; fi
-  cat "$FAKE_STORE/${url#https://dl.rustfs.com/}" || return 22
-  [[ "$FAKE_MODE" != incomplete-download ]] || return 47
-}
-sudo() {
-  [[ "$*" == 'apt-get update' || "$*" == 'apt-get install -y -qq python3-venv' ]] || return 49
-  [[ "$FAKE_MODE" != bootstrap-failure ]] || return 48
-}
-python3() {
-  [[ "$1 $2" == '-m venv' ]] || return 50
-  mkdir -p "$3/bin"
-  cat > "$3/bin/python" <<'SH'
-#!/usr/bin/env bash
-[[ "$*" == '-m pip install --disable-pip-version-check awscli==1.44.79' ]] || exit 51
-[[ "$FAKE_MODE" != install-failure ]] || exit 52
-: > "$FAKE_INSTALLED"
-SH
-  printf '#!/usr/bin/env bash\naws "$@"\n' > "$3/bin/aws"
-  chmod +x "$3/bin/python" "$3/bin/aws"
-}
-''')
-        self.env = dict(os.environ, BASH_ENV=str(self.shims), DEB_FILE=self.package.name,
-                        R2_ACCESS_KEY_ID="fake-access", R2_SECRET_ACCESS_KEY="fake-secret", R2_ENDPOINT="https://r2.example.invalid", R2_BUCKET="test-bucket",
-                        RUNNER_TEMP=str(self.root), GITHUB_SHA=self.sha, GITHUB_RUN_ID="12345", GITHUB_RUN_ATTEMPT="1", GITHUB_OUTPUT=str(self.output),
-                        FAKE_STORE=str(self.store), FAKE_AWS_LOG=str(self.root / "aws.log"), FAKE_CURL_LOG=str(self.root / "curl.log"), FAKE_INSTALLED=str(self.root / "installed"), FAKE_MODE="success")
-        source = WORKFLOW.read_text()
-        job = yaml_block(source.splitlines(), "build", 2)
-        starts = [i for i, line in enumerate(job) if line.startswith("      - name: ")]
-        self.steps = {
-            job[start].split(": ", 1)[1]: job[start:end]
-            for start, end in zip(starts, starts[1:] + [len(job)])
-        }
-        self.publish = self.steps["Upload DEB to Cloudflare R2"]
-        start = self.publish.index("        run: |") + 1
-        self.shell = "\n".join(line[10:] for line in self.publish[start:] if not line.strip() or line.startswith("          "))
-
-    def run_publish(self, **overrides):
-        self.output.unlink(missing_ok=True)
-        return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.shell],
-                              cwd=self.root, env=dict(self.env, **overrides), capture_output=True, text=True)
-
-    def manifest(self):
-        output = self.output.read_text().strip()
-        self.assertTrue(output.startswith("candidate_file="), output)
-        return json.loads(Path(output.split("=", 1)[1]).read_text())
-
-    def test_success_binds_actual_package_checkout_and_attempt(self):
-        result = self.run_publish()
-        self.assertEqual(result.returncode, 0, result.stderr)
-        manifest = self.manifest()
-        self.assertEqual(manifest, {"schema": 1, "source_sha": self.sha, "build_run_id": 12345, "build_run_attempt": 1,
-                                   "package_sha256": self.digest, "package_url": f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/12345/1/{self.digest}/rustfs.deb"})
-        for path in (f"runs/12345/1/{self.digest}/rustfs.deb", self.package.name, "rustfs-nightly-latest.deb"):
-            self.assertEqual((self.store / "artifacts/rustfs/packages/nightly" / path).read_bytes(), self.package.read_bytes())
-        self.assertEqual((self.root / "curl.log").read_text().strip(), manifest["package_url"])
-
-    def test_missing_credentials_remain_artifact_only(self):
-        for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"):
-            with self.subTest(missing=key):
-                result = self.run_publish(**{key: ""})
-                self.assertEqual(result.returncode, 0, result.stderr)
-                self.assertFalse(self.output.exists())
-                self.assertFalse((self.root / "aws.log").exists())
-                self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
-
-    def test_publication_failures_never_emit_a_candidate(self):
-        for index, mode in enumerate(("upload-failure", "missing-public-url", "wrong-public-bytes", "incomplete-download", "alias-failure")):
-            with self.subTest(mode=mode):
-                result = self.run_publish(FAKE_MODE=mode, GITHUB_RUN_ID=str(20000 + index))
-                self.assertNotEqual(result.returncode, 0, result.stdout)
-                self.assertFalse(self.output.exists())
-                self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
-
-    def test_old_cli_is_upgraded_in_an_isolated_temporary_environment(self):
-        result = self.run_publish(FAKE_MODE="old-cli")
-        self.assertEqual(result.returncode, 0, result.stderr)
-        self.assertTrue((self.root / "installed").exists())
-        self.assertEqual(self.manifest()["package_sha256"], self.digest)
-        self.assertEqual(list(self.root.glob("nightly-awscli.*")), [])
-
-    def test_failed_cli_bootstrap_cannot_publish(self):
-        for mode in ("bootstrap-failure", "install-failure", "broken-install"):
-            with self.subTest(mode=mode):
-                (self.root / "installed").unlink(missing_ok=True)
-                result = self.run_publish(FAKE_MODE=mode)
-                self.assertNotEqual(result.returncode, 0)
-                self.assertFalse(self.output.exists())
-                self.assertFalse(self.store.exists())
-                self.assertEqual(list(self.root.glob("nightly-awscli.*")), [])
-
-    def test_checkout_sha_mismatch_fails_before_upload(self):
-        result = self.run_publish(GITHUB_SHA="f" * 40)
-        self.assertNotEqual(result.returncode, 0)
-        self.assertIn("Checkout SHA", result.stderr)
-        self.assertFalse(self.output.exists())
-        self.assertFalse((self.root / "aws.log").exists())
-
-    def test_same_date_builds_and_reruns_keep_distinct_candidates(self):
-        urls = []
-        for run, attempt in (("12345", "1"), ("54321", "1"), ("12345", "2")):
-            result = self.run_publish(GITHUB_RUN_ID=run, GITHUB_RUN_ATTEMPT=attempt)
-            self.assertEqual(result.returncode, 0, result.stderr)
-            urls.append(self.manifest()["package_url"])
-        self.assertEqual(len(set(urls)), 3)
-        self.assertEqual(len(list(self.root.glob("nightly-candidate-*.json"))), 3)
-
-    def test_duplicate_key_is_not_overwritten_or_recertified(self):
-        result = self.run_publish()
-        self.assertEqual(result.returncode, 0, result.stderr)
-        key = self.manifest()["package_url"].removeprefix("https://dl.rustfs.com/")
-        stored = self.store / key
-        stored.write_bytes(b"preexisting conflicting object")
-        (self.root / "nightly-candidate-12345-1.json").unlink()
-        result = self.run_publish()
-        self.assertNotEqual(result.returncode, 0)
-        self.assertEqual(stored.read_bytes(), b"preexisting conflicting object")
-        self.assertFalse(self.output.exists())
-        self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
-
-    def test_manifest_upload_requires_publication_output(self):
-        upload = self.steps["Upload nightly candidate manifest"]
-        self.assertIn("        id: publish", self.publish)
-        self.assertIn("          DEB_FILE: ${{ steps.deb.outputs.deb_file }}", self.publish)
-        self.assertIn("        if: ${{ steps.publish.outputs.candidate_file != '' }}", upload)
-        self.assertIn("          name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}", upload)
-        self.assertIn("          path: ${{ steps.publish.outputs.candidate_file }}", upload)
-        self.assertIn("          if-no-files-found: error", upload)
-        self.assertNotIn("        continue-on-error: true", self.publish)
-        self.assertNotIn("          overwrite: true", upload)
-
-
-if __name__ == "__main__":
-    unittest.main()