Merge branch 'main' into overtrue/fix-1911-ilm-metadata-decommission

# Conflicts:
#	crates/ecstore/src/core/pools.rs

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=e78234828a8893b8bb0b27f57135edaff5066123cd592c38256be117c350d9d0
-sha256-linux=8084e3e013286bdea17ee18fbd44f28ade8ad0fe2726fa644835278819a41082
+sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52
+sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6

.config/nextest.toml

@@ -78,6 +78,12 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'
 
+# The durable ILM decommission regressions build isolated multi-pool stores and
+# deliberately take source or target disks offline while checking fencing.
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the bucket-incarnation / lifecycle-fence tests. They drive
 # init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
 # OnceLock state that serial_test's #[serial] cannot protect across nextest's
@@ -190,6 +196,10 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'
 
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
 # too (see the matching default-profile override near the top). No retries.
 [[profile.ci.overrides]]
@@ -399,7 +409,7 @@ path = "junit.xml"
 #     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (55 slow) lanes and reserves
 #     it for those, so e2e-full does not double-run it.
 #   * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
-#     manual-localhost:9000 reliant tests are ci-13's migration.
+#     manual-localhost:9000 reliant/policy tests are ci-13's migration.
 #
 # Each e2e test spawns its own single-node rustfs server on a random port with
 # an isolated temp dir (crates/e2e_test/src/common.rs), so the set is

crates/e2e_test/README.md

@@ -72,7 +72,7 @@ The reason string on each attribute is the classifier. Current classes:
 
 - **Needs a pre-started server** — `"requires running RustFS server at
   localhost:9000"` / `"Connects to existing rustfs server"`. These are the
-  `reliant/*` tests; start a server first (e.g.
+  `reliant/*` and `policy/test_runner` tests; start a server first (e.g.
   [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh)) or use
   `--run-ignored`.
 - **Heavy / external tool** — `"Starts a rustfs server; enable when running

crates/e2e_test/src/policy/README.md

@@ -11,20 +11,29 @@ The tests cover the following AWS policy variable scenarios:
 3. **Variable concatenation** - Combining variables with static text like `prefix-${aws:username}-suffix`
 4. **Nested variables** - Complex nested variable patterns like `${${aws:username}-test}`
 5. **Deny scenarios** - Testing deny policies with variables
-6. **STS credentials** - Variable resolution inherited by temporary credentials
 
 ## Prerequisites
 
+- RustFS server binary
 - `awscurl` utility for admin API calls
 - AWS SDK for Rust (included in the project)
 
 ## Running Tests
 
+### Run All Policy Tests Using Unified Test Runner
+
+```bash
+# Run all policy tests with comprehensive reporting
+# Note: Requires a RustFS server running on localhost:9000
+cargo test -p e2e_test policy::test_runner::test_policy_full_suite -- --nocapture --ignored  --test-threads=1
+
+# Run only critical policy tests
+cargo test -p e2e_test policy::test_runner::test_policy_critical_suite -- --nocapture --ignored --test-threads=1
+```
+
 ### Run All Policy Tests
 
 ```bash
 # From the project root directory
-cargo test -p e2e_test policy:: -- --nocapture
-```
-
-Each test starts an isolated RustFS server on a dynamically allocated local port and cleans it up afterward.
+cargo test -p e2e_test policy:: -- --nocapture --ignored --test-threads=1
+```

crates/e2e_test/src/policy/mod.rs

@@ -18,3 +18,5 @@
 //! including single-value, multi-value, and nested variable scenarios.
 
 mod policy_variables_test;
+mod test_env;
+mod test_runner;

crates/e2e_test/src/policy/policy_variables_test.rs

@@ -14,17 +14,14 @@
 
 //! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios
 
-use crate::common::{
-    RustFSTestEnvironment, awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging,
-};
-use aws_sdk_s3::Client;
-use aws_sdk_s3::error::ProvideErrorMetadata;
+use crate::common::{awscurl_delete, awscurl_put, init_logging};
+use crate::policy::test_env::PolicyTestEnvironment;
 use aws_sdk_s3::primitives::ByteStream;
 use tracing::info;
 
 /// Helper function to create a regular user with given credentials
 async fn create_user(
-    env: &RustFSTestEnvironment,
+    env: &PolicyTestEnvironment,
     username: &str,
     password: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -39,9 +36,20 @@ async fn create_user(
     Ok(())
 }
 
+/// Helper function to create an STS user with given credentials
+async fn create_sts_user(
+    env: &PolicyTestEnvironment,
+    username: &str,
+    password: &str,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    // For STS, we create a regular user first, then use it to assume roles
+    create_user(env, username, password).await?;
+    Ok(())
+}
+
 /// Helper function to create and attach a policy
 async fn create_and_attach_policy(
-    env: &RustFSTestEnvironment,
+    env: &PolicyTestEnvironment,
     policy_name: &str,
     username: &str,
     policy_document: serde_json::Value,
@@ -62,9 +70,9 @@ async fn create_and_attach_policy(
 }
 
 /// Helper function to clean up test resources
-async fn cleanup_user_and_policy(env: &RustFSTestEnvironment, username: &str, policy_name: &str) {
+async fn cleanup_user_and_policy(env: &PolicyTestEnvironment, username: &str, policy_name: &str) {
     // Create admin client for cleanup
-    let admin_client = env.create_s3_client();
+    let admin_client = env.create_s3_client(&env.access_key, &env.secret_key);
 
     // Delete buckets that might have been created by this user
     let bucket_patterns = [
@@ -76,7 +84,7 @@ async fn cleanup_user_and_policy(env: &RustFSTestEnvironment, username: &str, po
         format!("{username}-test"),
         format!("{username}-sts-bucket"),
         format!("{username}-service-bucket"),
-        format!("{username}-private-bucket"),
+        "private-test-bucket".to_string(), // For deny test
     ];
 
     // Try to delete objects and buckets
@@ -113,18 +121,24 @@ async fn cleanup_user_and_policy(env: &RustFSTestEnvironment, username: &str, po
 
 /// Test AWS policy variables with single-value scenarios
 #[tokio::test(flavor = "multi_thread")]
+#[ignore = "Starts a rustfs server; enable when running full E2E"]
 pub async fn test_aws_policy_variables_single_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    test_aws_policy_variables_single_value_impl().await
+}
+
+/// Implementation function for single-value policy variables test
+pub async fn test_aws_policy_variables_single_value_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Starting AWS policy variables single-value test");
 
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
 
     test_aws_policy_variables_single_value_impl_with_env(&env).await
 }
 
-async fn test_aws_policy_variables_single_value_impl_with_env(
-    env: &RustFSTestEnvironment,
+/// Implementation function for single-value policy variables test with shared environment
+pub async fn test_aws_policy_variables_single_value_impl_with_env(
+    env: &PolicyTestEnvironment,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Create test user
     let test_user = "testuser1";
@@ -184,7 +198,9 @@ async fn test_aws_policy_variables_single_value_impl_with_env(
     awscurl_put(&attach_policy_url, "", &env.access_key, &env.secret_key).await?;
 
     // Create S3 client for test user
-    let test_client = env.create_s3_client_with_credentials(test_user, test_password);
+    let test_client = env.create_s3_client(test_user, test_password);
+
+    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
 
     // Test 1: User should be able to list buckets (allowed by policy)
     info!("Test 1: User listing buckets");
@@ -241,13 +257,11 @@ async fn test_aws_policy_variables_single_value_impl_with_env(
     // Test 6: User should NOT be able to create bucket NOT matching username pattern
     info!("Test 6: User attempting to create bucket NOT matching pattern");
     let other_bucket_name = "other-user-bucket";
-    let denied = test_client
-        .create_bucket()
-        .bucket(other_bucket_name)
-        .send()
-        .await
-        .expect_err("a bucket outside the username pattern must be denied");
-    assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+    let create_other_result = test_client.create_bucket().bucket(other_bucket_name).send().await;
+    if create_other_result.is_ok() {
+        cleanup().await;
+        return Err("User should NOT be able to create bucket NOT matching username pattern".into());
+    }
 
     // Cleanup
     info!("Cleaning up test resources");
@@ -259,18 +273,24 @@ async fn test_aws_policy_variables_single_value_impl_with_env(
 
 /// Test AWS policy variables with multi-value scenarios
 #[tokio::test(flavor = "multi_thread")]
+#[ignore = "Starts a rustfs server; enable when running full E2E"]
 pub async fn test_aws_policy_variables_multi_value() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    test_aws_policy_variables_multi_value_impl().await
+}
+
+/// Implementation function for multi-value policy variables test
+pub async fn test_aws_policy_variables_multi_value_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Starting AWS policy variables multi-value test");
 
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
 
     test_aws_policy_variables_multi_value_impl_with_env(&env).await
 }
 
-async fn test_aws_policy_variables_multi_value_impl_with_env(
-    env: &RustFSTestEnvironment,
+/// Implementation function for multi-value policy variables test with shared environment
+pub async fn test_aws_policy_variables_multi_value_impl_with_env(
+    env: &PolicyTestEnvironment,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Create test user
     let test_user = "testuser2";
@@ -318,7 +338,7 @@ async fn test_aws_policy_variables_multi_value_impl_with_env(
     create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
 
     // Create S3 client for test user
-    let test_client = env.create_s3_client_with_credentials(test_user, test_password);
+    let test_client = env.create_s3_client(test_user, test_password);
 
     // Test 1: User should be able to create buckets matching any of the multi-value patterns
     info!("Test 1: User creating first bucket matching multi-value pattern");
@@ -348,13 +368,11 @@ async fn test_aws_policy_variables_multi_value_impl_with_env(
     // Test 4: User should NOT be able to create bucket NOT matching any multi-value pattern
     info!("Test 4: User attempting to create bucket NOT matching any pattern");
     let other_bucket_name = format!("{test_user}-other-bucket");
-    let denied = test_client
-        .create_bucket()
-        .bucket(&other_bucket_name)
-        .send()
-        .await
-        .expect_err("a bucket outside all allowed patterns must be denied");
-    assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+    let create_other_result = test_client.create_bucket().bucket(&other_bucket_name).send().await;
+    if create_other_result.is_ok() {
+        cleanup().await;
+        return Err("User should NOT be able to create bucket NOT matching any multi-value pattern".into());
+    }
 
     // Test 5: User should be able to list objects in their allowed buckets
     info!("Test 5: User listing objects in allowed buckets");
@@ -380,18 +398,24 @@ async fn test_aws_policy_variables_multi_value_impl_with_env(
 
 /// Test AWS policy variables with variable concatenation
 #[tokio::test(flavor = "multi_thread")]
+#[ignore = "Starts a rustfs server; enable when running full E2E"]
 pub async fn test_aws_policy_variables_concatenation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    test_aws_policy_variables_concatenation_impl().await
+}
+
+/// Implementation function for concatenation policy variables test
+pub async fn test_aws_policy_variables_concatenation_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Starting AWS policy variables concatenation test");
 
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
 
     test_aws_policy_variables_concatenation_impl_with_env(&env).await
 }
 
-async fn test_aws_policy_variables_concatenation_impl_with_env(
-    env: &RustFSTestEnvironment,
+/// Implementation function for concatenation policy variables test with shared environment
+pub async fn test_aws_policy_variables_concatenation_impl_with_env(
+    env: &PolicyTestEnvironment,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Create test user
     let test_user = "testuser3";
@@ -431,7 +455,10 @@ async fn test_aws_policy_variables_concatenation_impl_with_env(
     create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
 
     // Create S3 client for test user
-    let test_client = env.create_s3_client_with_credentials(test_user, test_password);
+    let test_client = env.create_s3_client(test_user, test_password);
+
+    // Add a small delay to allow policy to propagate
+    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
 
     // Test: User should be able to create bucket matching concatenated pattern
     info!("Test: User creating bucket matching concatenated pattern");
@@ -460,30 +487,41 @@ async fn test_aws_policy_variables_concatenation_impl_with_env(
 
 /// Test AWS policy variables with nested scenarios
 #[tokio::test(flavor = "multi_thread")]
+#[ignore = "Starts a rustfs server; enable when running full E2E"]
 pub async fn test_aws_policy_variables_nested() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    test_aws_policy_variables_nested_impl().await
+}
+
+/// Implementation function for nested policy variables test
+pub async fn test_aws_policy_variables_nested_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Starting AWS policy variables nested test");
 
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
 
     test_aws_policy_variables_nested_impl_with_env(&env).await
 }
 
 /// Test AWS policy variables with STS temporary credentials
 #[tokio::test(flavor = "multi_thread")]
+#[ignore = "Starts a rustfs server; enable when running full E2E"]
 pub async fn test_aws_policy_variables_sts() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    test_aws_policy_variables_sts_impl().await
+}
+
+/// Implementation function for STS policy variables test
+pub async fn test_aws_policy_variables_sts_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Starting AWS policy variables STS test");
 
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
 
     test_aws_policy_variables_sts_impl_with_env(&env).await
 }
 
-async fn test_aws_policy_variables_nested_impl_with_env(
-    env: &RustFSTestEnvironment,
+/// Implementation function for nested policy variables test with shared environment
+pub async fn test_aws_policy_variables_nested_impl_with_env(
+    env: &PolicyTestEnvironment,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Create test user
     let test_user = "testuser4";
@@ -523,7 +561,10 @@ async fn test_aws_policy_variables_nested_impl_with_env(
     create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
 
     // Create S3 client for test user
-    let test_client = env.create_s3_client_with_credentials(test_user, test_password);
+    let test_client = env.create_s3_client(test_user, test_password);
+
+    // Add a small delay to allow policy to propagate
+    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
 
     // Test nested variable resolution
     info!("Test: Nested variable resolution");
@@ -540,14 +581,14 @@ async fn test_aws_policy_variables_nested_impl_with_env(
         return Err(format!("User should be able to create bucket with nested variable: {e}").into());
     }
 
-    // Verify a valid bucket name outside the resolved resource is denied.
-    let denied = test_client
-        .create_bucket()
-        .bucket("other-user-test")
-        .send()
-        .await
-        .expect_err("a bucket outside the resolved nested variable must be denied");
-    assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+    // Verify bucket creation fails with unresolved variable
+    let unresolved_bucket = format!("${{}}-test {test_user}");
+    let create_unresolved = test_client.create_bucket().bucket(&unresolved_bucket).send().await;
+
+    if create_unresolved.is_ok() {
+        cleanup().await;
+        return Err("User should NOT be able to create bucket with unresolved variable".into());
+    }
 
     // Cleanup
     info!("Cleaning up test resources");
@@ -557,8 +598,9 @@ async fn test_aws_policy_variables_nested_impl_with_env(
     Ok(())
 }
 
-async fn test_aws_policy_variables_sts_impl_with_env(
-    env: &RustFSTestEnvironment,
+/// Implementation function for STS policy variables test with shared environment
+pub async fn test_aws_policy_variables_sts_impl_with_env(
+    env: &PolicyTestEnvironment,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Create test user for STS
     let test_user = "testuser-sts";
@@ -570,7 +612,8 @@ async fn test_aws_policy_variables_sts_impl_with_env(
         cleanup_user_and_policy(env, test_user, policy_name).await;
     };
 
-    create_user(env, test_user, test_password).await?;
+    // Create STS user
+    create_sts_user(env, test_user, test_password).await?;
 
     // Create policy with STS-compatible variables
     let policy_document = serde_json::json!({
@@ -581,11 +624,6 @@ async fn test_aws_policy_variables_sts_impl_with_env(
                 "Action": ["s3:ListAllMyBuckets"],
                 "Resource": ["arn:aws:s3:::*"]
             },
-            {
-                "Effect": "Allow",
-                "Action": ["sts:AssumeRole"],
-                "Resource": ["arn:aws:s3:::*"]
-            },
             {
                 "Effect": "Allow",
                 "Action": ["s3:CreateBucket"],
@@ -593,12 +631,7 @@ async fn test_aws_policy_variables_sts_impl_with_env(
             },
             {
                 "Effect": "Allow",
-                "Action": ["s3:ListBucket"],
-                "Resource": [format!("arn:aws:s3:::{}-sts-bucket", "${aws:username}")]
-            },
-            {
-                "Effect": "Allow",
-                "Action": ["s3:PutObject", "s3:GetObject"],
+                "Action": ["s3:ListBucket", "s3:PutObject", "s3:GetObject"],
                 "Resource": [format!("arn:aws:s3:::{}-sts-bucket/*", "${aws:username}")]
             }
         ]
@@ -606,22 +639,11 @@ async fn test_aws_policy_variables_sts_impl_with_env(
 
     create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
 
-    let assumed = build_test_sts_client(&env.url, test_user, test_password, None, "policy-variable-sts")
-        .assume_role()
-        .role_arn("arn:aws:iam::123456789012:role/policy-variable")
-        .role_session_name("policy-variable-e2e")
-        .send()
-        .await?;
-    let credentials = assumed
-        .credentials()
-        .ok_or("AssumeRole response should contain temporary credentials")?;
-    let test_client = Client::from_conf(build_test_s3_config(
-        &env.url,
-        credentials.access_key_id(),
-        credentials.secret_access_key(),
-        Some(credentials.session_token()),
-        "policy-variable-sts-session",
-    ));
+    // Create S3 client for test user
+    let test_client = env.create_s3_client(test_user, test_password);
+
+    // Add a small delay to allow policy to propagate
+    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
 
     // Test: User should be able to create bucket matching STS pattern
     info!("Test: User creating bucket matching STS pattern");
@@ -677,18 +699,24 @@ async fn test_aws_policy_variables_sts_impl_with_env(
 
 /// Test AWS policy variables with deny scenarios
 #[tokio::test(flavor = "multi_thread")]
+#[ignore = "Starts a rustfs server; enable when running full E2E"]
 pub async fn test_aws_policy_variables_deny() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    test_aws_policy_variables_deny_impl().await
+}
+
+/// Implementation function for deny policy variables test
+pub async fn test_aws_policy_variables_deny_impl() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
     info!("Starting AWS policy variables deny test");
 
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    let env = PolicyTestEnvironment::with_address("127.0.0.1:9000").await?;
 
     test_aws_policy_variables_deny_impl_with_env(&env).await
 }
 
-async fn test_aws_policy_variables_deny_impl_with_env(
-    env: &RustFSTestEnvironment,
+/// Implementation function for deny policy variables test with shared environment
+pub async fn test_aws_policy_variables_deny_impl_with_env(
+    env: &PolicyTestEnvironment,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Create test user
     let test_user = "testuser5";
@@ -731,7 +759,10 @@ async fn test_aws_policy_variables_deny_impl_with_env(
     create_and_attach_policy(env, policy_name, test_user, policy_document).await?;
 
     // Create S3 client for test user
-    let test_client = env.create_s3_client_with_credentials(test_user, test_password);
+    let test_client = env.create_s3_client(test_user, test_password);
+
+    // Add a small delay to allow policy to propagate
+    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
 
     // Test 1: User should be able to create bucket matching username pattern
     info!("Test 1: User creating bucket matching username pattern");
@@ -744,14 +775,12 @@ async fn test_aws_policy_variables_deny_impl_with_env(
 
     // Test 2: User should NOT be able to create bucket with "private" in the name (deny rule)
     info!("Test 2: User attempting to create bucket with 'private' in name (should be denied)");
-    let private_bucket_name = format!("{test_user}-private-bucket");
-    let denied = test_client
-        .create_bucket()
-        .bucket(&private_bucket_name)
-        .send()
-        .await
-        .expect_err("the explicit deny must reject a matching bucket name");
-    assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
+    let private_bucket_name = "private-test-bucket";
+    let create_private_result = test_client.create_bucket().bucket(private_bucket_name).send().await;
+    if create_private_result.is_ok() {
+        cleanup().await;
+        return Err("User should NOT be able to create bucket with 'private' in name due to deny rule".into());
+    }
 
     // Cleanup
     info!("Cleaning up test resources");

crates/e2e_test/src/policy/test_env.rs

@@ -0,0 +1,100 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Custom test environment for policy variables tests
+//!
+//! This module provides a custom test environment that doesn't automatically
+//! stop servers when destroyed, addressing the server stopping issue.
+
+use aws_sdk_s3::Client;
+use aws_sdk_s3::config::{Config, Credentials, Region};
+use std::net::TcpStream;
+use std::time::Duration;
+use tokio::time::sleep;
+use tracing::{info, warn};
+
+// Default credentials
+const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
+const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
+
+/// Custom test environment that doesn't automatically stop servers
+pub struct PolicyTestEnvironment {
+    pub temp_dir: String,
+    pub address: String,
+    pub url: String,
+    pub access_key: String,
+    pub secret_key: String,
+}
+
+impl PolicyTestEnvironment {
+    /// Create a new test environment with specific address
+    /// This environment won't stop any server when dropped
+    pub async fn with_address(address: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
+        let temp_dir = format!("/tmp/rustfs_policy_test_{}", uuid::Uuid::new_v4());
+        tokio::fs::create_dir_all(&temp_dir).await?;
+
+        let url = format!("http://{address}");
+
+        Ok(Self {
+            temp_dir,
+            address: address.to_string(),
+            url,
+            access_key: DEFAULT_ACCESS_KEY.to_string(),
+            secret_key: DEFAULT_SECRET_KEY.to_string(),
+        })
+    }
+
+    /// Create an AWS S3 client configured for this RustFS instance
+    pub fn create_s3_client(&self, access_key: &str, secret_key: &str) -> Client {
+        let credentials = Credentials::new(access_key, secret_key, None, None, "policy-test");
+        let config = Config::builder()
+            .credentials_provider(credentials)
+            .region(Region::new("us-east-1"))
+            .endpoint_url(&self.url)
+            .force_path_style(true)
+            .behavior_version_latest()
+            .build();
+        Client::from_conf(config)
+    }
+
+    /// Wait for RustFS server to be ready by checking TCP connectivity
+    pub async fn wait_for_server_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        info!("Waiting for RustFS server to be ready on {}", self.address);
+
+        for i in 0..30 {
+            if TcpStream::connect(&self.address).is_ok() {
+                info!("✅ RustFS server is ready after {} attempts", i + 1);
+                return Ok(());
+            }
+
+            if i == 29 {
+                return Err("RustFS server failed to become ready within 30 seconds".into());
+            }
+
+            sleep(Duration::from_secs(1)).await;
+        }
+
+        Ok(())
+    }
+}
+
+// Implement Drop trait that doesn't stop servers
+impl Drop for PolicyTestEnvironment {
+    fn drop(&mut self) {
+        // Clean up temp directory only, don't stop any server
+        if let Err(e) = std::fs::remove_dir_all(&self.temp_dir) {
+            warn!("Failed to clean up temp directory {}: {}", self.temp_dir, e);
+        }
+    }
+}

crates/e2e_test/src/policy/test_runner.rs

@@ -0,0 +1,230 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::init_logging;
+use crate::policy::test_env::PolicyTestEnvironment;
+use std::time::Instant;
+use tokio::time::{Duration, sleep};
+use tracing::{error, info};
+
+/// Test case definition
+#[derive(Debug, Clone)]
+pub struct TestDefinition {
+    pub name: String,
+    pub is_critical: bool,
+}
+
+impl TestDefinition {
+    pub fn new(name: impl Into<String>, is_critical: bool) -> Self {
+        Self {
+            name: name.into(),
+            is_critical,
+        }
+    }
+}
+
+/// Test result
+#[derive(Debug, Clone)]
+pub struct TestResult {
+    pub test_name: String,
+    pub success: bool,
+    pub error_message: Option<String>,
+}
+
+impl TestResult {
+    pub fn success(test_name: String) -> Self {
+        Self {
+            test_name,
+            success: true,
+            error_message: None,
+        }
+    }
+
+    pub fn failure(test_name: String, error: String) -> Self {
+        Self {
+            test_name,
+            success: false,
+            error_message: Some(error),
+        }
+    }
+}
+
+/// Test suite configuration
+#[derive(Debug, Clone, Default)]
+pub struct TestSuiteConfig {
+    pub include_critical_only: bool,
+}
+
+/// Policy test suite
+pub struct PolicyTestSuite {
+    tests: Vec<TestDefinition>,
+    config: TestSuiteConfig,
+}
+
+impl PolicyTestSuite {
+    /// Create default test suite
+    pub fn new() -> Self {
+        let tests = vec![
+            TestDefinition::new("test_aws_policy_variables_single_value", true),
+            TestDefinition::new("test_aws_policy_variables_multi_value", true),
+            TestDefinition::new("test_aws_policy_variables_concatenation", true),
+            TestDefinition::new("test_aws_policy_variables_nested", true),
+            TestDefinition::new("test_aws_policy_variables_deny", true),
+            TestDefinition::new("test_aws_policy_variables_sts", true),
+        ];
+
+        Self {
+            tests,
+            config: TestSuiteConfig::default(),
+        }
+    }
+
+    /// Configure test suite
+    pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
+        self.config = config;
+        self
+    }
+
+    /// Run test suite
+    pub async fn run_test_suite(&self) -> Vec<TestResult> {
+        init_logging();
+        info!("Starting Policy Variables test suite");
+
+        let start_time = Instant::now();
+        let mut results = Vec::new();
+
+        // Create test environment
+        let env = match PolicyTestEnvironment::with_address("127.0.0.1:9000").await {
+            Ok(env) => env,
+            Err(e) => {
+                error!("Failed to create test environment: {}", e);
+                return vec![TestResult::failure("env_creation".into(), e.to_string())];
+            }
+        };
+
+        // Wait for server to be ready
+        if env.wait_for_server_ready().await.is_err() {
+            error!("Server is not ready");
+            return vec![TestResult::failure("server_check".into(), "Server not ready".into())];
+        }
+
+        // Filter tests
+        let tests_to_run: Vec<&TestDefinition> = self
+            .tests
+            .iter()
+            .filter(|test| !self.config.include_critical_only || test.is_critical)
+            .collect();
+
+        info!("Scheduled {} tests", tests_to_run.len());
+
+        // Run tests
+        for (i, test_def) in tests_to_run.iter().enumerate() {
+            info!("Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
+            let test_start = Instant::now();
+
+            let result = self.run_single_test(test_def, &env).await;
+            let test_duration = test_start.elapsed();
+
+            match result {
+                Ok(_) => {
+                    info!("Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
+                    results.push(TestResult::success(test_def.name.clone()));
+                }
+                Err(e) => {
+                    error!("Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
+                    results.push(TestResult::failure(test_def.name.clone(), e.to_string()));
+                }
+            }
+
+            // Delay between tests to avoid resource conflicts
+            if i < tests_to_run.len() - 1 {
+                sleep(Duration::from_secs(2)).await;
+            }
+        }
+
+        // Print summary
+        self.print_summary(&results, start_time.elapsed());
+
+        results
+    }
+
+    /// Run a single test
+    async fn run_single_test(
+        &self,
+        test_def: &TestDefinition,
+        env: &PolicyTestEnvironment,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        match test_def.name.as_str() {
+            "test_aws_policy_variables_single_value" => {
+                super::policy_variables_test::test_aws_policy_variables_single_value_impl_with_env(env).await
+            }
+            "test_aws_policy_variables_multi_value" => {
+                super::policy_variables_test::test_aws_policy_variables_multi_value_impl_with_env(env).await
+            }
+            "test_aws_policy_variables_concatenation" => {
+                super::policy_variables_test::test_aws_policy_variables_concatenation_impl_with_env(env).await
+            }
+            "test_aws_policy_variables_nested" => {
+                super::policy_variables_test::test_aws_policy_variables_nested_impl_with_env(env).await
+            }
+            "test_aws_policy_variables_deny" => {
+                super::policy_variables_test::test_aws_policy_variables_deny_impl_with_env(env).await
+            }
+            "test_aws_policy_variables_sts" => {
+                super::policy_variables_test::test_aws_policy_variables_sts_impl_with_env(env).await
+            }
+            _ => Err(format!("Test {} not implemented", test_def.name).into()),
+        }
+    }
+
+    /// Print test summary
+    fn print_summary(&self, results: &[TestResult], total_duration: Duration) {
+        info!("=== Test Suite Summary ===");
+        info!("Total duration: {:.2}s", total_duration.as_secs_f64());
+        info!("Total tests: {}", results.len());
+
+        let passed = results.iter().filter(|r| r.success).count();
+        let failed = results.len() - passed;
+        let success_rate = (passed as f64 / results.len() as f64) * 100.0;
+
+        info!("Passed: {} | Failed: {}", passed, failed);
+        info!("Success rate: {:.1}%", success_rate);
+
+        if failed > 0 {
+            error!("Failed tests:");
+            for result in results.iter().filter(|r| !r.success) {
+                error!("  - {}: {}", result.test_name, result.error_message.as_ref().unwrap());
+            }
+        }
+    }
+}
+
+/// Test suite
+#[tokio::test]
+#[ignore = "Connects to existing rustfs server"]
+async fn test_policy_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let config = TestSuiteConfig {
+        include_critical_only: true,
+    };
+    let suite = PolicyTestSuite::new().with_config(config);
+    let results = suite.run_test_suite().await;
+
+    let failed = results.iter().filter(|r| !r.success).count();
+    if failed > 0 {
+        return Err(format!("Critical tests failed: {failed} failures").into());
+    }
+
+    info!("All critical tests passed");
+    Ok(())
+}

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -9136,6 +9136,7 @@ mod tests {
         assert_eq!(loaded.report.scanned, 37);
         assert_eq!(loaded.report.eligible, 11);
         assert_eq!(loaded.report.enqueued, 5);
+        assert_eq!(loaded.cursor_revision, Some(1));
         assert!(loaded.lease_expires_at_unix_nanos > 0);
         let token = loaded
             .report
@@ -9154,6 +9155,30 @@ mod tests {
         assert_eq!(admission.lease_id, loaded.lease_id);
         assert_eq!(admission.lease_expires_at_unix_nanos, loaded.lease_expires_at_unix_nanos);
 
+        let mut same_marker_report = report.clone();
+        same_marker_report.scanned += 1;
+        persist_manual_transition_page_checkpoint(
+            &checkpoint_options,
+            &same_marker_report,
+            Some("logs/page-end".to_string()),
+            Some("opaque-next-version".to_string()),
+        )
+        .await
+        .expect("same-marker version checkpoint should persist through the durable progress sink");
+        let same_marker_checkpointed = load_manual_transition_job_record(ecstore.clone(), job_id)
+            .await
+            .expect("same-marker version checkpoint should reload");
+        assert_eq!(same_marker_checkpointed.cursor_revision, Some(2));
+        let (_, version_marker) = decode_manual_transition_continuation_token(
+            same_marker_checkpointed
+                .report
+                .continuation_token
+                .as_deref()
+                .expect("same-marker version checkpoint should persist a cursor"),
+        )
+        .expect("same-marker version cursor should decode");
+        assert_eq!(version_marker.as_deref(), Some("opaque-next-version"));
+
         create_test_bucket(&ecstore, &bucket).await;
         let lifecycle_xml = format!(
             r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -9210,6 +9235,7 @@ mod tests {
         assert_eq!(checkpointed.report.scanned, 1000);
         assert_eq!(checkpointed.report.eligible, 1000);
         assert_eq!(checkpointed.report.dry_run_eligible, 1000);
+        assert_eq!(checkpointed.cursor_revision, Some(3));
         let token = checkpointed
             .report
             .continuation_token

crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs

@@ -24,6 +24,10 @@ use crate::bucket::lifecycle::bucket_lifecycle_ops::{
     ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport,
 };
 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::{
+    MANUAL_TRANSITION_JOB_NAMESPACE, MANUAL_TRANSITION_SCOPE_NAMESPACE, MANUAL_TRANSITION_TASK_NAMESPACE,
+    MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
+};
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result as EcstoreResult};
 use crate::object_api::ObjectOptions;
@@ -34,10 +38,10 @@ use crate::store::ECStore;
 pub const MANUAL_TRANSITION_JOB_SCHEMA: &str = "rustfs-manual-transition-job-v1";
 pub const MANUAL_TRANSITION_TASK_SCHEMA: &str = "rustfs-manual-transition-task-v1";
 pub const MANUAL_TRANSITION_WORKER_RESULT_SCHEMA: &str = "rustfs-manual-transition-worker-result-v1";
-pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = "ilm/manual-transition/jobs";
-pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = "ilm/manual-transition/scopes";
-pub const MANUAL_TRANSITION_TASK_PREFIX: &str = "ilm/manual-transition/tasks";
-pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = "ilm/manual-transition/results";
+pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = MANUAL_TRANSITION_JOB_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = MANUAL_TRANSITION_SCOPE_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_TASK_PREFIX: &str = MANUAL_TRANSITION_TASK_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE.prefix;
 pub const MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE: usize = 64 * 1024;
 pub const MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE: usize = 16 * 1024;
 pub const MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE: usize = 8 * 1024;
@@ -195,6 +199,8 @@ pub struct ManualTransitionJobRecord {
     pub updated_at_unix_nanos: i128,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub completed_at_unix_nanos: Option<i128>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub cursor_revision: Option<u64>,
     pub report: ManualTransitionRunReport,
     pub queue_snapshot: ManualTransitionQueueSnapshot,
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -224,6 +230,7 @@ impl ManualTransitionJobRecord {
             created_at_unix_nanos: now,
             updated_at_unix_nanos: now,
             completed_at_unix_nanos: None,
+            cursor_revision: Some(0),
             report: ManualTransitionRunReport {
                 bucket: bucket.to_string(),
                 prefix: options.prefix.clone(),
@@ -238,7 +245,7 @@ impl ManualTransitionJobRecord {
 
     pub fn complete(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
         self.scan_completed = true;
-        self.report.merge_scan_report_preserving_worker(&report);
+        self.merge_scan_report(&report);
         self.queue_snapshot = queue_snapshot;
         self.error = None;
         self.mark_terminal_if_worker_drained();
@@ -316,7 +323,7 @@ impl ManualTransitionJobRecord {
             }
         }
         self.queue_snapshot = queue_snapshot;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
         self.mark_terminal_if_worker_drained();
     }
 
@@ -359,14 +366,14 @@ impl ManualTransitionJobRecord {
         self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed);
         self.report.tier_failure_by_reason = scan_tier_failure_by_reason;
         self.queue_snapshot = queue_snapshot;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
         self.mark_terminal_if_worker_drained();
         true
     }
 
     pub fn mark_cancel_requested(&mut self) {
         self.cancel_requested = true;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
     }
 
     pub fn claim_recovery_lease(&mut self, owner_id: impl Into<String>, queue_snapshot: ManualTransitionQueueSnapshot) {
@@ -380,7 +387,7 @@ impl ManualTransitionJobRecord {
     pub fn abandon_recovery_lease(&mut self, lease_id: Uuid) {
         if self.state == ManualTransitionJobState::Running && self.lease_id == lease_id {
             self.lease_expires_at_unix_nanos = 0;
-            self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+            self.advance_updated_at();
         }
     }
 
@@ -398,7 +405,7 @@ impl ManualTransitionJobRecord {
 
     pub fn renew_lease(&mut self, queue_snapshot: ManualTransitionQueueSnapshot) {
         let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
-        self.updated_at_unix_nanos = now;
+        self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
         self.lease_expires_at_unix_nanos = manual_transition_job_lease_expires_at(now);
         self.queue_snapshot = queue_snapshot;
     }
@@ -438,11 +445,18 @@ impl ManualTransitionJobRecord {
 
     pub fn update_running_progress(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
         if self.state == ManualTransitionJobState::Running {
-            self.report.merge_scan_report_preserving_worker(&report);
+            self.merge_scan_report(&report);
             self.renew_lease(queue_snapshot);
         }
     }
 
+    fn merge_scan_report(&mut self, report: &ManualTransitionRunReport) {
+        if self.report.continuation_token != report.continuation_token {
+            self.cursor_revision = Some(self.cursor_revision.unwrap_or(0).saturating_add(1));
+        }
+        self.report.merge_scan_report_preserving_worker(report);
+    }
+
     pub fn mark_unknown_if_unowned(&mut self) {
         if self.state == ManualTransitionJobState::Running {
             self.state = ManualTransitionJobState::Unknown;
@@ -463,9 +477,13 @@ impl ManualTransitionJobRecord {
     }
 
     fn mark_updated_terminal(&mut self) {
+        self.advance_updated_at();
+        self.completed_at_unix_nanos = Some(self.updated_at_unix_nanos);
+    }
+
+    fn advance_updated_at(&mut self) {
         let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
-        self.updated_at_unix_nanos = now;
-        self.completed_at_unix_nanos = Some(now);
+        self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
     }
 
     fn mark_terminal_if_worker_drained(&mut self) {
@@ -1109,7 +1127,8 @@ pub fn manual_transition_scope_record_object_name(scope_key: &str) -> Result<Str
 pub async fn save_manual_transition_job_record(api: Arc<ECStore>, job: &ManualTransitionJobRecord) -> EcstoreResult<()> {
     let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
     let data = job.encode().map_err(manual_transition_job_store_error)?;
-    config_boundary::save_config(api, &object, data).await
+    config_boundary::save_config(api.clone(), &object, data.clone()).await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 pub async fn load_manual_transition_job_record(api: Arc<ECStore>, job_id: Uuid) -> EcstoreResult<ManualTransitionJobRecord> {
@@ -1142,9 +1161,9 @@ pub async fn save_manual_transition_job_record_if_current(
     let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
     let data = job.encode().map_err(manual_transition_job_store_error)?;
     config_boundary::save_config_with_opts_quiet(
-        api,
+        api.clone(),
         &object,
-        data,
+        data.clone(),
         &ObjectOptions {
             max_parity: true,
             http_preconditions: Some(HTTPPreconditions {
@@ -1154,7 +1173,8 @@ pub async fn save_manual_transition_job_record_if_current(
             ..Default::default()
         },
     )
-    .await
+    .await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 /// Applies a job-record mutation with optimistic concurrency control.
@@ -1592,9 +1612,9 @@ pub async fn save_manual_transition_scope_admission_if_absent(
     let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
     let data = serde_json::to_vec(admission).map_err(Error::other)?;
     config_boundary::save_config_with_opts(
-        api,
+        api.clone(),
         &object,
-        data,
+        data.clone(),
         &ObjectOptions {
             max_parity: true,
             http_preconditions: Some(HTTPPreconditions {
@@ -1604,7 +1624,8 @@ pub async fn save_manual_transition_scope_admission_if_absent(
             ..Default::default()
         },
     )
-    .await
+    .await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 pub async fn load_manual_transition_scope_admission(
@@ -1642,9 +1663,9 @@ pub async fn save_manual_transition_scope_admission_if_current(
     let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
     let data = serde_json::to_vec(admission).map_err(Error::other)?;
     match config_boundary::save_config_with_opts(
-        api,
+        api.clone(),
         &object,
-        data,
+        data.clone(),
         &ObjectOptions {
             max_parity: true,
             http_preconditions: Some(HTTPPreconditions {
@@ -1660,7 +1681,8 @@ pub async fn save_manual_transition_scope_admission_if_current(
             Err(Error::PreconditionFailed)
         }
         result => result,
-    }
+    }?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 pub async fn claim_manual_transition_scope_admission(
@@ -1953,13 +1975,15 @@ pub async fn delete_manual_transition_scope_admission_if_current(
     job_id: Uuid,
     lease_id: Uuid,
 ) -> EcstoreResult<bool> {
-    let etag = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
-        Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => etag,
+    let (admission, etag) = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
+        Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => (admission, etag),
         Ok(_) => return Ok(false),
         Err(Error::ConfigNotFound) => return Ok(true),
         Err(err) => return Err(err),
     };
     let object = manual_transition_scope_record_object_name(scope_key).map_err(manual_transition_job_store_error)?;
+    let data = serde_json::to_vec(&admission).map_err(Error::other)?;
+    api.record_durable_ilm_decommission_terminal(&object, &data).await?;
     match config_boundary::delete_config_if_match(api, &object, &etag).await {
         Ok(()) | Err(Error::ConfigNotFound) => Ok(true),
         Err(Error::PreconditionFailed) => Ok(false),
@@ -2577,6 +2601,25 @@ mod tests {
         assert!(decoded.report.tier_failure_by_reason.is_empty());
     }
 
+    #[test]
+    fn manual_transition_job_record_decodes_legacy_cursor_without_revision() {
+        let options = ManualTransitionRunOptions::default();
+        let record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
+        let encoded = record.encode().expect("job record should encode");
+        let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
+        value["job"]
+            .as_object_mut()
+            .expect("job should be object")
+            .remove("cursor_revision");
+        let record_bytes = serde_json::to_vec(&value["job"]).expect("legacy job should encode");
+        value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
+        let legacy = serde_json::to_vec(&value).expect("legacy envelope should encode");
+
+        let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job should decode");
+
+        assert_eq!(decoded.cursor_revision, None);
+    }
+
     #[test]
     fn manual_transition_job_record_rejects_unknown_report_fields() {
         let options = ManualTransitionRunOptions::default();

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -16,6 +16,7 @@ pub mod bucket_lifecycle_audit;
 pub mod bucket_lifecycle_ops;
 mod config_boundary;
 pub mod core;
+mod durable_namespace;
 pub mod evaluator;
 pub mod manual_transition_job;
 mod metadata_boundary;
@@ -31,3 +32,8 @@ pub mod tier_free_version_recovery;
 pub mod tier_last_day_stats;
 pub mod tier_sweeper;
 pub mod transition_transaction;
+
+pub(crate) use durable_namespace::{
+    DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
+    validate_durable_ilm_record,
+};

crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs

@@ -20,6 +20,7 @@ use tokio_util::sync::CancellationToken;
 use tracing::{debug, warn};
 
 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::TIER_DELETE_JOURNAL_NAMESPACE;
 use crate::bucket::lifecycle::runtime_boundary;
 use crate::bucket::lifecycle::tier_sweeper::{
     Jentry, TierDeleteJournalState, TierDeleteSourceIdentity,
@@ -49,7 +50,7 @@ const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
 const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
 const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
 const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
-pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
+pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix;
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
@@ -432,6 +433,21 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
         )
         .await?;
     }
+    let path = tier_delete_journal_object_name(je);
+    let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
+    let target_pool_indices = api
+        .record_durable_ilm_decommission_terminal_target_pools(&path, &data)
+        .await
+        .map_err(std::io::Error::other)?;
+    if let Some(target_pool_indices) = target_pool_indices {
+        for target_pool_idx in target_pool_indices {
+            match config_boundary::delete_config(api.pools[target_pool_idx].clone(), &path).await {
+                Ok(()) | Err(Error::ConfigNotFound) => {}
+                Err(err) => return Err(std::io::Error::other(err)),
+            }
+        }
+        return Ok(());
+    }
     remove_tier_delete_journal_entry(api, je).await
 }
 

crates/ecstore/src/bucket/lifecycle/transition_transaction.rs

@@ -21,6 +21,7 @@ use tracing::{debug, warn};
 use uuid::Uuid;
 
 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
 use crate::bucket::lifecycle::tier_sweeper::{
     delete_confirmed_transition_candidate_exact_with_lease_idempotent,
@@ -42,7 +43,7 @@ const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(6
 const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
 pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
 pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
-pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = "ilm/transition-transactions/records";
+pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
 pub const MAX_TRANSITION_TRANSACTION_SIZE: usize = 64 * 1024;
 
 pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
@@ -584,7 +585,8 @@ pub(crate) async fn save_transition_transaction_record(
     let object =
         transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
     let data = transaction.encode().map_err(transition_transaction_store_error)?;
-    config_boundary::save_config(api, &object, data).await
+    config_boundary::save_config(api.clone(), &object, data.clone()).await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }
 
 pub(crate) async fn load_transition_transaction_record(
@@ -596,8 +598,14 @@ pub(crate) async fn load_transition_transaction_record(
     TransitionTransaction::decode(transaction_id, &data).map_err(transition_transaction_store_error)
 }
 
-pub(crate) async fn delete_transition_transaction_record(api: Arc<ECStore>, transaction_id: Uuid) -> EcstoreResult<()> {
-    let object = transition_transaction_record_object_name(transaction_id).map_err(transition_transaction_store_error)?;
+pub(crate) async fn delete_transition_transaction_record(
+    api: Arc<ECStore>,
+    transaction: &TransitionTransaction,
+) -> EcstoreResult<()> {
+    let object =
+        transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
+    let data = transaction.encode().map_err(transition_transaction_store_error)?;
+    api.record_durable_ilm_decommission_terminal(&object, &data).await?;
     match config_boundary::delete_config(api, &object).await {
         Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
         Err(err) => Err(err),
@@ -813,7 +821,7 @@ pub async fn finalize_missing_transition_transaction_for_operator(
     if probe != TransitionOperatorProbe::Missing {
         return Err(TransitionOperatorError::CandidateNotMissing(probe));
     }
-    delete_transition_transaction_record(api, transaction_id)
+    delete_transition_transaction_record(api, &transaction)
         .await
         .map_err(TransitionOperatorError::Store)
 }
@@ -849,22 +857,22 @@ pub async fn process_transition_transaction_record(
     match transaction.state {
         TransitionTransactionState::Uploaded => {
             delete_transition_remote_candidate(api.clone(), transaction).await?;
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
             Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
         }
         TransitionTransactionState::CleanupPending => match local_commit_matches_transaction(api.clone(), transaction).await {
             Ok(true) => {
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                 Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
             }
             Ok(false) => {
                 delete_transition_remote_candidate(api.clone(), transaction).await?;
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                 Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
             }
             Err(err) if transition_source_is_missing(&err) => {
                 delete_transition_remote_candidate(api.clone(), transaction).await?;
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                 Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
             }
             Err(err) => Err(err),
@@ -872,7 +880,7 @@ pub async fn process_transition_transaction_record(
         TransitionTransactionState::LocalCommitStarted => {
             match local_commit_matches_transaction(api.clone(), transaction).await {
                 Ok(true) => {
-                    delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                    delete_transition_transaction_record(api, transaction).await?;
                     Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
                 }
                 Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
@@ -881,7 +889,7 @@ pub async fn process_transition_transaction_record(
             }
         }
         TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
             Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
         }
         TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await,
@@ -907,7 +915,7 @@ async fn recover_unknown_upload_outcome(
         .map_err(Error::other)?
     {
         TransitionCandidateProbe::Missing => {
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
             Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
         }
         TransitionCandidateProbe::UnversionedPresent => {
@@ -925,7 +933,7 @@ async fn recover_unknown_upload_outcome(
             )
             .await
             .map_err(Error::other)?;
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
             Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
         }
         TransitionCandidateProbe::VersionedPresent(version_id) => {
@@ -958,7 +966,7 @@ async fn cleanup_recovered_unknown_upload_candidate(
         .map_err(transition_transaction_store_error)?;
     save_transition_transaction_record(api.clone(), &cleanup).await?;
     delete_transition_remote_candidate(api.clone(), &cleanup).await?;
-    delete_transition_transaction_record(api, cleanup.transaction_id).await?;
+    delete_transition_transaction_record(api, &cleanup).await?;
     Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
 }
 

crates/ecstore/src/config/com.rs

@@ -406,6 +406,25 @@ where
     Ok(data)
 }
 
+pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
+where
+    S: EcstoreObjectIO,
+{
+    let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?;
+    Ok(data)
+}
+
+pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
+    api: Arc<S>,
+    file: &str,
+    max_bytes: usize,
+) -> Result<(Vec<u8>, ObjectInfo)>
+where
+    S: EcstoreObjectIO,
+{
+    read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await
+}
+
 /// Read an existing config object without treating an empty payload as absent.
 /// Callers that validate their own payload format need to distinguish corruption
 /// from `ConfigNotFound`.

crates/ecstore/src/set_disk/ops/object.rs

@@ -37,7 +37,7 @@ use crate::bucket::lifecycle::{
     transition_transaction::{
         TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
         TransitionTransactionInit, TransitionTransactionState, delete_transition_transaction_record,
-        save_transition_transaction_record,
+        load_transition_transaction_record, save_transition_transaction_record,
     },
 };
 use crate::bucket::quota::reservation;
@@ -4248,7 +4248,12 @@ fn record_transition_uploaded_save_attempt(transaction: &TransitionTransaction,
 
 async fn delete_transition_transaction_if_available(api: Option<&Arc<ECStore>>, transaction_id: Uuid) -> Result<()> {
     if let Some(api) = api {
-        return delete_transition_transaction_record(api.clone(), transaction_id).await;
+        let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
+            Ok(transaction) => transaction,
+            Err(Error::ConfigNotFound) => return Ok(()),
+            Err(err) => return Err(err),
+        };
+        return delete_transition_transaction_record(api.clone(), &transaction).await;
     }
     Ok(())
 }

docs/testing/e2e-suite-inventory.md

@@ -80,7 +80,6 @@
 | object_lambda_test | 16 | 🌙 |
 | object_lock | 34 |  |
 | overwrite_cleanup_regression_test | 1 |  |
-| policy | 6 |  |
 | presigned_negative_test | 7 | ✅ |
 | protocols | 16 | 🌙 |
 | quota_test | 14 |  |
@@ -100,4 +99,4 @@
 | tls_hot_reload_test | 1 | ✅ |
 | version_id_regression_test | 10 | ✅ |
 
-**Total listed: 581 tests across 83 modules · PR smoke: 163 tests / 36 modules · merge/main full: 459 tests / 74 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23.
+**Total listed: 575 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 453 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23.