Merge branch 'main' into overtrue/activate-group-e2e

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52
-sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6
+sha256-darwin=f832043fcca8c0b616c5d820a3a652da7544298ef5812a8668a3a9a3e4607b8b
+sha256-linux=93b94adb110b86a41d0b7313909e0bf53cb1515e2d08e8f105652b29b249990f

crates/e2e_test/src/group_delete_test.rs

@@ -14,7 +14,7 @@
 
 //! E2E tests for group management (fixes #2028).
 
-use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
+use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::{Client, Config};
 use tracing::info;
@@ -83,7 +83,6 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo
 
 /// Test that deleting a group with members fails, and deleting an empty group succeeds.
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -91,29 +90,58 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
     env.start_rustfs_server(vec![]).await?;
 
     // 1. Create a user
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
     let user_body = serde_json::json!({
         "secretKey": "testuser1secret",
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/add-user?accessKey=testuser1",
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created testuser1");
 
     // 2. Create a group with testuser1 as a member
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": "testgroup",
         "members": ["testuser1"],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added testuser1 to testgroup");
 
     // 3. Attempt to delete the group while it still has members — should fail
-    let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
-    let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
-    assert!(delete_result.is_err(), "deleting a non-empty group should fail");
+    let (delete_status, delete_body) = admin_request(
+        &env.url,
+        http::Method::DELETE,
+        "/rustfs/admin/v3/group/testgroup",
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(
+        delete_status,
+        reqwest::StatusCode::BAD_REQUEST,
+        "deleting a non-empty group must return HTTP 400, body: {delete_body}"
+    );
+    assert!(
+        delete_body.contains("<Code>InvalidRequest</Code>"),
+        "deleting a non-empty group must return InvalidRequest, body: {delete_body}"
+    );
+    assert!(
+        delete_body.contains("<Message>group is not empty</Message>"),
+        "deleting a non-empty group returned an unexpected message: {delete_body}"
+    );
     info!("Delete of non-empty group correctly rejected");
 
     // 4. Remove the member from the group
@@ -123,17 +151,42 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
         "isRemove": true,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(remove_member_body.to_string()),
+    )
+    .await?;
     info!("Removed testuser1 from testgroup");
 
     // 5. Delete the now-empty group — should succeed
-    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(&env, http::Method::DELETE, "/rustfs/admin/v3/group/testgroup", None).await?;
     info!("Deleted empty testgroup successfully");
 
     // 6. Verify the group no longer exists
-    let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
-    let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
-    assert!(get_result.is_err(), "group should no longer exist after deletion");
+    let (get_status, get_body) = admin_request(
+        &env.url,
+        http::Method::GET,
+        "/rustfs/admin/v3/group?group=testgroup",
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(
+        get_status,
+        reqwest::StatusCode::NOT_FOUND,
+        "a deleted group must return HTTP 404, body: {get_body}"
+    );
+    assert!(
+        get_body.contains("<Code>NoSuchResource</Code>"),
+        "a deleted group must return NoSuchResource, body: {get_body}"
+    );
+    assert!(
+        get_body.contains("<Message>group &apos;testgroup&apos; does not exist</Message>"),
+        "a deleted group returned an unexpected message: {get_body}"
+    );
     info!("Confirmed testgroup no longer exists");
 
     Ok(())
@@ -142,7 +195,6 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
 /// Test that a user with only group membership (no explicit user policy) gets group policies
 /// and can perform actions allowed by the group (regression test for #2028.1).
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -160,39 +212,56 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
         "Statement": [{
             "Effect": "Allow",
             "Action": ["s3:ListAllMyBuckets"],
-            "Resource": ["*"]
+            "Resource": ["arn:aws:s3:::*"]
         }]
     });
-    let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
-    awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
+        Some(policy_doc.to_string()),
+    )
+    .await?;
     info!("Created canned policy {}", policy_name);
 
     // 2. Create user with no explicit policy
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
     let user_body = serde_json::json!({
         "secretKey": user_secret,
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created user {} with no explicit policy", user_name);
 
     // 3. Add user to group (creates group with this member; user_group_memberships must be updated)
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": group_name,
         "members": [user_name],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added {} to group {}", user_name, group_name);
 
     // 4. Attach policy to group
-    let set_policy_url = format!(
-        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
-        env.url, policy_name, group_name
-    );
-    awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
+        Some(String::new()),
+    )
+    .await?;
     info!("Attached policy {} to group {}", policy_name, group_name);
 
     // 5. User with only group (no user policy) should be able to list buckets
@@ -209,7 +278,6 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
 /// Test that after deleting a user who was the only member of a group, the group can be deleted
 /// (regression test for #2028.2: delete group uses backend membership, not stale cache).
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
@@ -221,33 +289,47 @@ async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::erro
     let group_name = "soledeletegroup";
 
     // 1. Create user
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
     let user_body = serde_json::json!({
         "secretKey": user_secret,
         "status": "enabled"
     });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
+        Some(user_body.to_string()),
+    )
+    .await?;
     info!("Created user {}", user_name);
 
     // 2. Add user to group
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
     let add_member_body = serde_json::json!({
         "group": group_name,
         "members": [user_name],
         "isRemove": false,
         "groupStatus": "enabled"
     });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
     info!("Added {} to group {}", user_name, group_name);
 
     // 3. Delete the user (backend and cache update so group membership becomes empty)
-    let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
-    awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::DELETE,
+        &format!("/rustfs/admin/v3/remove-user?accessKey={user_name}"),
+        None,
+    )
+    .await?;
     info!("Deleted user {}", user_name);
 
     // 4. Deleting the group should succeed (backend has empty members; no stale cache)
-    let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
-    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(&env, http::Method::DELETE, &format!("/rustfs/admin/v3/group/{group_name}"), None).await?;
     info!("Deleted group {} after user was removed", group_name);
 
     Ok(())

crates/ecstore/src/core/pools.rs

@@ -462,16 +462,19 @@ pub(crate) fn local_decommission_queue_prefix(endpoints: &EndpointServerPools, i
     Ok(local)
 }
 
-fn resumable_decommission_queue_indices(meta: &PoolMeta) -> Vec<usize> {
+fn first_resumable_decommission_queue_indices(meta: &PoolMeta) -> Vec<usize> {
     let mut indices = Vec::new();
     for (idx, pool) in meta.pools.iter().enumerate() {
         if let Some(decommission) = &pool.decommission {
             if !decommission.has_decommission_state() {
                 continue;
             }
-            if decommission.complete || decommission.failed || decommission.canceled {
+            if decommission.complete {
                 continue;
             }
+            if decommission.failed || decommission.canceled {
+                break;
+            }
             indices.push(idx);
         }
     }
@@ -2403,10 +2406,24 @@ impl PoolMeta {
     }
 
     pub fn return_resumable_pools(&self) -> Vec<PoolStatus> {
-        resumable_decommission_queue_indices(self)
-            .into_iter()
-            .map(|idx| self.pools[idx].clone())
-            .collect()
+        let mut new_pools = Vec::new();
+        for pool in &self.pools {
+            if let Some(decommission) = &pool.decommission {
+                if !decommission.has_decommission_state() {
+                    continue;
+                }
+                if decommission.complete || decommission.failed || decommission.canceled {
+                    // Recovery is not required when:
+                    // - Decommissioning completed
+                    // - Decommissioning failed and must be explicitly restarted or cleared
+                    // - Decommissioning was cancelled
+                    continue;
+                }
+                // All other scenarios require recovery
+                new_pools.push(pool.clone());
+            }
+        }
+        new_pools
     }
 }
 
@@ -3315,7 +3332,7 @@ impl ECStore {
         let _start_guard = self.start_gate.lock().await;
         let indices = {
             let pool_meta = self.pool_meta.read().await;
-            resumable_decommission_queue_indices(&pool_meta)
+            first_resumable_decommission_queue_indices(&pool_meta)
                 .into_iter()
                 .filter(|idx| indices.contains(idx))
                 .collect::<Vec<_>>()
@@ -3360,7 +3377,7 @@ impl ECStore {
     pub async fn spawn_missing_local_decommission_routines(self: &Arc<Self>) -> Result<()> {
         let indices = {
             let pool_meta = self.pool_meta.read().await;
-            resumable_decommission_queue_indices(&pool_meta)
+            first_resumable_decommission_queue_indices(&pool_meta)
         };
         let indices = local_decommission_queue_prefix(&self.endpoints(), &indices)?;
         if indices.is_empty() {
@@ -6346,12 +6363,12 @@ mod pools_tests {
         ensure_decommission_start_keeps_active_pool, ensure_decommission_start_local_leader,
         ensure_decommission_start_pool_states, ensure_decommission_start_rebalance_meta_allowed,
         ensure_decommission_start_target_capacity, ensure_decommission_terminal_operation_supported,
-        ensure_local_decommission_pool_leaders, ensure_valid_decommission_pool_index, get_by_index, guard_decommission_cancelers,
-        has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested,
-        load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done,
-        merge_pool_status_refresh, missing_decommission_worker_prefix, observe_decommission_terminal_reload_result,
-        pool_meta_has_active_decommission, require_decommission_store, reserve_decommission_start_cancelers,
-        resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state,
+        ensure_local_decommission_pool_leaders, ensure_valid_decommission_pool_index, first_resumable_decommission_queue_indices,
+        get_by_index, guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active,
+        is_decommission_cancel_requested, load_decommission_entry_versions, local_decommission_queue_prefix,
+        mark_decommission_bucket_done, merge_pool_status_refresh, missing_decommission_worker_prefix,
+        observe_decommission_terminal_reload_result, pool_meta_has_active_decommission, require_decommission_store,
+        reserve_decommission_start_cancelers, resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state,
         resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result,
         resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result,
         resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result,
@@ -6359,9 +6376,9 @@ mod pools_tests {
         resolve_decommission_preflight_heal_result, resolve_decommission_progress_save_result,
         resolve_decommission_terminal_mark_after_error_result, resolve_decommission_terminal_mark_result,
         resolve_decommission_update_after_result, resolve_start_decommission_pool_meta_reload_result,
-        resumable_decommission_queue_indices, rollback_start_decommission_pool_meta, run_decommission_buckets_bounded,
-        run_decommission_listing_with_retry, run_decommission_listing_with_retry_and_drain, run_decommission_side_effect,
-        should_cleanup_decommission_source_entry, should_continue_decommission_queue, should_count_decommission_version_complete,
+        rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, run_decommission_listing_with_retry,
+        run_decommission_listing_with_retry_and_drain, run_decommission_side_effect, should_cleanup_decommission_source_entry,
+        should_continue_decommission_queue, should_count_decommission_version_complete,
         should_preserve_decommission_canceled_state, should_reject_decommission_cancel_as_terminal,
         should_retry_decommission_cancel_reload, should_retry_decommission_listing, should_skip_canceled_decommission_routine,
         spawn_decommission_index_cancelers, split_decommission_buckets, take_and_cancel_decommission_canceler,
@@ -9370,7 +9387,7 @@ mod pools_tests {
     }
 
     #[test]
-    fn test_resumable_decommission_queue_indices_skip_terminal_predecessors() {
+    fn test_first_resumable_decommission_queue_indices_stops_at_failed_or_canceled_state() {
         let meta = PoolMeta {
             pools: vec![
                 decommission_test_pool_status(
@@ -9406,11 +9423,11 @@ mod pools_tests {
             ..Default::default()
         };
 
-        assert_eq!(resumable_decommission_queue_indices(&meta), vec![3]);
+        assert!(first_resumable_decommission_queue_indices(&meta).is_empty());
     }
 
     #[test]
-    fn test_resumable_decommission_queue_indices_preserve_active_predecessor_order() {
+    fn test_first_resumable_decommission_queue_indices_allows_after_completed_prefix() {
         let meta = PoolMeta {
             pools: vec![
                 decommission_test_pool_status(
@@ -9423,7 +9440,7 @@ mod pools_tests {
                 decommission_test_pool_status(
                     1,
                     Some(PoolDecommissionInfo {
-                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
+                        queued: true,
                         ..Default::default()
                     }),
                 ),
@@ -9438,11 +9455,11 @@ mod pools_tests {
             ..Default::default()
         };
 
-        assert_eq!(resumable_decommission_queue_indices(&meta), vec![1, 2]);
+        assert_eq!(first_resumable_decommission_queue_indices(&meta), vec![1, 2]);
     }
 
     #[test]
-    fn test_runtime_and_startup_use_the_same_resumable_queue() {
+    fn test_return_resumable_pools_skips_failed_decommission() {
         let meta = PoolMeta {
             pools: vec![
                 decommission_test_pool_status(
@@ -9454,20 +9471,6 @@ mod pools_tests {
                 ),
                 decommission_test_pool_status(
                     1,
-                    Some(PoolDecommissionInfo {
-                        canceled: true,
-                        ..Default::default()
-                    }),
-                ),
-                decommission_test_pool_status(
-                    2,
-                    Some(PoolDecommissionInfo {
-                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
-                        ..Default::default()
-                    }),
-                ),
-                decommission_test_pool_status(
-                    3,
                     Some(PoolDecommissionInfo {
                         queued: true,
                         ..Default::default()
@@ -9477,15 +9480,10 @@ mod pools_tests {
             ..Default::default()
         };
 
-        let runtime_indices = resumable_decommission_queue_indices(&meta);
-        let startup_ids = meta
-            .return_resumable_pools()
-            .into_iter()
-            .map(|pool| pool.id)
-            .collect::<Vec<_>>();
+        let resumable = meta.return_resumable_pools();
 
-        assert_eq!(runtime_indices, vec![2, 3]);
-        assert_eq!(startup_ids, vec![2, 3]);
+        assert_eq!(resumable.len(), 1);
+        assert_eq!(resumable[0].id, 1);
     }
 
     #[test]

docs/testing/e2e-suite-inventory.md

@@ -52,7 +52,7 @@
 | fault_proxy | 7 |  |
 | get_codec_streaming_compat_test | 1 |  |
 | get_stream_failure_observability_test | 1 |  |
-| group_delete_test | 1 |  |
+| group_delete_test | 4 |  |
 | head_object_consistency_test | 1 | ✅ |
 | head_object_range_test | 1 | ✅ |
 | heal_erasure_disk_rebuild_test | 4 | 🌙 |
@@ -99,4 +99,4 @@
 | tls_hot_reload_test | 1 | ✅ |
 | version_id_regression_test | 10 | ✅ |
 
-**Total listed: 575 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 453 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23.
+**Total listed: 578 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 456 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23.