fix(heal): recover shards after node rejoin (#3814)

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -16,8 +16,12 @@
 
 #[cfg(test)]
 mod tests {
-    use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, execute_awscurl, init_logging};
+    use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client};
     use aws_sdk_s3::primitives::ByteStream;
+    use http::header::{CONTENT_TYPE, HOST};
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::sign_v4;
+    use s3s::Body;
     use serial_test::serial;
     use std::collections::HashSet;
     use std::error::Error;
@@ -61,6 +65,45 @@ mod tests {
         assert_eq!(body.as_ref(), expected, "object body changed for {key}");
     }
 
+    async fn signed_admin_post(
+        url: &str,
+        body: Option<&str>,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<String, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let mut request = http::Request::builder()
+            .method(http::Method::POST)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+
+        if body.is_some() {
+            request = request.header(CONTENT_TYPE, "application/json");
+        }
+
+        let content_len = body.map(str::len).unwrap_or_default() as i64;
+        let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+        let mut request_builder = local_http_client().post(url);
+        for (name, value) in signed.headers() {
+            request_builder = request_builder.header(name, value);
+        }
+        if let Some(body) = body {
+            request_builder = request_builder.body(body.to_string());
+        }
+
+        let response = request_builder.send().await?;
+        let status = response.status();
+        let body = response.text().await?;
+        if !status.is_success() {
+            return Err(format!("admin POST failed: {status} {body}").into());
+        }
+
+        Ok(body)
+    }
+
     #[tokio::test]
     #[serial]
     async fn test_auto_heal_rebuilds_runtime_wiped_disk_without_restart() {
@@ -292,7 +335,7 @@ mod tests {
 
         let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
         let heal_url = format!("{}/rustfs/admin/v3/heal/{}?forceStart=true", env.url, bucket);
-        execute_awscurl(&heal_url, "POST", Some(heal_body), &env.access_key, &env.secret_key)
+        signed_admin_post(&heal_url, Some(heal_body), &env.access_key, &env.secret_key)
             .await
             .expect("admin deep heal should be accepted");
 
@@ -332,9 +375,9 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
-    async fn test_cluster_admin_heal_rebuilds_replaced_remote_disk() -> Result<(), Box<dyn Error + Send + Sync>> {
+    async fn test_cluster_root_heal_rebuilds_replaced_remote_disk() -> Result<(), Box<dyn Error + Send + Sync>> {
         init_logging();
-        info!("Admin deep heal should rebuild data on a remote node after its disk is replaced and the node rejoins");
+        info!("Root recursive heal should rebuild data on a remote node after its disk is replaced and the node rejoins");
 
         let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
         cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
@@ -383,15 +426,15 @@ mod tests {
         cluster.start_node(1).await?;
 
         let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
-        let status_body = execute_awscurl(&status_url, "POST", None, &cluster.access_key, &cluster.secret_key).await?;
+        let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
         assert!(
             !status_body.contains("MissingContentLength"),
             "background heal status should not fail without an explicit Content-Length: {status_body}"
         );
 
         let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
-        let heal_url = format!("{}/rustfs/admin/v3/heal/{}?forceStart=true", cluster.nodes[0].url, bucket);
-        execute_awscurl(&heal_url, "POST", Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
+        let heal_url = format!("{}/rustfs/admin/v3/heal/?forceStart=true", cluster.nodes[0].url);
+        signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
 
         let expected_objects = [(online_key, online_body.as_slice()), (outage_key, outage_body.as_slice())];
         let mut remaining_rebuild_keys: HashSet<&str> = expected_objects.iter().map(|(key, _)| *key).collect();