fix(ecstore): remove reachable panics in tiering, replication, and heal paths (#4205)

* fix(ecstore): remove reachable panics in tiering, replication, and heal paths

- Parse x-amz-expiration leniently in tier PUT responses; any lifecycle
  rule on the remote tier bucket returns an RFC1123 date that the previous
  ISO8601 unwrap turned into a panic of the ILM transition worker
- Skip invalid user-metadata header values (with a warning) when building
  tier and replication PUT headers instead of panicking on non-ASCII input
- Heal: tolerate absent data_dir for delete markers and remote objects
- transition_object: don't unwrap version_id on unversioned buckets when
  recording partial writes for offline disks
- Admin server info: use port_or_known_default() so default-port (80/443)
  endpoints don't panic is_server_resolvable
- Tier ListObjectsV2 client: decode response body with from_utf8_lossy
- walk_internal: log merge_entry_channels errors instead of dropping them

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(ecstore): fail heal explicitly when data_dir is missing

Address review feedback: unwrap_or_default() silently substituted a nil
UUID when latest metadata lacked data_dir. Delete markers and remote
objects legitimately have no data_dir and skip the data-heal block, but
for a regular object a missing data_dir means corrupt metadata — return
FileCorrupt with a descriptive log instead of building part paths under
a nil UUID directory.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

crates/ecstore/src/client/api_list.rs

@@ -133,7 +133,7 @@ impl TransitionClient {
                 body_vec.extend_from_slice(data);
             }
         }
-        let mut list_bucket_result = match quick_xml::de::from_str::<ListBucketV2Result>(&String::from_utf8(body_vec).unwrap()) {
+        let mut list_bucket_result = match quick_xml::de::from_str::<ListBucketV2Result>(&String::from_utf8_lossy(&body_vec)) {
             Ok(result) => result,
             Err(err) => {
                 return Err(std::io::Error::other(err.to_string()));

crates/ecstore/src/client/api_put_object.rs

@@ -234,13 +234,17 @@ impl PutObjectOptions {
         }
 
         for (k, v) in &self.user_metadata {
+            let Ok(header_value) = HeaderValue::from_str(v) else {
+                warn!("skipping user metadata header with invalid value: {}", k);
+                continue;
+            };
             if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k)
             {
                 if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
-                    header.insert(header_name, HeaderValue::from_str(&v).expect("operation should succeed"));
+                    header.insert(header_name, header_value);
                 }
             } else if let Ok(header_name) = HeaderName::from_bytes(format!("x-amz-meta-{}", k).as_bytes()) {
-                header.insert(header_name, HeaderValue::from_str(&v).expect("operation should succeed"));
+                header.insert(header_name, header_value);
             }
         }
 

crates/ecstore/src/client/api_put_object_multipart.rs

@@ -388,14 +388,13 @@ impl TransitionClient {
 
         let complete_multipart_upload_result: CompleteMultipartUploadResult = CompleteMultipartUploadResult::default();
 
-        let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
-            (
-                OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
-                "".to_string(),
-            )
-        } else {
-            (OffsetDateTime::now_utc(), "".to_string())
-        };
+        let exp_time = resp
+            .headers()
+            .get(X_AMZ_EXPIRATION)
+            .and_then(|v| v.to_str().ok())
+            .and_then(|s| OffsetDateTime::parse(s, ISO8601_DATEFORMAT).ok())
+            .unwrap_or_else(OffsetDateTime::now_utc);
+        let rule_id = "".to_string();
 
         Ok(UploadInfo {
             bucket: complete_multipart_upload_result.bucket,

crates/ecstore/src/client/api_put_object_streaming.rs

@@ -523,14 +523,13 @@ impl TransitionClient {
             )));
         }
 
-        let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
-            (
-                OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
-                "".to_string(),
-            )
-        } else {
-            (OffsetDateTime::now_utc(), "".to_string())
-        };
+        let exp_time = resp
+            .headers()
+            .get(X_AMZ_EXPIRATION)
+            .and_then(|v| v.to_str().ok())
+            .and_then(|s| OffsetDateTime::parse(s, ISO8601_DATEFORMAT).ok())
+            .unwrap_or_else(OffsetDateTime::now_utc);
+        let rule_id = "".to_string();
         let h = resp.headers();
         Ok(UploadInfo {
             bucket: bucket_name.to_string(),