test(internode): extend fallback and transition coverage

Add decode-error metrics for internode msgpack/json compatibility paths, extend the mixed fallback e2e assertions for transitioned multipart partNumber reads, and cover manual transition async status polling plus inactive-owner status behavior.

Co-Authored-By: heihutu <heihutu@gmail.com>

crates/e2e_test/src/inline_fast_path_cluster_test.rs

@@ -808,13 +808,14 @@ async fn assert_part_number_reader_path(
     key: &str,
     expected_part: &[u8],
     full_object_size: usize,
+    object_class: &str,
     expected_path: &str,
 ) -> TestResult {
     let paths = [INLINE_DIRECT, LEGACY_DUPLEX, EMPTY, REMOTE_TRANSITION];
     let size_bucket = size_bucket(full_object_size);
     let mut before = BTreeMap::<&str, u64>::new();
     for path in paths {
-        before.insert(path, collector.reader_path_total(path, MULTIPART, size_bucket).await);
+        before.insert(path, collector.reader_path_total(path, object_class, size_bucket).await);
     }
     let response = client.get_object().bucket(bucket).key(key).part_number(2).send().await?;
     assert_eq!(
@@ -825,17 +826,17 @@ async fn assert_part_number_reader_path(
     let body = response.body.collect().await?.into_bytes();
     assert_eq!(body.as_ref(), expected_part, "partNumber GET body changed for {key}");
     collector
-        .wait_for_reader_path_total(expected_path, MULTIPART, size_bucket, before[expected_path] + 1)
+        .wait_for_reader_path_total(expected_path, object_class, size_bucket, before[expected_path] + 1)
         .await?;
-    collector.wait_for_reader_paths_to_settle(MULTIPART, size_bucket).await?;
+    collector.wait_for_reader_paths_to_settle(object_class, size_bucket).await?;
     for path in paths {
         if path == expected_path {
             continue;
         }
         assert_eq!(
-            collector.reader_path_total(path, MULTIPART, size_bucket).await,
+            collector.reader_path_total(path, object_class, size_bucket).await,
             before[path],
-            "{READER_PATH_COUNTER}{{path={path}, object_class={MULTIPART}, size_bucket={size_bucket}}} must not advance for partNumber GET {key}; expected {expected_path} only"
+            "{READER_PATH_COUNTER}{{path={path}, object_class={object_class}, size_bucket={size_bucket}}} must not advance for partNumber GET {key}; expected {expected_path} only"
         );
     }
     Ok(())
@@ -1290,6 +1291,7 @@ async fn four_node_inline_fallback_controls() -> TestResult {
         multipart_key,
         &second_part,
         multipart_body.len(),
+        MULTIPART,
         LEGACY_DUPLEX,
     )
     .await?;
@@ -1385,7 +1387,7 @@ async fn four_node_multipart_ignores_disk_compression_fallback() -> TestResult {
         ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), LEGACY_DUPLEX, MULTIPART),
     )
     .await?;
-    assert_part_number_reader_path(&collector, &client, bucket, key, &second_part, body.len(), LEGACY_DUPLEX).await?;
+    assert_part_number_reader_path(&collector, &client, bucket, key, &second_part, body.len(), MULTIPART, LEGACY_DUPLEX).await?;
 
     Ok(())
 }
@@ -1428,6 +1430,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls() -> Te
         multipart_key,
         &second_part,
         multipart_body.len(),
+        MULTIPART,
         LEGACY_DUPLEX,
     )
     .await?;
@@ -1517,7 +1520,7 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
     put_lifecycle_with_transition_retry(&hot_client, bucket).await?;
 
     let key = "transition/mixed-multipart.bin";
-    let (body, _, etag) = put_two_part_multipart(&hot_client, bucket, key).await?;
+    let (body, second_part, etag) = put_two_part_multipart(&hot_client, bucket, key).await?;
     wait_for_transition(&hot_client, bucket, key).await?;
     assert!(
         cold_tier_object_count(&cold_client).await? >= 1,
@@ -1529,6 +1532,8 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
         ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, etag.as_deref(), None), REMOTE_TRANSITION, REMOTE),
     )
     .await?;
+    assert_part_number_reader_path(&collector, &hot_client, bucket, key, &second_part, body.len(), REMOTE, REMOTE_TRANSITION)
+        .await?;
 
     let encrypted_key = "transition/encrypted-sse.bin";
     let encrypted_body = payload(16 * KIB, 0xAB);

crates/e2e_test/src/reliant/tiering.rs

@@ -70,11 +70,13 @@ const MANUAL_DUE_BUCKET: &str = "ilm7-manual-due";
 const MANUAL_DRY_RUN_BUCKET: &str = "ilm7-manual-dry-run";
 const MANUAL_NOT_DUE_BUCKET: &str = "ilm7-manual-not-due";
 const MANUAL_QUEUE_PRESSURE_BUCKET: &str = "ilm7-manual-queue-pressure";
+const MANUAL_ASYNC_STATUS_BUCKET: &str = "ilm7-manual-async-status";
 const MANUAL_QUEUE_PRESSURE_PREFIX: &str = "manual-queue-pressure/";
 const OBJECT_KEY: &str = "tier/鲁A12345/report.bin";
 const MANUAL_DUE_KEY: &str = "manual-due/report.bin";
 const MANUAL_DRY_RUN_KEY: &str = "manual-dry-run/report.bin";
 const MANUAL_NOT_DUE_KEY: &str = "manual-not-due/report.bin";
+const MANUAL_ASYNC_STATUS_KEY: &str = "manual-async-status/report.bin";
 const CONTENT_TYPE: &str = "application/x-ilm7";
 const USER_META_KEY: &str = "ilm7-origin";
 const USER_META_VAL: &str = "hermetic-transition";
@@ -350,6 +352,16 @@ struct ManualTransitionRunReport {
     truncated_by_duration: bool,
 }
 
+#[derive(Debug, Deserialize)]
+struct ManualTransitionJobStatusResponse {
+    job_id: String,
+    status_endpoint: String,
+    status: String,
+    cancel_requested: bool,
+    failure_reason: Option<String>,
+    report: ManualTransitionRunReport,
+}
+
 async fn manual_transition_run(
     hot: &RustFSTestEnvironment,
     bucket: &str,
@@ -379,6 +391,66 @@ async fn manual_transition_run_with_max(
     Ok(serde_json::from_str(&body)?)
 }
 
+async fn manual_transition_async_run(
+    hot: &RustFSTestEnvironment,
+    bucket: &str,
+    prefix: &str,
+    dry_run: bool,
+    max_objects: u64,
+) -> Result<ManualTransitionRunResponse, Box<dyn std::error::Error + Send + Sync>> {
+    let bucket = urlencoding::encode(bucket);
+    let prefix = urlencoding::encode(prefix);
+    let tier = urlencoding::encode(TIER_NAME);
+    let path = format!(
+        "/rustfs/admin/v3/ilm/transition/run?bucket={bucket}&prefix={prefix}&tier={tier}&dryRun={dry_run}&maxObjects={max_objects}&mode=async"
+    );
+    let (status, body) = signed_admin_request(&hot.url, Method::POST, &path, None, &hot.access_key, &hot.secret_key).await?;
+    assert_eq!(status, reqwest::StatusCode::ACCEPTED, "async manual transition response: {body}");
+    Ok(serde_json::from_str(&body)?)
+}
+
+async fn manual_transition_job_status(
+    hot: &RustFSTestEnvironment,
+    status_endpoint: &str,
+) -> Result<ManualTransitionJobStatusResponse, Box<dyn std::error::Error + Send + Sync>> {
+    let (status, body) =
+        signed_admin_request(&hot.url, Method::GET, status_endpoint, None, &hot.access_key, &hot.secret_key).await?;
+    assert_eq!(status, reqwest::StatusCode::OK, "manual transition job status response: {body}");
+    Ok(serde_json::from_str(&body)?)
+}
+
+async fn manual_transition_job_cancel(
+    hot: &RustFSTestEnvironment,
+    status_endpoint: &str,
+) -> Result<ManualTransitionJobStatusResponse, Box<dyn std::error::Error + Send + Sync>> {
+    let (status, body) =
+        signed_admin_request(&hot.url, Method::DELETE, status_endpoint, None, &hot.access_key, &hot.secret_key).await?;
+    assert_eq!(status, reqwest::StatusCode::OK, "manual transition job cancel response: {body}");
+    Ok(serde_json::from_str(&body)?)
+}
+
+async fn wait_for_manual_transition_job_terminal(
+    hot: &RustFSTestEnvironment,
+    status_endpoint: &str,
+    deadline: StdDuration,
+) -> Result<ManualTransitionJobStatusResponse, Box<dyn std::error::Error + Send + Sync>> {
+    let start = Instant::now();
+    loop {
+        let status = manual_transition_job_status(hot, status_endpoint).await?;
+        if matches!(status.status.as_str(), "completed" | "partial" | "cancelled" | "failed" | "unknown") {
+            return Ok(status);
+        }
+        if start.elapsed() >= deadline {
+            return Err(format!(
+                "manual transition job at {status_endpoint} did not reach a terminal state within {}s; last={status:#?}",
+                deadline.as_secs()
+            )
+            .into());
+        }
+        tokio::time::sleep(StdDuration::from_millis(250)).await;
+    }
+}
+
 /// Number of objects currently stored in the cold-tier bucket.
 async fn cold_tier_object_count(cold_client: &Client) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
     let resp = cold_client.list_objects_v2().bucket(TIER_BUCKET).send().await?;
@@ -661,6 +733,86 @@ async fn test_manual_transition_run_black_box_semantics() -> TestResult {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn test_manual_transition_async_job_status_polling() -> TestResult {
+    let mut cold = RustFSTestEnvironment::new().await?;
+    cold.access_key = "manualasynccoldtieradmin".to_string();
+    cold.secret_key = "manualasynccoldtiersecret".to_string();
+    cold.start_rustfs_server_without_cleanup(vec![]).await?;
+    let cold_client = cold.create_s3_client();
+    cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
+
+    let mut hot = RustFSTestEnvironment::new().await?;
+    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
+        .await?;
+    let hot_client = hot.create_s3_client();
+    add_rustfs_tier(&hot, &cold).await?;
+
+    hot_client.create_bucket().bucket(MANUAL_ASYNC_STATUS_BUCKET).send().await?;
+    put_lifecycle_transition_rule(&hot_client, MANUAL_ASYNC_STATUS_BUCKET, "manual-async-status", "manual-async-status/", 1)
+        .await?;
+    put_single_part_object(
+        &hot_client,
+        MANUAL_ASYNC_STATUS_BUCKET,
+        MANUAL_ASYNC_STATUS_KEY,
+        b"manual async status not-yet-due dry-run object",
+    )
+    .await?;
+
+    let before_dry_run_remote_count = cold_tier_object_count(&cold_client).await?;
+    let accepted = manual_transition_async_run(&hot, MANUAL_ASYNC_STATUS_BUCKET, "manual-async-status/", true, 10).await?;
+    assert_eq!(accepted.state, "accepted");
+    assert_eq!(accepted.mode, "durable_job");
+    assert_eq!(accepted.report.bucket, MANUAL_ASYNC_STATUS_BUCKET);
+    assert_eq!(accepted.report.prefix, "manual-async-status/");
+    assert!(accepted.report.dry_run);
+    assert_eq!(accepted.report.scanned, 0);
+    assert_eq!(accepted.report.eligible, 0);
+    let job_id = accepted.job_id.as_deref().ok_or("async response must include job_id")?;
+    let status_endpoint = accepted
+        .status_endpoint
+        .as_deref()
+        .ok_or("async response must include status_endpoint")?;
+    assert!(
+        status_endpoint.ends_with(job_id),
+        "status endpoint must embed job id: endpoint={status_endpoint}, job_id={job_id}"
+    );
+
+    let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, StdDuration::from_secs(30)).await?;
+    assert_eq!(terminal.job_id, job_id);
+    assert_eq!(terminal.status_endpoint, status_endpoint);
+    assert_eq!(terminal.status, "completed", "terminal job response: {terminal:#?}");
+    assert!(!terminal.cancel_requested);
+    assert_eq!(terminal.failure_reason, None);
+    assert_eq!(terminal.report.bucket, MANUAL_ASYNC_STATUS_BUCKET);
+    assert_eq!(terminal.report.prefix, "manual-async-status/");
+    assert_eq!(terminal.report.tier.as_deref(), Some(TIER_NAME));
+    assert!(terminal.report.dry_run);
+    assert!(terminal.report.lifecycle_config_found);
+    assert_eq!(terminal.report.scanned, 1, "terminal job response: {terminal:#?}");
+    assert_eq!(terminal.report.eligible, 0, "terminal job response: {terminal:#?}");
+    assert_eq!(terminal.report.dry_run_eligible, 0, "terminal job response: {terminal:#?}");
+    assert_eq!(terminal.report.enqueued, 0, "terminal job response: {terminal:#?}");
+    assert_eq!(terminal.report.skipped_not_transition, 1, "terminal job response: {terminal:#?}");
+    assert_eq!(terminal.report.skipped_queue_full, 0);
+    assert_eq!(terminal.report.skipped_queue_closed, 0);
+    assert_eq!(terminal.report.skipped_queue_timeout, 0);
+    assert!(!terminal.report.truncated_by_limit);
+    assert!(!terminal.report.truncated_by_duration);
+
+    let after_cancel = manual_transition_job_cancel(&hot, status_endpoint).await?;
+    assert_eq!(after_cancel.status, "completed");
+    assert!(!after_cancel.cancel_requested);
+    assert_eq!(
+        cold_tier_object_count(&cold_client).await?,
+        before_dry_run_remote_count,
+        "not-yet-due async dry-run job must not write to cold tier"
+    );
+    assert_not_transitioned(&hot_client, MANUAL_ASYNC_STATUS_BUCKET, MANUAL_ASYNC_STATUS_KEY).await?;
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn test_manual_transition_run_contract_no_status_cancel_fields() -> TestResult {
     let mut cold = RustFSTestEnvironment::new().await?;