gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-07 04:25:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(heal): expose compatible canonical v3 outcomes (#7256)

Browse Source 
* feat(heal): expose compatible canonical v3 outcomes

Serialize the existing canonical heal outcome, retain the v3 summary vocabulary, and reject or conservatively adapt contradictory peer success responses. Preserve progress and bounded result cursors without treating legacy storage responses as repair proof.

Add optional SDK outcome and cursor support with shared Rust fixtures, pinned legacy Go decoder and mc polling checks, and explicit compatibility limits.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix(madmin): box heal stop task status outcome

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
2026-09-06 16:46:42 +08:00
committed by GitHub
parent 0ee5408b94
commit 5b962b6c58
9 changed files with 1300 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/heal/src/heal/channel.rs
+155 -19
View File
@@ -76,6 +76,8 @@ struct HealTaskStatusPayload<'a> {
min_seq: u64,
#[serde(skip_serializing_if = "Option::is_none")]
progress: Option<&'a HealProgress>,
#[serde(skip_serializing_if = "Option::is_none")]
outcome: Option<&'a super::outcome::HealTaskOutcome>,
}
fn u64_is_zero(value: &u64) -> bool {
@@ -87,17 +89,18 @@ fn encode_heal_task_status_payload(
mut items: Vec<HealResultItem>,
progress: Option<&HealProgress>,
mut truncated: bool,
next_seq: u64,
min_seq: u64,
sequence: (u64, u64),
outcome: Option<&super::outcome::HealTaskOutcome>,
) -> Result<(Vec<u8>, bool)> {
loop {
let data = serde_json::to_vec(&HealTaskStatusPayload {
summary,
items: &items,
truncated,
next_seq,
min_seq,
next_seq: sequence.0,
min_seq: sequence.1,
progress,
outcome,
})
.map_err(|e| Error::Serialization(format!("failed to serialize heal task status: {e}")))?;
if data.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE {
@@ -111,25 +114,21 @@ fn encode_heal_task_status_payload(
}
}
fn heal_status_detail(detail: Option<String>, truncated: bool) -> Option<String> {
if !truncated {
return detail;
}
let truncation = "heal result items were truncated";
Some(detail.map_or_else(|| truncation.to_string(), |detail| format!("{detail}; {truncation}")))
}
fn encode_heal_status_response(
summary: &str,
items: Vec<HealResultItem>,
progress: Option<&HealProgress>,
detail: Option<String>,
truncated: bool,
next_seq: u64,
min_seq: u64,
sequence: (u64, u64),
outcome: Option<&super::outcome::HealTaskOutcome>,
) -> Result<(Vec<u8>, Option<String>)> {
let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated, next_seq, min_seq)?;
Ok((data, heal_status_detail(detail, truncated)))
let (summary, detail) = match outcome {
Some(outcome) => outcome.legacy_status(summary, detail),
None => (summary, detail),
};
let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated, sequence, outcome)?;
Ok((data, super::outcome::heal_status_detail(detail, truncated)))
}
impl HealChannelProcessor {
@@ -439,6 +438,7 @@ impl HealChannelProcessor {
.await
};
let outcome = report.as_ref().ok().and_then(|report| report.outcome.clone());
let (summary, detail, items, truncated, progress, next_seq, min_seq) = match report {
Ok(HealTaskReport {
status: HealTaskStatus::Pending | HealTaskStatus::Running,
@@ -576,8 +576,15 @@ impl HealChannelProcessor {
}
};
let (data, detail) =
encode_heal_status_response(&summary, items, progress.as_ref(), detail, truncated, next_seq, min_seq)?;
let (data, detail) = encode_heal_status_response(
&summary,
items,
progress.as_ref(),
detail,
truncated,
(next_seq, min_seq),
outcome.as_deref(),
)?;
let response = HealChannelResponse {
request_id: client_token,
@@ -866,7 +873,7 @@ mod tests {
..Default::default()
}];
let (data, detail) = encode_heal_status_response("running", items, None, None, false, 0, 0).unwrap();
let (data, detail) = encode_heal_status_response("running", items, None, None, false, (0, 0), None).unwrap();
assert!(data.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE);
let payload: serde_json::Value = serde_json::from_slice(&data).unwrap();
@@ -875,6 +882,135 @@ mod tests {
assert_eq!(detail.as_deref(), Some("heal result items were truncated"));
}
#[test]
fn outcome_v3_fixture_matches_canonical_owner_and_preserves_legacy_terminals() {
use crate::heal::outcome::*;
let cases: serde_json::Value = serde_json::from_str(include_str!("../../../madmin/tests/fixtures/heal-outcome-v3.json"))
.expect("shared client fixtures");
for case in cases.as_array().expect("fixture cases") {
if case.get("remoteResponse").is_some() {
continue;
}
let mut outcome = HealTaskOutcome::default();
let name = case["name"].as_str().expect("case name");
if matches!(name, "unknown" | "completed_with_errors") {
let disposition = if name == "unknown" {
HealObjectDisposition::Unknown
} else {
outcome.attempt_failed();
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
};
outcome.record(HealObjectOutcome {
identity: HealObjectIdentity {
kind: HealObjectKind::Object,
bucket: "bucket".into(),
object: "object".into(),
version_id: None,
bucket_incarnation_id: None,
pool_index: None,
set_index: None,
},
disposition,
detail: None,
});
}
let abort = match name {
"cancelled" => Some(HealAbortReason::Cancelled),
"deadline" => Some(HealAbortReason::Deadline),
"untraversable" => Some(HealAbortReason::Untraversable),
_ => None,
};
outcome.finish(abort);
let expected = &case["response"];
let initial_detail = abort.map(|reason| {
match reason {
HealAbortReason::Cancelled => "heal task cancelled",
HealAbortReason::Deadline => "heal task timed out",
HealAbortReason::Untraversable => "heal listing is untraversable",
}
.to_string()
});
let (bytes, detail) = encode_heal_status_response(
if abort.is_some() { "stopped" } else { "finished" },
Vec::new(),
None,
initial_detail,
true,
(9, 4),
Some(&outcome),
)
.expect("canonical owner encoding");
let decoded: serde_json::Value = serde_json::from_slice(&bytes).expect("wire payload");
assert_eq!(decoded["summary"], expected["summary"], "{name}");
assert_eq!(detail.unwrap_or_default(), expected["detail"].as_str().expect("detail"), "{name}");
assert_eq!(decoded["outcome"], expected["outcome"], "{name}");
assert_eq!((decoded["next_seq"].as_u64(), decoded["min_seq"].as_u64()), (Some(9), Some(4)));
assert!(decoded["outcome"].get("retainedObjectBytes").is_none());
assert!(decoded["outcome"].get("untraversable").is_none());
}
}
#[test]
fn outcome_v3_abort_cannot_be_hidden_by_a_finished_status() {
use crate::heal::outcome::{HealAbortReason, HealTaskOutcome};
for reason in [
HealAbortReason::Cancelled,
HealAbortReason::Deadline,
HealAbortReason::Untraversable,
] {
let mut outcome = HealTaskOutcome::default();
outcome.finish(Some(reason));
let (data, detail) = encode_heal_status_response("finished", Vec::new(), None, None, false, (0, 0), Some(&outcome))
.expect("canonical abort adapter");
let json: serde_json::Value = serde_json::from_slice(&data).expect("public state");
assert_eq!(json["summary"], "stopped");
assert_eq!(json["outcome"]["execution"]["state"], "aborted");
assert!(detail.is_some());
}
}
#[test]
fn outcome_v3_payload_bound_keeps_cumulative_outcome_and_cursors() {
use crate::heal::outcome::*;
let mut outcome = HealTaskOutcome::default();
outcome.start();
for index in 0..256 {
outcome.record(HealObjectOutcome {
identity: HealObjectIdentity {
kind: HealObjectKind::Object,
bucket: "bucket".into(),
object: format!("object-{index}"),
version_id: None,
bucket_incarnation_id: None,
pool_index: None,
set_index: None,
},
disposition: HealObjectDisposition::Unknown,
detail: Some("\"".repeat(1024)),
});
}
let retained = outcome.objects.len();
let items = vec![
HealResultItem::default(),
HealResultItem {
detail: "x".repeat(MAX_HEAL_STATUS_PAYLOAD_SIZE + 1),
..Default::default()
},
];
let (bytes, detail) = encode_heal_status_response("running", items, None, None, false, (9, 4), Some(&outcome))
.expect("bounded status with cumulative outcome");
assert!(bytes.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE);
let wire: serde_json::Value = serde_json::from_slice(&bytes).expect("bounded payload");
assert_eq!(wire["items"].as_array().expect("items").len(), 1);
assert_eq!(wire["truncated"], true);
assert_eq!((wire["next_seq"].as_u64(), wire["min_seq"].as_u64()), (Some(9), Some(4)));
assert_eq!(wire["outcome"]["counters"]["processed"], 256);
assert_eq!(wire["outcome"]["counters"]["healed"], 0);
assert_eq!(wire["outcome"]["objects"].as_array().expect("outcome window").len(), retained);
assert!(retained < 256 && outcome.objects_truncated);
assert_eq!(detail.as_deref(), Some("heal result items were truncated"));
}
#[test]
fn admission_response_preserves_all_admission_outcomes() {
let cases = [
crates/heal/src/heal/outcome.rs
+179 -11
View File
@@ -15,6 +15,7 @@
//! Execution results are separate from repair responsibility. A legacy
//! successful storage call supplies no authoritative repair receipt.
use serde::{Deserialize, Serialize};
use std::{collections::VecDeque, time::SystemTime};
use uuid::Uuid;
@@ -22,14 +23,16 @@ const MAX_OUTCOME_ITEMS: usize = 128;
const MAX_OUTCOME_BYTES: usize = 64 * 1024;
const MAX_OUTCOME_DETAIL_BYTES: usize = 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum HealObjectKind {
Object,
Metadata,
Decode,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct HealObjectIdentity {
pub kind: HealObjectKind,
pub bucket: String,
@@ -41,7 +44,8 @@ pub struct HealObjectIdentity {
pub set_index: Option<usize>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum HealDeferredReason {
DanglingDeleteGrace,
TransientUsageCache,
@@ -49,14 +53,21 @@ pub enum HealDeferredReason {
Deadline,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum HealFailureClass {
Recoverable,
RetryExhausted,
Permanent,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(
tag = "state",
content = "details",
rename_all = "snake_case",
rename_all_fields = "camelCase"
)]
pub enum HealObjectDisposition {
/// The legacy storage response does not prove the requested check or commit.
Unknown,
@@ -72,7 +83,8 @@ pub enum HealObjectDisposition {
DryRunObserved,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct HealObjectOutcome {
pub identity: HealObjectIdentity,
pub disposition: HealObjectDisposition,
@@ -89,7 +101,8 @@ impl HealObjectOutcome {
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum HealTraversalCoverage {
#[default]
Unknown,
@@ -97,14 +110,16 @@ pub enum HealTraversalCoverage {
Complete,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum HealAbortReason {
Cancelled,
Deadline,
Untraversable,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "state", content = "reason", rename_all = "snake_case")]
pub enum HealExecutionOutcome {
#[default]
Pending,
@@ -114,7 +129,8 @@ pub enum HealExecutionOutcome {
Aborted(HealAbortReason),
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct HealOutcomeCounters {
pub processed: u64,
pub healed: u64,
@@ -127,7 +143,8 @@ pub struct HealOutcomeCounters {
pub overflowed: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct HealTaskOutcome {
pub execution: HealExecutionOutcome,
pub coverage: HealTraversalCoverage,
@@ -135,11 +152,99 @@ pub struct HealTaskOutcome {
/// A bounded diagnostic window, not a complete responsibility ledger.
pub objects: VecDeque<HealObjectOutcome>,
pub objects_truncated: bool,
#[serde(skip)]
retained_object_bytes: usize,
#[serde(skip)]
untraversable: bool,
}
#[derive(Debug, thiserror::Error)]
pub enum HealOutcomeWireError {
#[error("heal outcome is missing execution or counters")]
MissingFields,
#[error("heal outcome has invalid or unsupported execution fields")]
InvalidFields(#[from] serde_json::Error),
#[error("finished heal summary contradicts its canonical outcome")]
ContradictoryCompletion,
}
/// Reconcile a peer's successful legacy summary using the canonical owner types.
/// A running retry may legitimately retain the preceding attempt's outcome.
pub fn legacy_wire_status<'a>(
summary: &'a str,
wire: &serde_json::Value,
truncated: bool,
) -> Result<(&'a str, Option<String>), HealOutcomeWireError> {
if summary != "finished" {
return Ok((summary, None));
}
let execution = HealExecutionOutcome::deserialize(wire.get("execution").ok_or(HealOutcomeWireError::MissingFields)?)?;
let counters = HealOutcomeCounters::deserialize(wire.get("counters").ok_or(HealOutcomeWireError::MissingFields)?)?;
if matches!(execution, HealExecutionOutcome::Pending | HealExecutionOutcome::Running)
|| (execution == HealExecutionOutcome::Completed && counters.failed > 0)
{
return Err(HealOutcomeWireError::ContradictoryCompletion);
}
let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters);
Ok((
adapted,
if adapted != summary {
heal_status_detail(detail, truncated)
} else {
None
},
))
}
pub(crate) fn heal_status_detail(detail: Option<String>, truncated: bool) -> Option<String> {
if !truncated {
return detail;
}
let truncation = "heal result items were truncated";
Some(detail.map_or_else(|| truncation.to_string(), |detail| format!("{detail}; {truncation}")))
}
fn legacy_execution_status<'a>(
summary: &'a str,
detail: Option<String>,
execution: HealExecutionOutcome,
counters: &HealOutcomeCounters,
) -> (&'a str, Option<String>) {
if summary != "finished" {
return (summary, detail);
}
match execution {
HealExecutionOutcome::CompletedWithErrors => (
"stopped",
Some(format!("heal traversal completed with errors: {} failed objects", counters.failed)),
),
HealExecutionOutcome::Aborted(reason) => {
let reason = match reason {
HealAbortReason::Cancelled => "cancelled",
HealAbortReason::Deadline => "timed out",
HealAbortReason::Untraversable => "untraversable",
};
("stopped", Some(format!("heal task {reason}")))
}
HealExecutionOutcome::Completed if counters.unknown > 0 => (
summary,
Some(format!(
"heal traversal completed; authoritative storage proof is unavailable for {} objects",
counters.unknown
)),
),
HealExecutionOutcome::Pending | HealExecutionOutcome::Running => {
("running", Some("heal execution has not reached a terminal outcome".to_string()))
}
HealExecutionOutcome::Completed => (summary, detail),
}
}
impl HealTaskOutcome {
pub(crate) fn legacy_status<'a>(&self, summary: &'a str, detail: Option<String>) -> (&'a str, Option<String>) {
legacy_execution_status(summary, detail, self.execution, &self.counters)
}
pub(crate) fn start(&mut self) {
if self.execution != HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) {
self.execution = HealExecutionOutcome::Running;
@@ -292,6 +397,69 @@ mod canonical_outcome_tests {
assert!(outcome.objects.len() < MAX_OUTCOME_ITEMS);
}
#[test]
fn outcome_v3_serialization_keeps_unverified_dispositions_and_window_bounds() {
let mut outcome = HealTaskOutcome::default();
for disposition in [
HealObjectDisposition::Unknown,
HealObjectDisposition::Deferred {
reason: HealDeferredReason::DanglingDeleteGrace,
retry_not_before: None,
},
HealObjectDisposition::DryRunObserved,
] {
outcome.record(item(disposition));
}
outcome.finish(None);
let wire = serde_json::to_value(&outcome).expect("canonical wire view");
assert_eq!(wire["execution"]["state"], "completed");
assert_eq!(wire["counters"]["healed"], 0);
assert_eq!(wire["counters"]["skipped"], 3);
assert_eq!(wire["objects"][1]["disposition"]["details"]["reason"], "dangling_delete_grace");
assert!(wire["objects"][1]["identity"]["bucketIncarnationId"].is_null());
for _ in 0..MAX_OUTCOME_ITEMS + 1 {
let mut result = item(HealObjectDisposition::Unknown);
result.detail = Some("\"".repeat(MAX_OUTCOME_DETAIL_BYTES));
outcome.record(result);
}
let bytes = serde_json::to_vec(&outcome).expect("bounded canonical samples");
assert!(
bytes.len() < 8 * MAX_OUTCOME_BYTES,
"JSON escaping remains bounded independently of object count"
);
assert!(outcome.objects_truncated);
}
#[test]
fn outcome_v3_wire_consistency_rejects_unknown_success_without_rejecting_extensions() {
let mut outcome = HealTaskOutcome::default();
outcome.finish(None);
let mut wire = serde_json::to_value(&outcome).expect("canonical snapshot");
wire["execution"]["futureField"] = serde_json::json!({"new": true});
wire["counters"]["futureCounter"] = serde_json::json!(42);
assert_eq!(
legacy_wire_status("finished", &wire, false).expect("unknown extension fields"),
("finished", None)
);
wire["execution"]["state"] = serde_json::json!("future_execution");
assert!(legacy_wire_status("finished", &wire, false).is_err());
assert_eq!(
legacy_wire_status("running", &wire, false).expect("unknown nonterminal outcome"),
("running", None)
);
wire["execution"] = serde_json::json!({"state":"completed"});
wire["counters"]["failed"] = serde_json::json!(1);
assert!(matches!(
legacy_wire_status("finished", &wire, false),
Err(HealOutcomeWireError::ContradictoryCompletion)
));
wire.as_object_mut().expect("object").remove("execution");
assert!(matches!(
legacy_wire_status("finished", &wire, false),
Err(HealOutcomeWireError::MissingFields)
));
}
#[test]
fn canonical_outcome_counter_overflow_cannot_claim_complete_coverage() {
let mut outcome = HealTaskOutcome::default();
crates/madmin/src/client.rs
+82 -4
View File
@@ -167,6 +167,13 @@ pub struct HealTaskStatus {
/// Live progress snapshot; the exact shape is owned by the heal runtime.
#[serde(default)]
pub progress: Option<serde_json::Value>,
/// Canonical heal-owner result. Missing or future states are not repair proof.
#[serde(default)]
pub outcome: Option<serde_json::Value>,
#[serde(default, alias = "next_seq")]
pub next_seq: Option<u64>,
#[serde(default, alias = "min_seq")]
pub min_seq: Option<u64>,
}
/// `POST /v3/background-heal/status` response. Known top-level fields are
@@ -358,8 +365,22 @@ impl AdminClient {
prefix: Option<&str>,
client_token: &str,
) -> Result<HealTaskStatus, AdminClientError> {
self.post_json(&heal_path(bucket, prefix), &[("clientToken", client_token.to_string())], Vec::new())
.await
self.heal_status_since(bucket, prefix, client_token, None).await
}
/// Query a retained result window. Missing cursors and outcome remain unknown.
pub async fn heal_status_since(
&self,
bucket: Option<&str>,
prefix: Option<&str>,
client_token: &str,
since_seq: Option<u64>,
) -> Result<HealTaskStatus, AdminClientError> {
let mut query = vec![("clientToken", client_token.to_string())];
if let Some(since_seq) = since_seq {
query.push(("sinceSeq", since_seq.to_string()));
}
self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await
}
/// Stop a heal: with a `client_token` only that task is cancelled and its
@@ -378,7 +399,7 @@ impl AdminClient {
match client_token {
Some(_) => {
let status: HealTaskStatus = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
Ok(HealStopOutcome::Stopped(status))
Ok(HealStopOutcome::Stopped(Box::new(status)))
}
None => {
let success: HealStartSuccess = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
@@ -533,7 +554,7 @@ impl AdminClient {
/// start-success-shaped receipt.
#[derive(Debug, Clone)]
pub enum HealStopOutcome {
Stopped(HealTaskStatus),
Stopped(Box<HealTaskStatus>),
PathStopped(HealStartSuccess),
}
@@ -635,6 +656,24 @@ mod tests {
assert!(status.progress.is_none());
}
#[test]
fn outcome_v3_decoder_preserves_canonical_unknown_and_future_fields() {
let cases: serde_json::Value =
serde_json::from_str(include_str!("../tests/fixtures/heal-outcome-v3.json")).expect("shared fixtures");
for case in cases.as_array().expect("cases") {
let status: HealTaskStatus = serde_json::from_value(case["response"].clone()).expect("optional outcome response");
assert_eq!(status.outcome.as_ref(), Some(&case["response"]["outcome"]));
assert_eq!((status.next_seq, status.min_seq), (Some(9), Some(4)));
assert!(status.truncated);
}
let old: HealTaskStatus = serde_json::from_value(json!({"summary":"finished"})).expect("legacy response");
assert!(old.outcome.is_none() && old.next_seq.is_none() && old.min_seq.is_none());
let future = json!({"execution":{"state":"future_state"},"newField":7});
let status: HealTaskStatus =
serde_json::from_value(json!({"summary":"running","outcome":future})).expect("future outcome remains opaque");
assert_eq!(status.outcome, Some(future));
}
#[test]
fn background_heal_status_types_known_fields_and_passes_the_rest_through() {
let raw = json!({
@@ -750,6 +789,26 @@ mod tests {
assert!(!request.query.contains("forceStop"));
}
#[tokio::test]
async fn outcome_v3_since_query_preserves_cursor_and_never_sends_force_start() {
let server = TestServer::spawn(
r#"{"summary":"running","nextSeq":9,"minSeq":4,"truncated":true,"outcome":{"execution":{"state":"future_state"}}}"#,
200,
)
.await;
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").expect("test client");
let status = client
.heal_status_since(Some("bucket"), None, "token-1", Some(3))
.await
.expect("window response");
assert_eq!((status.next_seq, status.min_seq), (Some(9), Some(4)));
assert!(status.truncated);
assert_eq!(status.outcome.expect("future state is preserved")["execution"]["state"], "future_state");
let request = server.recorded();
assert!(request.query.contains("sinceSeq=3") && request.query.contains("clientToken=token-1"));
assert!(!request.query.contains("forceStart") && !request.query.contains("forceStop"));
}
#[tokio::test]
async fn stop_without_token_takes_the_path_cancel_branch() {
let server = TestServer::spawn(r#"{"clientToken":"path","clientAddress":"c","startTime":"t"}"#, 200).await;
@@ -762,6 +821,25 @@ mod tests {
assert!(!request.query.contains("clientToken"));
}
#[tokio::test]
async fn stop_with_token_decodes_boxed_task_status() {
let body = r#"{"summary":"stopped","detail":"","settings":{"recursive":false},"items":[],"truncated":false}"#;
let server = TestServer::spawn(body, 200).await;
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
let outcome = client
.heal_stop(Some("bucket"), None, Some("token-1"))
.await
.expect("token stop decodes");
let super::HealStopOutcome::Stopped(status) = outcome else {
panic!("token stop should return task status");
};
assert_eq!(status.summary, "stopped");
let request = server.recorded();
assert!(request.query.contains("forceStop=true"));
assert!(request.query.contains("clientToken=token-1"));
}
#[tokio::test]
async fn background_heal_status_posts_to_the_registered_route() {
let body = r#"{"state":"idle","healQueueLength":0,"healActiveTasks":0,"clusterStatusComplete":true}"#;
crates/madmin/tests/fixtures/heal-outcome-v3.json
Vendored
+472
View File
@@ -0,0 +1,472 @@
[
{
"name": "completed",
"cliExit": 0,
"response": {
"summary": "finished",
"detail": "heal result items were truncated",
"startTime": "2026-01-01T00:00:00Z",
"settings": {
"recursive": true,
"scanMode": 1
},
"items": [],
"truncated": true,
"nextSeq": 9,
"minSeq": 4,
"progress": {
"objectsScanned": 11,
"objectsHealed": 7
},
"outcome": {
"execution": {
"state": "completed"
},
"coverage": "complete",
"counters": {
"processed": 0,
"healed": 0,
"unchanged": 0,
"skipped": 0,
"failed": 0,
"unknown": 0,
"attemptFailures": 0,
"overflowed": false
},
"objects": [],
"objectsTruncated": false
}
}
},
{
"name": "unknown",
"cliExit": 0,
"response": {
"summary": "finished",
"detail": "heal traversal completed; authoritative storage proof is unavailable for 1 objects; heal result items were truncated",
"startTime": "2026-01-01T00:00:00Z",
"settings": {
"recursive": true,
"scanMode": 1
},
"items": [],
"truncated": true,
"nextSeq": 9,
"minSeq": 4,
"progress": {
"objectsScanned": 11,
"objectsHealed": 7
},
"outcome": {
"execution": {
"state": "completed"
},
"coverage": "complete",
"counters": {
"processed": 1,
"healed": 0,
"unchanged": 0,
"skipped": 1,
"failed": 0,
"unknown": 1,
"attemptFailures": 0,
"overflowed": false
},
"objects": [
{
"identity": {
"kind": "object",
"bucket": "bucket",
"object": "object",
"versionId": null,
"bucketIncarnationId": null,
"poolIndex": null,
"setIndex": null
},
"disposition": {
"state": "unknown"
},
"detail": null
}
],
"objectsTruncated": false
}
}
},
{
"name": "completed_with_errors",
"cliExit": 1,
"response": {
"summary": "stopped",
"detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated",
"startTime": "2026-01-01T00:00:00Z",
"settings": {
"recursive": true,
"scanMode": 1
},
"items": [],
"truncated": true,
"nextSeq": 9,
"minSeq": 4,
"progress": {
"objectsScanned": 11,
"objectsHealed": 7
},
"outcome": {
"execution": {
"state": "completed_with_errors"
},
"coverage": "complete",
"counters": {
"processed": 1,
"healed": 0,
"unchanged": 0,
"skipped": 0,
"failed": 1,
"unknown": 0,
"attemptFailures": 1,
"overflowed": false
},
"objects": [
{
"identity": {
"kind": "object",
"bucket": "bucket",
"object": "object",
"versionId": null,
"bucketIncarnationId": null,
"poolIndex": null,
"setIndex": null
},
"disposition": {
"state": "failed",
"details": "retry_exhausted"
},
"detail": null
}
],
"objectsTruncated": false
}
}
},
{
"name": "cancelled",
"cliExit": 1,
"response": {
"summary": "stopped",
"detail": "heal task cancelled; heal result items were truncated",
"startTime": "2026-01-01T00:00:00Z",
"settings": {
"recursive": true,
"scanMode": 1
},
"items": [],
"truncated": true,
"nextSeq": 9,
"minSeq": 4,
"progress": {
"objectsScanned": 11,
"objectsHealed": 7
},
"outcome": {
"execution": {
"state": "aborted",
"reason": "cancelled"
},
"coverage": "partial",
"counters": {
"processed": 0,
"healed": 0,
"unchanged": 0,
"skipped": 0,
"failed": 0,
"unknown": 0,
"attemptFailures": 0,
"overflowed": false
},
"objects": [],
"objectsTruncated": false
}
}
},
{
"name": "deadline",
"cliExit": 1,
"response": {
"summary": "stopped",
"detail": "heal task timed out; heal result items were truncated",
"startTime": "2026-01-01T00:00:00Z",
"settings": {
"recursive": true,
"scanMode": 1
},
"items": [],
"truncated": true,
"nextSeq": 9,
"minSeq": 4,
"progress": {
"objectsScanned": 11,
"objectsHealed": 7
},
"outcome": {
"execution": {
"state": "aborted",
"reason": "deadline"
},
"coverage": "partial",
"counters": {
"processed": 0,
"healed": 0,
"unchanged": 0,
"skipped": 0,
"failed": 0,
"unknown": 0,
"attemptFailures": 0,
"overflowed": false
},
"objects": [],
"objectsTruncated": false
}
}
},
{
"name": "untraversable",
"cliExit": 1,
"response": {
"summary": "stopped",
"detail": "heal listing is untraversable; heal result items were truncated",
"startTime": "2026-01-01T00:00:00Z",
"settings": {
"recursive": true,
"scanMode": 1
},
"items": [],
"truncated": true,
"nextSeq": 9,
"minSeq": 4,
"progress": {
"objectsScanned": 11,
"objectsHealed": 7
},
"outcome": {
"execution": {
"state": "aborted",
"reason": "untraversable"
},
"coverage": "partial",
"counters": {
"processed": 0,
"healed": 0,
"unchanged": 0,
"skipped": 0,
"failed": 0,
"unknown": 0,
"attemptFailures": 0,
"overflowed": false
},
"objects": [],
"objectsTruncated": false
}
}
},
{
"name": "remote_completed_with_errors",
"cliExit": 1,
"response": {
"summary": "stopped",
"detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated",
"startTime": "2026-01-01T00:00:00Z",
"settings": {
"recursive": true,
"scanMode": 1
},
"items": [],
"truncated": true,
"nextSeq": 9,
"minSeq": 4,
"progress": {
"objectsScanned": 11,
"objectsHealed": 7
},
"outcome": {
"execution": {
"state": "completed_with_errors",
"futureExtension": {
"value": 7
}
},
"coverage": "complete",
"counters": {
"processed": 1,
"healed": 0,
"unchanged": 0,
"skipped": 0,
"failed": 1,
"unknown": 0,
"attemptFailures": 1,
"overflowed": false,
"futureCounter": 11
},
"objects": [
{
"identity": {
"kind": "object",
"bucket": "bucket",
"object": "object",
"versionId": null,
"bucketIncarnationId": null,
"poolIndex": null,
"setIndex": null
},
"disposition": {
"state": "failed",
"details": "retry_exhausted"
},
"detail": null
}
],
"objectsTruncated": false
}
},
"remoteResponse": {
"summary": "finished",
"detail": "",
"startTime": "2026-01-01T00:00:00Z",
"settings": {
"recursive": true,
"scanMode": 1
},
"items": [],
"truncated": true,
"nextSeq": 9,
"minSeq": 4,
"progress": {
"objectsScanned": 11,
"objectsHealed": 7
},
"outcome": {
"execution": {
"state": "completed_with_errors",
"futureExtension": {
"value": 7
}
},
"coverage": "complete",
"counters": {
"processed": 1,
"healed": 0,
"unchanged": 0,
"skipped": 0,
"failed": 1,
"unknown": 0,
"attemptFailures": 1,
"overflowed": false,
"futureCounter": 11
},
"objects": [
{
"identity": {
"kind": "object",
"bucket": "bucket",
"object": "object",
"versionId": null,
"bucketIncarnationId": null,
"poolIndex": null,
"setIndex": null
},
"disposition": {
"state": "failed",
"details": "retry_exhausted"
},
"detail": null
}
],
"objectsTruncated": false
}
}
},
{
"name": "remote_cancelled",
"cliExit": 1,
"response": {
"summary": "stopped",
"detail": "heal task cancelled; heal result items were truncated",
"startTime": "2026-01-01T00:00:00Z",
"settings": {
"recursive": true,
"scanMode": 1
},
"items": [],
"truncated": true,
"nextSeq": 9,
"minSeq": 4,
"progress": {
"objectsScanned": 11,
"objectsHealed": 7
},
"outcome": {
"execution": {
"state": "aborted",
"reason": "cancelled",
"futureExtension": {
"value": 7
}
},
"coverage": "partial",
"counters": {
"processed": 0,
"healed": 0,
"unchanged": 0,
"skipped": 0,
"failed": 0,
"unknown": 0,
"attemptFailures": 0,
"overflowed": false,
"futureCounter": 11
},
"objects": [],
"objectsTruncated": false
}
},
"remoteResponse": {
"summary": "finished",
"detail": "",
"startTime": "2026-01-01T00:00:00Z",
"settings": {
"recursive": true,
"scanMode": 1
},
"items": [],
"truncated": true,
"nextSeq": 9,
"minSeq": 4,
"progress": {
"objectsScanned": 11,
"objectsHealed": 7
},
"outcome": {
"execution": {
"state": "aborted",
"reason": "cancelled",
"futureExtension": {
"value": 7
}
},
"coverage": "partial",
"counters": {
"processed": 0,
"healed": 0,
"unchanged": 0,
"skipped": 0,
"failed": 0,
"unknown": 0,
"attemptFailures": 0,
"overflowed": false,
"futureCounter": 11
},
"objects": [],
"objectsTruncated": false
}
}
}
]