gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-18 10:43:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(heal): harden resumable set repair failures (#5693)

Browse Source 
* fix(heal): enforce resumable task control

* fix(ecstore): surface bucket and metadata heal errors

* chore: refresh guardrail path references

---------

Signed-off-by: cxymds <cxymds@gmail.com>
This commit is contained in:
cxymds
2026-08-04 20:35:58 +08:00
committed by GitHub
parent 3c8bd5b929
commit 31959b90db
15 changed files with 1276 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/protos/src/heal_control.rs
+32 -9
View File
@@ -27,7 +27,7 @@ use rustfs_common::heal_channel::{
use serde::{Deserialize, Serialize, de::SeqAccess, de::Visitor};
use std::{fmt, io::Cursor, io::Write};
const ENVELOPE_VERSION: u8 = 1;
const ENVELOPE_VERSION: u8 = 2;
pub const ENVELOPE_MAX_SIZE: usize = 64 * 1024;
pub const RESULT_MAX_SIZE: usize = 16 * 1024 * 1024;
pub const NONCE_SIZE: usize = 16;
@@ -101,6 +101,8 @@ pub struct StartCommand {
update_parity: Option<bool>,
recursive: Option<bool>,
dry_run: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
no_lock: Option<bool>,
timeout_seconds: Option<u64>,
source: HealRequestSource,
}
@@ -132,6 +134,7 @@ impl TryFrom<HealChannelRequest> for StartCommand {
update_parity: request.update_parity,
recursive: request.recursive,
dry_run: request.dry_run,
no_lock: request.no_lock,
timeout_seconds: request.timeout_seconds,
source: request.source,
})
@@ -164,6 +167,7 @@ impl StartCommand {
update_parity: self.update_parity,
recursive: self.recursive,
dry_run: self.dry_run,
no_lock: self.no_lock,
timeout_seconds: self.timeout_seconds,
source: self.source,
})
@@ -557,8 +561,8 @@ fn encode_bounded(value: &impl Serialize, value_name: &str, max_size: usize) ->
#[cfg(test)]
mod tests {
use super::{
Admission, ENVELOPE_MAX_SIZE, Envelope, Outcome, RESULT_MAX_SIZE, RequestMetadata, ResultEnvelope, decode_envelope,
decode_result, encode_result,
Admission, ENVELOPE_MAX_SIZE, Envelope, ExecutableCommand, Outcome, RESULT_MAX_SIZE, RequestMetadata, ResultEnvelope,
decode_envelope, decode_result, encode_result,
};
use rustfs_common::heal_channel::{HealChannelRequest, HealChannelResponse, HealRequestSource};
use serde::de::{DeserializeSeed, SeqAccess, Visitor, value::Error as ValueError};
@@ -632,7 +636,26 @@ mod tests {
}
#[test]
fn v1_query_and_result_wire_fixtures_are_stable() {
fn v2_start_round_trips_no_lock() {
for no_lock in [None, Some(false), Some(true)] {
let request_id = uuid::Uuid::new_v4().to_string();
let mut request = test_request(request_id);
request.no_lock = no_lock;
let envelope = Envelope::start(request, metadata(1, 7)).expect("v2 start should encode nolock");
let encoded = super::encode_envelope(&envelope).expect("v2 start should serialize");
let decoded = super::decode_envelope(&encoded).expect("v2 start should deserialize");
let (_, _, ExecutableCommand::Start { request }) = decoded.into_execution().expect("v2 start command should decode")
else {
panic!("expected start command");
};
assert_eq!(request.no_lock, no_lock);
}
}
#[test]
fn v2_command_and_result_wire_fixtures_are_stable() {
let request_id = "00112233-4455-6677-8899-aabbccddeeff".to_string();
let start = Envelope::start(
test_request(request_id.clone()),
@@ -700,23 +723,23 @@ mod tests {
.collect::<String>();
assert_eq!(
start_hex,
"87a776657273696f6e01a9726571756573744964d92430303131323233332d343435352d363637372d383839392d616162626363646465656666a56e6f6e6365dc001011111111111111111111111111111111ae6973737565644174556e69784d73cf0000018bcfe56800af657870697265734174556e69784d73cf0000018bcfe5dd30b0636f6f7264696e61746f7245706f636809a7636f6d6d616e6482a6616374696f6ea57374617274a772657175657374de0010a46469736bc0a66275636b6574a66275636b6574ac6f626a656374507265666978a6707265666978af6f626a65637456657273696f6e4964c0aa666f7263655374617274c2a87072696f72697479a66e6f726d616ca9706f6f6c496e64657801a8736574496e64657802a87363616e4d6f6465c0af72656d6f7665436f72727570746564c0af72656372656174654d697373696e67c0ac757064617465506172697479c0a9726563757273697665c0a664727952756ec0ae74696d656f75745365636f6e6473c0a6736f75726365a561646d696e"
"87a776657273696f6e02a9726571756573744964d92430303131323233332d343435352d363637372d383839392d616162626363646465656666a56e6f6e6365dc001011111111111111111111111111111111ae6973737565644174556e69784d73cf0000018bcfe56800af657870697265734174556e69784d73cf0000018bcfe5dd30b0636f6f7264696e61746f7245706f636809a7636f6d6d616e6482a6616374696f6ea57374617274a772657175657374de0010a46469736bc0a66275636b6574a66275636b6574ac6f626a656374507265666978a6707265666978af6f626a65637456657273696f6e4964c0aa666f7263655374617274c2a87072696f72697479a66e6f726d616ca9706f6f6c496e64657801a8736574496e64657802a87363616e4d6f6465c0af72656d6f7665436f72727570746564c0af72656372656174654d697373696e67c0ac757064617465506172697479c0a9726563757273697665c0a664727952756ec0ae74696d656f75745365636f6e6473c0a6736f75726365a561646d696e"
);
assert_eq!(
cancel_hex,
"87a776657273696f6e01a9726571756573744964d92430303131323233332d343435352d363637372d383839392d616162626363646465656666a56e6f6e6365dc001011111111111111111111111111111111ae6973737565644174556e69784d73cf0000018bcfe56800af657870697265734174556e69784d73cf0000018bcfe5dd30b0636f6f7264696e61746f7245706f636809a7636f6d6d616e6483a6616374696f6ea663616e63656ca96865616c5f70617468ad6275636b65742f707265666978ac636c69656e745f746f6b656eac636c69656e742d746f6b656e"
"87a776657273696f6e02a9726571756573744964d92430303131323233332d343435352d363637372d383839392d616162626363646465656666a56e6f6e6365dc001011111111111111111111111111111111ae6973737565644174556e69784d73cf0000018bcfe56800af657870697265734174556e69784d73cf0000018bcfe5dd30b0636f6f7264696e61746f7245706f636809a7636f6d6d616e6483a6616374696f6ea663616e63656ca96865616c5f70617468ad6275636b65742f707265666978ac636c69656e745f746f6b656eac636c69656e742d746f6b656e"
);
assert_eq!(
start_result_hex,
"84a776657273696f6e01a9726571756573744964d92430303131323233332d343435352d363637372d383839392d616162626363646465656666b0636f6f7264696e61746f7245706f636809a76f7574636f6d6583a6726573756c74a57374617274a77461736b5f6964d92466666565646463632d626261612d393938382d373736362d353534343333323231313030a961646d697373696f6eae64726f707065645f706f6c696379"
"84a776657273696f6e02a9726571756573744964d92430303131323233332d343435352d363637372d383839392d616162626363646465656666b0636f6f7264696e61746f7245706f636809a76f7574636f6d6583a6726573756c74a57374617274a77461736b5f6964d92466666565646463632d626261612d393938382d373736362d353534343333323231313030a961646d697373696f6eae64726f707065645f706f6c696379"
);
assert_eq!(
envelope_hex,
"87a776657273696f6e01a9726571756573744964d92430303131323233332d343435352d363637372d383839392d616162626363646465656666a56e6f6e6365dc001011111111111111111111111111111111ae6973737565644174556e69784d73cf0000018bcfe56800af657870697265734174556e69784d73cf0000018bcfe5dd30b0636f6f7264696e61746f7245706f636809a7636f6d6d616e6483a6616374696f6ea57175657279a96865616c5f70617468ad6275636b65742f707265666978ac636c69656e745f746f6b656eac636c69656e742d746f6b656e"
"87a776657273696f6e02a9726571756573744964d92430303131323233332d343435352d363637372d383839392d616162626363646465656666a56e6f6e6365dc001011111111111111111111111111111111ae6973737565644174556e69784d73cf0000018bcfe56800af657870697265734174556e69784d73cf0000018bcfe5dd30b0636f6f7264696e61746f7245706f636809a7636f6d6d616e6483a6616374696f6ea57175657279a96865616c5f70617468ad6275636b65742f707265666978ac636c69656e745f746f6b656eac636c69656e742d746f6b656e"
);
assert_eq!(
result_hex,
"84a776657273696f6e01a9726571756573744964d92430303131323233332d343435352d363637372d383839392d616162626363646465656666b0636f6f7264696e61746f7245706f636809a76f7574636f6d6584a6726573756c74a76368616e6e656ca773756363657373c3a46461746193010203a56572726f72c0"
"84a776657273696f6e02a9726571756573744964d92430303131323233332d343435352d363637372d383839392d616162626363646465656666b0636f6f7264696e61746f7245706f636809a76f7574636f6d6584a6726573756c74a76368616e6e656ca773756363657373c3a46461746193010203a56572726f72c0"
);
}
crates/protos/src/lib.rs
+9 -9
View File
@@ -168,9 +168,9 @@ pub fn internode_rpc_max_message_size() -> usize {
}
pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SIZE + 1024;
pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2;
pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 3;
pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1;
pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0";
pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v3\0";
pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0";
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
@@ -250,7 +250,7 @@ pub fn canonical_heal_control_request_body(
topology_fingerprint: &str,
command: &[u8],
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-heal-control-v2\0";
const DOMAIN: &[u8] = b"rustfs-heal-control-v3\0";
let fingerprint = topology_fingerprint.as_bytes();
let mut body = Vec::with_capacity(DOMAIN.len() + 4 + 8 + fingerprint.len() + 8 + command.len());
@@ -270,7 +270,7 @@ pub fn canonical_heal_control_capability_ack(
topology_fingerprint: &str,
probe: &[u8],
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-heal-control-capability-ack-v2\0";
const DOMAIN: &[u8] = b"rustfs-heal-control-capability-ack-v3\0";
let fingerprint = topology_fingerprint.as_bytes();
let mut body = Vec::with_capacity(DOMAIN.len() + 4 + 8 + fingerprint.len() + 8 + probe.len());
@@ -289,7 +289,7 @@ pub fn canonical_heal_control_response_body(
command: &[u8],
result: &[u8],
) -> Result<Vec<u8>, std::num::TryFromIntError> {
const DOMAIN: &[u8] = b"rustfs-heal-control-response-v2\0";
const DOMAIN: &[u8] = b"rustfs-heal-control-response-v3\0";
let fingerprint = topology_fingerprint.as_bytes();
let mut body = Vec::with_capacity(DOMAIN.len() + 4 + 8 + fingerprint.len() + 8 + command.len() + 8 + result.len());
@@ -1713,7 +1713,7 @@ mod heal_control_tests {
#[test]
fn canonical_heal_control_body_binds_every_field_and_boundary() {
let baseline = canonical_heal_control_request_body(1, "ab", b"c").expect("small request should encode");
let mut golden = b"rustfs-heal-control-v2\0".to_vec();
let mut golden = b"rustfs-heal-control-v3\0".to_vec();
golden.extend_from_slice(&1_u32.to_be_bytes());
golden.extend_from_slice(&2_u64.to_be_bytes());
golden.extend_from_slice(b"ab");
@@ -1741,11 +1741,11 @@ mod heal_control_tests {
#[test]
fn canonical_capability_ack_binds_version_and_topology() {
assert_eq!(HEAL_CONTROL_PROTOCOL_VERSION, 2);
assert!(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX.starts_with(b"rustfs-heal-control-capability-v2"));
assert_eq!(HEAL_CONTROL_PROTOCOL_VERSION, 3);
assert!(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX.starts_with(b"rustfs-heal-control-capability-v3"));
let probe = heal_control_capability_probe(&[7; 16]);
let ack = canonical_heal_control_capability_ack(1, "ab", &probe).expect("small acknowledgement should encode");
let mut golden = b"rustfs-heal-control-capability-ack-v2\0".to_vec();
let mut golden = b"rustfs-heal-control-capability-ack-v3\0".to_vec();
golden.extend_from_slice(&1_u32.to_be_bytes());
golden.extend_from_slice(&2_u64.to_be_bytes());
golden.extend_from_slice(b"ab");