fix(tier): handle mutation intent CAS races (#5227)

* test(scripts): expand internode grpc ab env coverage

* fix(tier): handle mutation intent CAS races

Make tier mutation intent advancement retry idempotently when an If-Match CAS loses to a matching committed peer update.

Expand four-node inline fallback E2E coverage for mixed msgpack controls, transition readiness evidence, and per-node environment overrides.

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-07-25 19:10:13 +08:00
committed by GitHub
parent a63b79004c
commit 8e83087ba4
7 changed files with 747 additions and 96 deletions
+45
View File
@@ -892,6 +892,7 @@ pub struct RustFSTestClusterEnvironment {
pub access_key: String,
pub secret_key: String,
pub extra_env: Vec<(String, String)>,
pub node_extra_env: Vec<Vec<(String, String)>>,
pub topology: ClusterTopology,
}
@@ -990,6 +991,7 @@ impl RustFSTestClusterEnvironment {
access_key: "rustfs-cluster-test-access".to_string(),
secret_key: "rustfs-cluster-test-secret".to_string(),
extra_env,
node_extra_env: vec![Vec::new(); topology.node_count],
topology,
})
}
@@ -1003,6 +1005,22 @@ impl RustFSTestClusterEnvironment {
self.extra_env.push((key.into(), value.into()));
}
/// Add an extra environment variable applied to a single cluster node.
pub fn set_node_env<K, V>(
&mut self,
node_idx: usize,
key: K,
value: V,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
where
K: Into<String>,
V: Into<String>,
{
self.ensure_node_index(node_idx)?;
self.node_extra_env[node_idx].push((key.into(), value.into()));
Ok(())
}
fn ensure_node_index(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
if node_idx >= self.nodes.len() {
return Err(format!("node_idx {node_idx} is invalid").into());
@@ -1089,6 +1107,9 @@ impl RustFSTestClusterEnvironment {
for (key, value) in &self.extra_env {
command.env(key, value);
}
for (key, value) in &self.node_extra_env[i] {
command.env(key, value);
}
let process = command.current_dir(&node.data_dir).spawn()?;
@@ -1130,6 +1151,9 @@ impl RustFSTestClusterEnvironment {
for (key, value) in &self.extra_env {
command.env(key, value);
}
for (key, value) in &self.node_extra_env[node_idx] {
command.env(key, value);
}
let process = command.current_dir(&node.data_dir).spawn()?;
node.process = Some(process);
@@ -1371,6 +1395,7 @@ mod tests {
access_key: DEFAULT_ACCESS_KEY.to_string(),
secret_key: DEFAULT_SECRET_KEY.to_string(),
extra_env: Vec::new(),
node_extra_env: vec![Vec::new(); topology.node_count],
topology,
}
}
@@ -1455,4 +1480,24 @@ mod tests {
assert!(ClusterTopology::single_pool_multidrive(4, 4).validate().is_ok());
assert!(ClusterTopology::single_pool_multidrive(1, 1).validate().is_ok());
}
#[test]
fn cluster_node_env_supports_per_node_overrides() {
let mut env = fake_cluster(ClusterTopology::single_pool(4));
env.set_node_env(2, "RUSTFS_INTERNODE_RPC_MSGPACK_ONLY", "true").unwrap();
assert_eq!(
env.node_extra_env[2].as_slice(),
[("RUSTFS_INTERNODE_RPC_MSGPACK_ONLY".to_string(), "true".to_string())]
);
}
#[test]
fn cluster_node_env_rejects_invalid_index() {
let mut env = fake_cluster(ClusterTopology::single_pool(4));
let err = env
.set_node_env(4, "RUSTFS_INTERNODE_RPC_MSGPACK_ONLY", "true")
.unwrap_err()
.to_string();
assert!(err.contains("invalid"), "unexpected error: {err}");
}
}