mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-10 07:06:53 +00:00
fix(cluster): clarify peer health and listing timeouts (#5086)
* fix(cluster): surface observed peer health Refs rustfs/backlog#1387 Co-Authored-By: heihutu <heihutu@gmail.com> * fix(admin): distinguish unreported peer health Refs rustfs/backlog#1388 Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): decouple metacache peek timeout Refs rustfs/backlog#1389 Co-Authored-By: heihutu <heihutu@gmail.com> * docs(ops): diagnose metacache listing timeouts Refs rustfs/backlog#1390 Co-Authored-By: heihutu <heihutu@gmail.com> * refactor(cluster): clarify observed peer health Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): route capability state through contract Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::disk_store::get_drive_walkdir_stall_timeout;
|
||||
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
|
||||
use metrics::counter;
|
||||
@@ -175,6 +175,10 @@ pub(crate) enum TestReaderBehavior {
|
||||
ProducerError(DiskError),
|
||||
PrimaryErrorThenFallback(DiskError),
|
||||
PartialThenTimeout(Vec<MetaCacheEntry>),
|
||||
DelayedEntries {
|
||||
delay: Duration,
|
||||
entries: Vec<MetaCacheEntry>,
|
||||
},
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -339,6 +343,13 @@ async fn list_path_raw_inner(
|
||||
drop(out);
|
||||
Some(err)
|
||||
}
|
||||
TestReaderBehavior::DelayedEntries { delay, entries } => {
|
||||
tokio::time::sleep(delay).await;
|
||||
let mut out = rustfs_filemeta::MetacacheWriter::new(&mut wr);
|
||||
out.write(&entries).await.expect("delayed test entries should be written");
|
||||
out.close().await.expect("delayed test entries should close");
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
@@ -473,6 +484,17 @@ async fn list_path_raw_inner(
|
||||
record_producer_error(&producer_errs_clone, disk_idx, &err);
|
||||
return Err(err);
|
||||
}
|
||||
TestReaderBehavior::DelayedEntries { delay, entries } => {
|
||||
tokio::time::sleep(delay).await;
|
||||
let mut out = rustfs_filemeta::MetacacheWriter::new(&mut wr);
|
||||
out.write(&entries)
|
||||
.await
|
||||
.expect("delayed test fallback entries should be written");
|
||||
out.close().await.expect("delayed test fallback entries should close");
|
||||
need_fallback = false;
|
||||
last_err = None;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -609,47 +631,23 @@ async fn list_path_raw_inner(
|
||||
// Consumer-side peek timeout, and a caveat worth understanding
|
||||
// (rustfs/backlog#1217).
|
||||
//
|
||||
// This budget is the SAME SOURCE and SAME VALUE as the producer-side
|
||||
// walk stall budget: both default to `walkdir_stall_timeout` and fall
|
||||
// back to `get_drive_walkdir_stall_timeout()` (5s). But the two measure
|
||||
// different things:
|
||||
// This budget must not be stricter than the producer-side walk stall
|
||||
// budget. The two measure different things:
|
||||
// * producer stall: bounds a single drive READ inside the walk (see
|
||||
// `with_walk_stall_deadline` in `disk/local.rs`);
|
||||
// * this consumer peek: bounds the interval between two ADJACENT
|
||||
// entries arriving from a drive's reader (`peek_with_timeout`).
|
||||
//
|
||||
// Because they are coupled to the same value, the consumer cannot wait
|
||||
// meaningfully longer for the next entry than the producer is allowed to
|
||||
// spend producing one. When a drive walks a region dense with
|
||||
// non-listable internal items (many entries the producer filters out
|
||||
// before emitting the next visible one), a HEALTHY drive can take longer
|
||||
// than one budget to hand the consumer its next entry. The consumer then
|
||||
// classifies it as `PeekOutcome::TimedOut` and DETACHES that reader (it
|
||||
// is replaced by a drained duplex below), dropping a good drive from the
|
||||
// merge. That caps the "large prefix always succeeds" guarantee: a wide
|
||||
// enough non-listable stretch can knock healthy drives out of quorum.
|
||||
//
|
||||
// This is left documented, not decoupled. Giving the consumer peek an
|
||||
// independent, strictly-larger budget would reduce these false detaches,
|
||||
// but it also delays detaching a genuinely dead drive by the same amount
|
||||
// and shifts listing tail-latency semantics; that trade-off wants soak
|
||||
// data before it changes the default, so it is deferred to a follow-up.
|
||||
// The constraint for any such change: the consumer peek must be >= the
|
||||
// producer stall (never stricter), so it can never declare a drive
|
||||
// stalled before the producer itself would have failed.
|
||||
let peek_timeout = opts
|
||||
.walkdir_stall_timeout
|
||||
.or({
|
||||
#[cfg(test)]
|
||||
{
|
||||
opts.peek_timeout
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_else(get_drive_walkdir_stall_timeout);
|
||||
// A drive can spend one stall budget walking a dense non-listable region
|
||||
// before it can publish the next visible entry. Use an independent,
|
||||
// profile-aware consumer budget so the merge does not detach that reader
|
||||
// before the producer itself would have failed.
|
||||
let producer_stall_timeout = opts.walkdir_stall_timeout.unwrap_or_else(get_drive_walkdir_stall_timeout);
|
||||
let configured_peek_timeout = get_drive_walkdir_peek_timeout().max(producer_stall_timeout);
|
||||
#[cfg(not(test))]
|
||||
let peek_timeout = configured_peek_timeout;
|
||||
#[cfg(test)]
|
||||
let peek_timeout = opts.peek_timeout.unwrap_or(configured_peek_timeout);
|
||||
let mut errs: Vec<Option<DiskError>> = Vec::with_capacity(readers.len());
|
||||
for _ in 0..readers.len() {
|
||||
errs.push(None);
|
||||
@@ -1246,6 +1244,57 @@ mod tests {
|
||||
assert_eq!(err, DiskError::Timeout);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() {
|
||||
let entry = MetaCacheEntry {
|
||||
name: "bucket/visible-object".to_string(),
|
||||
metadata: vec![1, 2, 3],
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
let seen = Arc::new(Mutex::new(Vec::new()));
|
||||
let seen_clone = seen.clone();
|
||||
|
||||
timeout(
|
||||
Duration::from_secs(1),
|
||||
list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None, None],
|
||||
min_disks: 2,
|
||||
walkdir_stall_timeout: Some(Duration::from_millis(20)),
|
||||
test_reader_behaviors: vec![
|
||||
TestReaderBehavior::DelayedEntries {
|
||||
delay: Duration::from_millis(60),
|
||||
entries: vec![entry.clone()],
|
||||
},
|
||||
TestReaderBehavior::Entries(vec![entry]),
|
||||
],
|
||||
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
|
||||
let seen = seen_clone.clone();
|
||||
Box::pin(async move {
|
||||
let mut names = entries.0.iter().flatten().map(|entry| entry.name.as_str());
|
||||
if let (Some(first), Some(second), None) = (names.next(), names.next(), names.next())
|
||||
&& first == second
|
||||
{
|
||||
seen.lock().expect("seen mutex poisoned").push(first.to_owned());
|
||||
}
|
||||
})
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("slow-progressing reader should complete inside the consumer peek budget")
|
||||
.expect("slow-progressing reader should not be detached by the producer stall budget");
|
||||
|
||||
assert_eq!(
|
||||
seen.lock().expect("seen mutex poisoned").as_slice(),
|
||||
&["bucket/visible-object".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_prefers_timeout_for_mixed_errors_beyond_quorum_budget() {
|
||||
let err = list_path_raw(
|
||||
|
||||
@@ -32,6 +32,9 @@ const FAILURE_DOMAIN_NOT_REPORTED: &str = "failure domain labels not reported by
|
||||
const NUMA_NOT_WIRED: &str = "NUMA topology not wired into runtime";
|
||||
const PROFILING_NOT_WIRED: &str = "profiling capability not wired into ECStore";
|
||||
const PEER_HEALTH_NOT_REPORTED: &str = "peer health not reported by endpoints";
|
||||
const PEER_HEALTH_LOCAL_NODE: &str = "local node does not require peer health probing";
|
||||
const PEER_HEALTH_REACHABLE: &str = "peer marked reachable by internode health tracker";
|
||||
const PEER_HEALTH_UNREACHABLE: &str = "peer marked unreachable by internode health tracker";
|
||||
const CONTROL_RPC_SEPARATED: &str = "control RPC remains on the gRPC control plane";
|
||||
const DATA_STREAM_RPC_SEPARATED: &str = "remote disk data streams remain on the internode data transport";
|
||||
|
||||
@@ -341,12 +344,24 @@ pub fn peer_health_snapshot_from_membership(membership: &ClusterMembershipSnapsh
|
||||
.map(|node| ClusterPeerHealth {
|
||||
node_id: node.node_id.clone(),
|
||||
is_local: node.is_local,
|
||||
status: CapabilityStatus::unknown().with_reason(PEER_HEALTH_NOT_REPORTED),
|
||||
status: peer_health_status_for_node(node),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn peer_health_status_for_node(node: &ClusterNodeMembership) -> CapabilityStatus {
|
||||
if node.is_local {
|
||||
return CapabilityStatus::supported().with_reason(PEER_HEALTH_LOCAL_NODE);
|
||||
}
|
||||
|
||||
match rustfs_io_metrics::internode_metrics::cluster_peer_observed_online_status(&node.grid_host) {
|
||||
Some(true) => CapabilityStatus::supported().with_reason(PEER_HEALTH_REACHABLE),
|
||||
Some(false) => CapabilityStatus::unknown().with_reason(PEER_HEALTH_UNREACHABLE),
|
||||
None => CapabilityStatus::disabled().with_reason(PEER_HEALTH_NOT_REPORTED),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn rpc_boundary_snapshot() -> ClusterRpcBoundarySnapshot {
|
||||
ClusterRpcBoundarySnapshot {
|
||||
control_channels: ["metadata", "lock", "health", "administrative"]
|
||||
@@ -498,6 +513,7 @@ mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
|
||||
use crate::storage_api_contracts::topology::CapabilityState;
|
||||
|
||||
#[test]
|
||||
fn topology_snapshot_maps_endpoint_sets_without_local_paths() {
|
||||
@@ -588,16 +604,55 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_health_snapshot_reports_static_unknown_status() {
|
||||
let membership = membership_snapshot_from_endpoint_pools(&sample_url_endpoint_pools());
|
||||
fn peer_health_snapshot_reports_observed_peer_status() {
|
||||
let online_host = "http://cluster-control-plane-peer-online-test:9000";
|
||||
let offline_host = "http://cluster-control-plane-peer-offline-test:9000";
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable(online_host);
|
||||
rustfs_io_metrics::internode_metrics::record_peer_unreachable(offline_host, 1);
|
||||
let membership = ClusterMembershipSnapshot {
|
||||
nodes: vec![
|
||||
ClusterNodeMembership {
|
||||
node_id: LOCAL_NODE_ID.to_string(),
|
||||
grid_host: String::new(),
|
||||
is_local: true,
|
||||
pools: vec![0],
|
||||
},
|
||||
ClusterNodeMembership {
|
||||
node_id: "cluster-control-plane-peer-online-test:9000".to_string(),
|
||||
grid_host: online_host.to_string(),
|
||||
is_local: false,
|
||||
pools: vec![0],
|
||||
},
|
||||
ClusterNodeMembership {
|
||||
node_id: "cluster-control-plane-peer-offline-test:9000".to_string(),
|
||||
grid_host: offline_host.to_string(),
|
||||
is_local: false,
|
||||
pools: vec![0],
|
||||
},
|
||||
ClusterNodeMembership {
|
||||
node_id: "cluster-control-plane-peer-unknown-test:9000".to_string(),
|
||||
grid_host: "http://cluster-control-plane-peer-unknown-test:9000".to_string(),
|
||||
is_local: false,
|
||||
pools: vec![0],
|
||||
},
|
||||
],
|
||||
drives: Vec::new(),
|
||||
};
|
||||
let snapshot = peer_health_snapshot_from_membership(&membership);
|
||||
|
||||
assert_eq!(snapshot.peers.len(), 2);
|
||||
assert_eq!(snapshot.peers[0].node_id, "node1.example:9000");
|
||||
assert_eq!(snapshot.peers.len(), 4);
|
||||
assert_eq!(snapshot.peers[0].node_id, LOCAL_NODE_ID);
|
||||
assert!(snapshot.peers[0].is_local);
|
||||
assert_eq!(snapshot.peers[0].status.reason.as_deref(), Some(PEER_HEALTH_NOT_REPORTED));
|
||||
assert_eq!(snapshot.peers[1].node_id, "node2.example:9000");
|
||||
assert!(snapshot.peers[0].status.state.is_supported());
|
||||
assert_eq!(snapshot.peers[0].status.reason.as_deref(), Some(PEER_HEALTH_LOCAL_NODE));
|
||||
assert_eq!(snapshot.peers[1].node_id, "cluster-control-plane-peer-online-test:9000");
|
||||
assert!(!snapshot.peers[1].is_local);
|
||||
assert!(snapshot.peers[1].status.state.is_supported());
|
||||
assert_eq!(snapshot.peers[1].status.reason.as_deref(), Some(PEER_HEALTH_REACHABLE));
|
||||
assert_eq!(snapshot.peers[2].status.reason.as_deref(), Some(PEER_HEALTH_UNREACHABLE));
|
||||
assert!(!snapshot.peers[2].status.state.is_supported());
|
||||
assert_eq!(snapshot.peers[3].status.reason.as_deref(), Some(PEER_HEALTH_NOT_REPORTED));
|
||||
assert_eq!(snapshot.peers[3].status.state, CapabilityState::Disabled);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -209,6 +209,16 @@ pub fn get_drive_walkdir_stall_timeout() -> Duration {
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_drive_walkdir_peek_timeout() -> Duration {
|
||||
let stall_timeout = get_drive_walkdir_stall_timeout();
|
||||
let configured = get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS.saturating_mul(2)),
|
||||
);
|
||||
configured.max(stall_timeout)
|
||||
}
|
||||
|
||||
pub fn get_object_disk_read_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT,
|
||||
@@ -1656,6 +1666,64 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_walkdir_peek_timeout_uses_wider_default_when_unset() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, || {
|
||||
assert_eq!(
|
||||
get_drive_walkdir_peek_timeout(),
|
||||
Duration::from_secs(rustfs_config::DEFAULT_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS)
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_walkdir_peek_timeout_is_never_stricter_than_stall_timeout() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS, Some("3"), || {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS, Some("13"), || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
assert_eq!(get_drive_walkdir_peek_timeout(), Duration::from_secs(13));
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_walkdir_peek_timeout_prefers_canonical_over_legacy() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS, Some("23"), || {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS, || {
|
||||
assert_eq!(get_drive_walkdir_peek_timeout(), Duration::from_secs(23));
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_walkdir_peek_timeout_uses_high_latency_profile_default() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_WALKDIR_PEEK_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
temp_env::with_var(
|
||||
rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY),
|
||||
|| {
|
||||
assert_eq!(
|
||||
get_drive_walkdir_peek_timeout(),
|
||||
Duration::from_secs(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS * 2)
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_walkdir_timeout_prefers_canonical_over_legacy() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("11"), || {
|
||||
|
||||
@@ -127,4 +127,7 @@ pub(crate) mod topology {
|
||||
CapabilityStatus, DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet,
|
||||
TopologySnapshot,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_storage_api::CapabilityState;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user