Compare commits

..

1 Commits

Author SHA1 Message Date
overtrue d59a2b5ac8 fix(admin): probe set drives concurrently for storage info
The admin storage walk probed a set's drives one after another, each
bounded by the disk_info timeout, so a few drives still recovering
after a power cut pushed the local snapshot past the peer probe budget
and healthy peers rendered as unknown. Probe all drives at once so the
walk costs one timeout at most, and add a test-only probe delay hook to
pin that bound.
2026-09-07 11:06:36 +08:00
7 changed files with 173 additions and 201 deletions
+8 -26
View File
@@ -121,22 +121,6 @@ jobs:
create_latest=false
source_ref="$GITHUB_SHA"
# Pre-GA policy: until the first stable (vX.Y.Z) tag exists, every
# prerelease (alpha/beta/rc) also moves `latest`, so users pulling
# `latest` get the newest test build. Once a stable tag is published
# this returns false and `latest` follows stable releases only.
prerelease_moves_latest() {
local stable_tags
stable_tags=$(git ls-remote --tags --refs origin 2>/dev/null \
| awk '{print $2}' \
| grep -E '^refs/tags/v?[0-9]+\.[0-9]+\.[0-9]+$' || true)
if [[ -z "$stable_tags" ]]; then
return 0
fi
echo "️ Stable release tag(s) already exist; prereleases no longer update latest"
return 1
}
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
# Triggered by build workflow completion
echo "🔗 Triggered by build workflow completion"
@@ -200,8 +184,8 @@ jobs:
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
build_type="prerelease"
is_prerelease=true
# Pre-GA policy: prereleases update latest until the first stable tag exists.
if prerelease_moves_latest; then
# Current policy: create latest tags for stable releases and selected prereleases (alpha/beta).
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]]; then
create_latest=true
echo "🧪 Building Docker image for prerelease: $version (creating latest tag)"
else
@@ -259,8 +243,8 @@ jobs:
v*alpha*|v*beta*|v*rc*|*alpha*|*beta*|*rc*)
build_type="prerelease"
is_prerelease=true
# Pre-GA policy: prereleases update latest until the first stable tag exists.
if prerelease_moves_latest; then
# Current policy: create latest tags for stable releases and selected prereleases (alpha/beta).
if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]]; then
create_latest=true
echo "🧪 Building with prerelease version: $input_version (creating latest tag)"
else
@@ -410,13 +394,11 @@ jobs:
TAG_BASE="${VERSION}${VARIANT_SUFFIX}"
TAGS="${{ env.REGISTRY_DOCKERHUB }}:$TAG_BASE,${{ env.REGISTRY_GHCR }}:$TAG_BASE,${{ env.REGISTRY_QUAY }}:$TAG_BASE"
# Add latest when requested (stable releases, and prereleases before GA)
# Add channel tags for prereleases and latest for stable
if [[ "$CREATE_LATEST" == "true" ]]; then
# Create latest tags for stable releases and selected prereleases when CREATE_LATEST=true.
TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_GHCR }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_QUAY }}:latest${VARIANT_SUFFIX}"
fi
# Always add the channel tag for prereleases, independent of latest
if [[ "$BUILD_TYPE" == "prerelease" ]]; then
elif [[ "$BUILD_TYPE" == "prerelease" ]]; then
# Prerelease channel tags (alpha, beta, rc)
if [[ "$VERSION" == *"alpha"* ]]; then
CHANNEL="alpha"
@@ -573,7 +555,7 @@ jobs:
"prerelease")
echo "🧪 Prerelease Docker image has been built with ${VERSION} tags"
echo "⚠️ This is a prerelease image - use with caution"
# Prereleases move latest until the first stable tag exists (pre-GA policy).
# Create latest tags for stable releases and selected prereleases when CREATE_LATEST=true.
if [[ "$CREATE_LATEST" == "true" ]]; then
echo "🏷️ Latest tag has been created for prerelease: $VERSION"
else
Generated
+5 -5
View File
@@ -6139,9 +6139,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
[[package]]
name = "libflate"
version = "2.3.2"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "561a8da1a50e1428d3c51321dafeca849df992a5bb67720c386131234caba82e"
checksum = "a4da9b700e758e57152a1fd1c52cbdc5727c1aa6d8743dc1acda917398f1d76c"
dependencies = [
"adler32",
"crc32fast",
@@ -10943,9 +10943,9 @@ dependencies = [
[[package]]
name = "rustfs-uring"
version = "0.2.2"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b29bc57b4bd62a73f4fae408b536adf578332e50e464797d09dc2382c7cb68c2"
checksum = "0486e62d0efe25db95c00aeacb2da84368adcba299216cda99fcb11328061c84"
dependencies = [
"io-uring",
"libc",
@@ -12406,7 +12406,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.4.3",
"getrandom 0.3.4",
"once_cell",
"rustix",
"windows-sys 0.59.0",
@@ -54,23 +54,6 @@ mod tests {
test_binary: EvidenceBuild,
}
#[derive(Clone, Copy)]
struct ScannerHealEvidenceCase {
id: &'static str,
oracle: &'static str,
}
const BACKGROUND_TARGET_RESTART_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
id: "background-target-restart",
oracle: "background-target-restart.json",
};
struct RestartEvidenceContext {
directory: PathBuf,
run: RestartEvidenceRun,
case: ScannerHealEvidenceCase,
}
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
let mut file = std::fs::File::open(path)?;
let mut digest = Sha256::new();
@@ -85,22 +68,10 @@ mod tests {
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
}
fn restart_evidence_run(
binary: &Path,
case: ScannerHealEvidenceCase,
) -> Result<Option<RestartEvidenceContext>, Box<dyn Error + Send + Sync>> {
fn restart_evidence_run(binary: &Path) -> Result<Option<(PathBuf, RestartEvidenceRun)>, Box<dyn Error + Send + Sync>> {
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
return Ok(None);
};
if case.id.is_empty()
|| case.oracle.is_empty()
|| !case.oracle.ends_with(".json")
|| case.oracle.contains('/')
|| case.oracle.contains('\\')
|| case.oracle.contains("..")
{
return Err("invalid scanner/heal evidence case".into());
}
let directory = PathBuf::from(directory);
let receipt = directory.join("run.json");
if receipt.metadata()?.len() > 1024 * 1024 {
@@ -120,10 +91,10 @@ mod tests {
run.test_binary.sha256,
"test executable must match the run receipt"
);
if directory.join(case.oracle).exists() {
if directory.join("background-target-restart.json").exists() {
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
}
Ok(Some(RestartEvidenceContext { directory, run, case }))
Ok(Some((directory, run)))
}
fn compiled_test_identity() -> serde_json::Value {
@@ -993,7 +964,7 @@ mod tests {
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
let server_binary = rustfs_binary_path();
let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
restart_evidence_run(&server_binary, BACKGROUND_TARGET_RESTART_EVIDENCE)?
restart_evidence_run(&server_binary)?
} else {
None
};
@@ -1642,20 +1613,15 @@ mod tests {
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
}
if let Some(evidence_context) = evidence_run {
if let Some((directory, run)) = evidence_run {
let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
assert_ne!(target_pid, restarted_pid, "target must be a new process");
assert_eq!(
file_sha256(&server_binary)?,
evidence_context.run.binary.sha256,
"server build changed during restart"
);
assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart");
let evidence = serde_json::json!({
"schema": 1, "case": evidence_context.case.id, "evidence": "process-restart",
"run_id": evidence_context.run.run_id, "source_revision": evidence_context.run.source_revision,
"schema": 1, "case": "background-target-restart", "evidence": "process-restart",
"run_id": run.run_id, "source_revision": run.source_revision,
"test_build": compiled_test_identity(),
"binary_sha256": evidence_context.run.binary.sha256,
"test_binary_sha256": evidence_context.run.test_binary.sha256,
"binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
"pid_before": target_pid, "pid_after": restarted_pid,
"objects": evidence_objects, "node_listings": node_listings,
@@ -1667,7 +1633,7 @@ mod tests {
let mut output = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(evidence_context.directory.join(evidence_context.case.oracle))?;
.open(directory.join("background-target-restart.json"))?;
output.write_all(&data)?;
output.sync_all()?;
}
+1 -1
View File
@@ -226,7 +226,7 @@ metrics = { workspace = true }
# crates.io. The guard scripts/check_no_tokio_io_uring.sh allows an explicit
# io-uring integration; only the tokio "io-uring" runtime feature is banned.
[target.'cfg(target_os = "linux")'.dependencies]
rustfs-uring = "0.2.2"
rustfs-uring = "0.2.1"
[target.'cfg(windows)'.dependencies]
winapi-util.workspace = true
+11
View File
@@ -195,6 +195,13 @@ fn resolve_drive_timeout_profile_from_env() -> DriveTimeoutProfile {
DriveTimeoutProfile::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE).unwrap_or(DriveTimeoutProfile::Default)
}
#[cfg(test)]
tokio::task_local! {
/// Artificial `disk_info` latency for tests that pin how the admin storage
/// walk composes per-drive probe time.
pub(crate) static DISK_INFO_PROBE_DELAY_FOR_TEST: Duration;
}
fn get_drive_timeout_profile() -> DriveTimeoutProfile {
#[cfg(test)]
{
@@ -2036,6 +2043,10 @@ impl DiskAPI for LocalDiskWrapper {
.track_disk_health_with_op_and_timeout_action(
"disk_info",
|| async {
#[cfg(test)]
if let Ok(delay) = DISK_INFO_PROBE_DELAY_FOR_TEST.try_with(|delay| *delay) {
tokio::time::sleep(delay).await;
}
let result = self.disk.disk_info(opts).await?;
if let Some(current_disk_id) = *self.disk_id.read().await
+138 -102
View File
@@ -6463,113 +6463,114 @@ pub fn should_heal_object_on_disk(
(false, false, None)
}
/// Probe every drive of the set at once. Each live probe is bounded by the
/// drive `disk_info` timeout, and the admin peer probe budget only covers one
/// such timeout; a sequential walk over several stalled drives after a power
/// cut would exceed it and make healthy peers render as unknown (#6488).
async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<rustfs_madmin::Disk> {
let mut ret = Vec::new();
join_all(disks.iter().zip(eps).map(|(disk, ep)| disk_admin_info(disk.as_ref(), ep))).await
}
for (i, pool) in disks.iter().enumerate() {
if let Some(disk) = pool {
let runtime_state = disk.runtime_state();
let offline_duration_seconds = disk.offline_duration_secs();
let capacity_snapshot = disk.last_capacity_snapshot();
let cached_disk_id = disk.cached_disk_id().await;
if runtime_state.should_probe_for_admin() || runtime_state == disk::health_state::RuntimeDriveHealthState::Suspect {
match disk
.disk_info(&DiskInfoOptions {
metrics: true,
..Default::default()
})
.await
{
Ok(res) => {
disk.record_capacity_probe(res.total, res.used, res.free);
ret.push(rustfs_madmin::Disk {
endpoint: eps[i].to_string(),
local: eps[i].is_local,
pool_index: eps[i].pool_idx,
set_index: eps[i].set_idx,
disk_index: eps[i].disk_idx,
state: "ok".to_owned(),
async fn disk_admin_info(disk: Option<&DiskStore>, ep: &Endpoint) -> rustfs_madmin::Disk {
let Some(disk) = disk else {
return rustfs_madmin::Disk {
endpoint: ep.to_string(),
drive_path: ep.get_file_path(),
local: ep.is_local,
pool_index: ep.pool_idx,
set_index: ep.set_idx,
disk_index: ep.disk_idx,
runtime_state: None,
offline_duration_seconds: None,
state: DiskError::DiskNotFound.to_string(),
capacity_observation_source: Some("missing".to_owned()),
capacity_observation_age_seconds: Some(0),
..Default::default()
};
};
root_disk: res.root_disk,
drive_path: res.mount_path.clone(),
healing: res.healing,
scanning: res.scanning,
runtime_state: Some(runtime_state.as_str().to_string()),
offline_duration_seconds,
capacity_observation_source: Some("live_probe".to_owned()),
capacity_observation_age_seconds: Some(0),
uuid: res.id.map_or_else(|| "".to_string(), |id| id.to_string()),
major: res.major as u32,
minor: res.minor as u32,
model: None,
total_space: res.total,
used_space: res.used,
available_space: res.free,
physical_device_ids: (!res.physical_device_ids.is_empty()).then_some(res.physical_device_ids.clone()),
utilization: utilization_percent(res.total, res.used),
used_inodes: res.used_inodes,
free_inodes: res.free_inodes,
metrics: Some(res.metrics),
..Default::default()
});
}
Err(err) => {
let mut disk_info = rustfs_madmin::Disk {
state: err.to_string(),
endpoint: eps[i].to_string(),
drive_path: eps[i].get_file_path(),
local: eps[i].is_local,
pool_index: eps[i].pool_idx,
set_index: eps[i].set_idx,
disk_index: eps[i].disk_idx,
runtime_state: Some(runtime_state.as_str().to_string()),
offline_duration_seconds,
metrics: disk.metrics_snapshot(),
uuid: cached_disk_id.map_or_else(String::new, |id| id.to_string()),
..Default::default()
};
if let Some((total, used, free, _)) = capacity_snapshot {
disk_info.total_space = total;
disk_info.used_space = used;
disk_info.available_space = free;
disk_info.utilization = utilization_percent(total, used);
disk_info.capacity_observation_source = Some("snapshot".to_owned());
disk_info.capacity_observation_age_seconds = capacity_snapshot
.map(|(_, _, _, probe_unix_secs)| capacity_snapshot_age_seconds(probe_unix_secs));
} else {
disk_info.capacity_observation_source = Some("missing".to_owned());
disk_info.capacity_observation_age_seconds = Some(0);
}
ret.push(disk_info);
}
}
} else {
let mut disk_info =
build_runtime_snapshot_disk(&eps[i], runtime_state, offline_duration_seconds, capacity_snapshot);
disk_info.metrics = disk.metrics_snapshot();
disk_info.uuid = cached_disk_id.map_or_else(String::new, |id| id.to_string());
ret.push(disk_info);
}
} else {
ret.push(rustfs_madmin::Disk {
endpoint: eps[i].to_string(),
drive_path: eps[i].get_file_path(),
local: eps[i].is_local,
pool_index: eps[i].pool_idx,
set_index: eps[i].set_idx,
disk_index: eps[i].disk_idx,
runtime_state: None,
offline_duration_seconds: None,
state: DiskError::DiskNotFound.to_string(),
capacity_observation_source: Some("missing".to_owned()),
capacity_observation_age_seconds: Some(0),
..Default::default()
})
}
let runtime_state = disk.runtime_state();
let offline_duration_seconds = disk.offline_duration_secs();
let capacity_snapshot = disk.last_capacity_snapshot();
let cached_disk_id = disk.cached_disk_id().await;
if !(runtime_state.should_probe_for_admin() || runtime_state == disk::health_state::RuntimeDriveHealthState::Suspect) {
let mut disk_info = build_runtime_snapshot_disk(ep, runtime_state, offline_duration_seconds, capacity_snapshot);
disk_info.metrics = disk.metrics_snapshot();
disk_info.uuid = cached_disk_id.map_or_else(String::new, |id| id.to_string());
return disk_info;
}
ret
match disk
.disk_info(&DiskInfoOptions {
metrics: true,
..Default::default()
})
.await
{
Ok(res) => {
disk.record_capacity_probe(res.total, res.used, res.free);
rustfs_madmin::Disk {
endpoint: ep.to_string(),
local: ep.is_local,
pool_index: ep.pool_idx,
set_index: ep.set_idx,
disk_index: ep.disk_idx,
state: "ok".to_owned(),
root_disk: res.root_disk,
drive_path: res.mount_path.clone(),
healing: res.healing,
scanning: res.scanning,
runtime_state: Some(runtime_state.as_str().to_string()),
offline_duration_seconds,
capacity_observation_source: Some("live_probe".to_owned()),
capacity_observation_age_seconds: Some(0),
uuid: res.id.map_or_else(|| "".to_string(), |id| id.to_string()),
major: res.major as u32,
minor: res.minor as u32,
model: None,
total_space: res.total,
used_space: res.used,
available_space: res.free,
physical_device_ids: (!res.physical_device_ids.is_empty()).then_some(res.physical_device_ids.clone()),
utilization: utilization_percent(res.total, res.used),
used_inodes: res.used_inodes,
free_inodes: res.free_inodes,
metrics: Some(res.metrics),
..Default::default()
}
}
Err(err) => {
let mut disk_info = rustfs_madmin::Disk {
state: err.to_string(),
endpoint: ep.to_string(),
drive_path: ep.get_file_path(),
local: ep.is_local,
pool_index: ep.pool_idx,
set_index: ep.set_idx,
disk_index: ep.disk_idx,
runtime_state: Some(runtime_state.as_str().to_string()),
offline_duration_seconds,
metrics: disk.metrics_snapshot(),
uuid: cached_disk_id.map_or_else(String::new, |id| id.to_string()),
..Default::default()
};
if let Some((total, used, free, _)) = capacity_snapshot {
disk_info.total_space = total;
disk_info.used_space = used;
disk_info.available_space = free;
disk_info.utilization = utilization_percent(total, used);
disk_info.capacity_observation_source = Some("snapshot".to_owned());
disk_info.capacity_observation_age_seconds =
capacity_snapshot.map(|(_, _, _, probe_unix_secs)| capacity_snapshot_age_seconds(probe_unix_secs));
} else {
disk_info.capacity_observation_source = Some("missing".to_owned());
disk_info.capacity_observation_age_seconds = Some(0);
}
disk_info
}
}
}
fn build_runtime_snapshot_disk(
@@ -10691,6 +10692,41 @@ mod tests {
);
}
#[tokio::test(start_paused = true)]
async fn test_get_disks_info_probes_drives_concurrently() {
use crate::disk::disk_store::DISK_INFO_PROBE_DELAY_FOR_TEST;
let format = FormatV3::new(1, 4);
let mut temp_dirs = Vec::new();
let mut endpoints = Vec::new();
let mut disks = Vec::new();
for disk_idx in 0..4 {
let (dir, endpoint, disk) = make_formatted_local_disk_for_info_test(disk_idx, &format).await;
temp_dirs.push(dir);
endpoints.push(endpoint);
disks.push(Some(disk));
}
let probe_delay = std::time::Duration::from_secs(2);
let started = tokio::time::Instant::now();
let info = DISK_INFO_PROBE_DELAY_FOR_TEST
.scope(probe_delay, get_disks_info(&disks, &endpoints))
.await;
let elapsed = started.elapsed();
assert_eq!(info.len(), 4);
assert!(info.iter().all(|disk| disk.state == "ok"), "every drive should still report a live probe");
assert_eq!(
info.iter().map(|disk| disk.disk_index).collect::<Vec<_>>(),
endpoints.iter().map(|ep| ep.disk_idx).collect::<Vec<_>>(),
"concurrent probes must keep endpoint order"
);
assert!(
elapsed < probe_delay * 2,
"four stalled drives must cost one probe delay, not four; took {elapsed:?}"
);
}
#[tokio::test]
async fn test_get_disks_info_preserves_remote_cached_disk_id_when_offline() {
let (endpoint, disk) = make_remote_disk_for_info_test(0).await;
-23
View File
@@ -1309,29 +1309,6 @@ class SelfTests(unittest.TestCase):
self.assertIn("alternate-target-restart.json", read_json(run_dir / "execution.json")["artifacts"])
self.assertEqual(check_scanner_heal_evidence(root, run_dir, "alternate-target-restart"), [])
def test_scanner_heal_release_consumes_multiple_registry_oracles(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root, run_dir = self.scanner_heal_fixture(Path(tmp))
registry = read_json(root / ".config/scanner-heal-required-tests.json")
alternate = dict(registry["cases"]["background-target-restart"])
alternate["oracle"] = "alternate-target-restart.json"
registry["cases"]["alternate-target-restart"] = alternate
write_json(root / ".config/scanner-heal-required-tests.json", registry)
oracle = read_json(run_dir / "background-target-restart.json")
oracle["case"] = "alternate-target-restart"
write_json(run_dir / "alternate-target-restart.json", oracle)
(run_dir / "execution.json").unlink()
finish_scanner_heal_receipt(run_dir, 0, root)
artifacts = read_json(run_dir / "execution.json")["artifacts"]
self.assertIn("background-target-restart.json", artifacts)
self.assertIn("alternate-target-restart.json", artifacts)
self.assertEqual(check_scanner_heal_evidence(root, run_dir, "alternate-target-restart"), [])
errors = check_scanner_heal_evidence(root, run_dir, "release")
self.assertEqual(len(errors), 21)
self.assertTrue(all(error.startswith("pending ") for error in errors))
def test_scanner_heal_rejects_broken_execution_and_artifacts(self) -> None:
for fault in ("exit", "missing", "zero", "skipped", "failed", "retry", "filtered", "ignored", "stale",
"hash", "binary", "synthetic", "wrong-run", "same-pid", "body", "parts", "listing", "topology"):