mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 15:37:02 +00:00
fix(scanner): avoid stalls after abandoned child listings (#2632)
This commit is contained in:
@@ -1169,9 +1169,6 @@ impl FolderScanner {
|
|||||||
let bucket_clone = bucket.clone();
|
let bucket_clone = bucket.clone();
|
||||||
let prefix_clone = prefix.clone();
|
let prefix_clone = prefix.clone();
|
||||||
let child_ctx_clone = child_ctx.clone();
|
let child_ctx_clone = child_ctx.clone();
|
||||||
let agreed_tx = agreed_tx.clone();
|
|
||||||
let partial_tx = partial_tx.clone();
|
|
||||||
let finished_tx = finished_tx.clone();
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
if let Err(e) = list_path_raw(
|
if let Err(e) = list_path_raw(
|
||||||
@@ -1219,13 +1216,28 @@ impl FolderScanner {
|
|||||||
});
|
});
|
||||||
|
|
||||||
let mut found_objects = false;
|
let mut found_objects = false;
|
||||||
|
let mut agreed_closed = false;
|
||||||
|
let mut partial_closed = false;
|
||||||
|
let mut finished_closed = false;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
|
if agreed_closed && partial_closed && finished_closed {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
select! {
|
select! {
|
||||||
Some(entry_name) = agreed_rx.recv() => {
|
entry_name = agreed_rx.recv(), if !agreed_closed => {
|
||||||
|
let Some(entry_name) = entry_name else {
|
||||||
|
agreed_closed = true;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
(self.update_current_path)(&entry_name).await;
|
(self.update_current_path)(&entry_name).await;
|
||||||
}
|
}
|
||||||
Some(entries) = partial_rx.recv() => {
|
entries = partial_rx.recv(), if !partial_closed => {
|
||||||
|
let Some(entries) = entries else {
|
||||||
|
partial_closed = true;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
if !self.should_heal().await {
|
if !self.should_heal().await {
|
||||||
child_ctx.cancel();
|
child_ctx.cancel();
|
||||||
break;
|
break;
|
||||||
@@ -1297,7 +1309,11 @@ impl FolderScanner {
|
|||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
Some(errs) = finished_rx.recv() => {
|
errs = finished_rx.recv(), if !finished_closed => {
|
||||||
|
let Some(errs) = errs else {
|
||||||
|
finished_closed = true;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
error!("scan_folder: list_path_raw: failed to get finished errs: {:?}", errs);
|
error!("scan_folder: list_path_raw: failed to get finished errs: {:?}", errs);
|
||||||
child_ctx.cancel();
|
child_ctx.cancel();
|
||||||
}
|
}
|
||||||
@@ -1840,6 +1856,74 @@ mod tests {
|
|||||||
assert!(result.is_ok(), "expected unreadable child directory to be skipped");
|
assert!(result.is_ok(), "expected unreadable child directory to be skipped");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
|
||||||
|
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||||
|
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||||
|
let _heal_responder = rustfs_common::heal_channel::init_heal_channel().ok().map(|mut heal_rx| {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
while let Some(command) = heal_rx.recv().await {
|
||||||
|
if let rustfs_common::heal_channel::HealChannelCommand::Start { response_tx, .. } = command {
|
||||||
|
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
|
let bucket = "src-archive";
|
||||||
|
tokio::fs::create_dir_all(temp_dir.join(bucket))
|
||||||
|
.await
|
||||||
|
.expect("failed to create bucket directory");
|
||||||
|
|
||||||
|
let mut disks = vec![scanner.local_disk.clone()];
|
||||||
|
for disk_name in ["disk2", "disk3", "disk4"] {
|
||||||
|
let disk_root = temp_dir.join(disk_name);
|
||||||
|
tokio::fs::create_dir_all(disk_root.join(bucket))
|
||||||
|
.await
|
||||||
|
.expect("failed to create extra disk bucket directory");
|
||||||
|
let endpoint =
|
||||||
|
Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("failed to create extra disk endpoint");
|
||||||
|
let disk = new_disk(
|
||||||
|
&endpoint,
|
||||||
|
&DiskOption {
|
||||||
|
cleanup: false,
|
||||||
|
health_check: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("failed to create extra disk");
|
||||||
|
disks.push(disk);
|
||||||
|
}
|
||||||
|
|
||||||
|
scanner.heal_object_select = 1;
|
||||||
|
scanner.disks = disks;
|
||||||
|
scanner.disks_quorum = 2;
|
||||||
|
scanner.old_cache.replace(
|
||||||
|
"src-archive/snapshots/37b3f20d941e2f5e6d99114d9bb2f3e67a8a2e5c9c4c5a1b0d6e7f8091a2b3c4",
|
||||||
|
bucket,
|
||||||
|
DataUsageEntry {
|
||||||
|
objects: 1,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut into = DataUsageEntry::default();
|
||||||
|
let folder = CachedFolder {
|
||||||
|
name: bucket.to_string(),
|
||||||
|
parent: None,
|
||||||
|
object_heal_prob_div: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
tokio::time::timeout(
|
||||||
|
Duration::from_millis(200),
|
||||||
|
scanner.scan_folder(CancellationToken::new(), folder, &mut into),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("scan_folder should not hang after list_path_raw finishes")
|
||||||
|
.expect("scan_folder should finish successfully");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
|
|||||||
Reference in New Issue
Block a user