mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-18 02:33:15 +00:00
test_list_path_raw done
This commit is contained in:
weisd
@@ -1,5 +1,6 @@
|
||||
use std::{future::Future, pin::Pin, sync::Arc};
|
||||
|
||||
use futures::{future::join_all, join};
|
||||
use tokio::{
|
||||
spawn,
|
||||
sync::{
|
||||
@@ -8,11 +9,16 @@ use tokio::{
|
||||
RwLock,
|
||||
},
|
||||
};
|
||||
use tracing::error;
|
||||
|
||||
use crate::{
|
||||
disk::{DiskAPI, DiskStore, MetaCacheEntries, MetaCacheEntry, WalkDirOptions},
|
||||
disk::{
|
||||
error::{is_err_eof, is_err_file_not_found, is_err_volume_not_found, DiskError},
|
||||
DiskAPI, DiskStore, MetaCacheEntries, MetaCacheEntry, WalkDirOptions,
|
||||
},
|
||||
error::{Error, Result},
|
||||
io::Writer,
|
||||
metacache::writer::MetacacheReader,
|
||||
};
|
||||
|
||||
type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
|
||||
@@ -62,44 +68,40 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
|
||||
return Err(Error::from_string("list_path_raw: 0 drives provided"));
|
||||
}
|
||||
|
||||
let mut jobs: Vec<tokio::task::JoinHandle<std::result::Result<(), Error>>> = Vec::new();
|
||||
let mut readers = Vec::with_capacity(opts.disks.len());
|
||||
let fds = Arc::new(RwLock::new(opts.fallback_disks.clone()));
|
||||
for disk in opts.disks.iter() {
|
||||
let disk = disk.clone();
|
||||
let opdisk = disk.clone();
|
||||
let opts_clone = opts.clone();
|
||||
let fds_clone = fds.clone();
|
||||
let (m_tx, m_rx) = mpsc::channel::<MetaCacheEntry>(100);
|
||||
readers.push(m_rx);
|
||||
spawn(async move {
|
||||
// let (m_tx, m_rx) = mpsc::channel::<MetaCacheEntry>(100);
|
||||
// readers.push(m_rx);
|
||||
let (rd, mut wr) = tokio::io::duplex(64);
|
||||
readers.push(MetacacheReader::new(rd));
|
||||
jobs.push(spawn(async move {
|
||||
let wakl_opts = WalkDirOptions {
|
||||
bucket: opts_clone.bucket.clone(),
|
||||
base_dir: opts_clone.path.clone(),
|
||||
recursive: opts_clone.recursice,
|
||||
report_notfound: opts_clone.report_not_found,
|
||||
filter_prefix: opts_clone.filter_prefix.clone(),
|
||||
forward_to: opts_clone.forward_to.clone(),
|
||||
limit: opts_clone.per_disk_limit,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut need_fallback = false;
|
||||
if disk.is_none() {
|
||||
need_fallback = true;
|
||||
} else {
|
||||
match disk
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.walk_dir(
|
||||
WalkDirOptions {
|
||||
bucket: opts_clone.bucket.clone(),
|
||||
base_dir: opts_clone.path.clone(),
|
||||
recursive: opts_clone.recursice,
|
||||
report_notfound: opts_clone.report_not_found,
|
||||
filter_prefix: opts_clone.filter_prefix.clone(),
|
||||
forward_to: opts_clone.forward_to.clone(),
|
||||
limit: opts_clone.per_disk_limit,
|
||||
..Default::default()
|
||||
},
|
||||
&mut Writer::NotUse,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(r) => {
|
||||
for v in r.iter() {
|
||||
let _ = m_tx.send(v.to_owned()).await;
|
||||
}
|
||||
if let Some(disk) = opdisk {
|
||||
match disk.walk_dir(wakl_opts, &mut wr).await {
|
||||
Ok(_res) => {}
|
||||
Err(err) => {
|
||||
error!("walk dir err {:?}", &err);
|
||||
need_fallback = true;
|
||||
}
|
||||
Err(_) => need_fallback = true,
|
||||
}
|
||||
} else {
|
||||
need_fallback = true;
|
||||
}
|
||||
|
||||
while need_fallback {
|
||||
@@ -108,133 +110,203 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
|
||||
if fds_w.is_empty() {
|
||||
break None;
|
||||
}
|
||||
let fd = fds_w.remove(0);
|
||||
if fd.is_some() && fd.as_ref().unwrap().is_online().await {
|
||||
break fd;
|
||||
|
||||
if let Some(fd) = fds_w.remove(0) {
|
||||
if fd.is_online().await {
|
||||
break Some(fd);
|
||||
}
|
||||
}
|
||||
};
|
||||
if f_disk.is_none() {
|
||||
|
||||
if let Some(disk) = f_disk {
|
||||
match disk
|
||||
.as_ref()
|
||||
.walk_dir(
|
||||
WalkDirOptions {
|
||||
bucket: opts_clone.bucket.clone(),
|
||||
base_dir: opts_clone.path.clone(),
|
||||
recursive: opts_clone.recursice,
|
||||
report_notfound: opts_clone.report_not_found,
|
||||
filter_prefix: opts_clone.filter_prefix.clone(),
|
||||
forward_to: opts_clone.forward_to.clone(),
|
||||
limit: opts_clone.per_disk_limit,
|
||||
..Default::default()
|
||||
},
|
||||
&mut wr,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_r) => {
|
||||
need_fallback = false;
|
||||
}
|
||||
Err(err) => {
|
||||
error!("walk dir2 err {:?}", &err);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
match disk
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.walk_dir(
|
||||
WalkDirOptions {
|
||||
bucket: opts_clone.bucket.clone(),
|
||||
base_dir: opts_clone.path.clone(),
|
||||
recursive: opts_clone.recursice,
|
||||
report_notfound: opts_clone.report_not_found,
|
||||
filter_prefix: opts_clone.filter_prefix.clone(),
|
||||
forward_to: opts_clone.forward_to.clone(),
|
||||
limit: opts_clone.per_disk_limit,
|
||||
..Default::default()
|
||||
},
|
||||
&mut Writer::NotUse,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(r) => {
|
||||
for v in r.iter() {
|
||||
let _ = m_tx.send(v.to_owned()).await;
|
||||
}
|
||||
need_fallback = false;
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}));
|
||||
}
|
||||
|
||||
let errs: Vec<Option<Error>> = vec![None; readers.len()];
|
||||
loop {
|
||||
let mut current = MetaCacheEntry::default();
|
||||
let (mut at_eof, mut has_err, mut agree) = (0, 0, 0);
|
||||
if rx.try_recv().is_ok() {
|
||||
return Err(Error::from_string("canceled"));
|
||||
}
|
||||
let mut top_entries: Vec<MetaCacheEntry> = Vec::with_capacity(readers.len());
|
||||
// top_entries.clear();
|
||||
let revjob = spawn(async move {
|
||||
let mut errs: Vec<Option<Error>> = vec![None; readers.len()];
|
||||
loop {
|
||||
let mut current = MetaCacheEntry::default();
|
||||
|
||||
for (i, r) in readers.iter_mut().enumerate() {
|
||||
if errs[i].is_none() {
|
||||
has_err += 1;
|
||||
continue;
|
||||
if rx.try_recv().is_ok() {
|
||||
return Err(Error::from_string("canceled"));
|
||||
}
|
||||
let entry = match r.recv().await {
|
||||
Some(entry) => entry,
|
||||
None => {
|
||||
at_eof += 1;
|
||||
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||
|
||||
let mut at_eof = 0;
|
||||
let mut fnf = 0;
|
||||
let mut vnf = 0;
|
||||
let mut has_err = 0;
|
||||
let mut agree = 0;
|
||||
|
||||
for (i, r) in readers.iter_mut().enumerate() {
|
||||
if errs[i].is_some() {
|
||||
has_err += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
// If no current, add it.
|
||||
if current.name.is_empty() {
|
||||
top_entries.insert(i, entry.clone());
|
||||
|
||||
let entry = match r.peek().await {
|
||||
Ok(res) => {
|
||||
if let Some(entry) = res {
|
||||
entry
|
||||
} else {
|
||||
// eof
|
||||
at_eof += 1;
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
if is_err_eof(&err) {
|
||||
at_eof += 1;
|
||||
continue;
|
||||
} else if is_err_file_not_found(&err) {
|
||||
at_eof += 1;
|
||||
fnf += 1;
|
||||
continue;
|
||||
} else if is_err_volume_not_found(&err) {
|
||||
at_eof += 1;
|
||||
fnf += 1;
|
||||
vnf += 1;
|
||||
continue;
|
||||
} else {
|
||||
has_err += 1;
|
||||
errs[i] = Some(err);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// If no current, add it.
|
||||
if current.name.is_empty() {
|
||||
top_entries.insert(i, Some(entry.clone()));
|
||||
current = entry;
|
||||
agree += 1;
|
||||
|
||||
continue;
|
||||
}
|
||||
// If exact match, we agree.
|
||||
if let Ok((_, true)) = current.matches(&entry, true) {
|
||||
top_entries.insert(i, Some(entry));
|
||||
agree += 1;
|
||||
|
||||
continue;
|
||||
}
|
||||
// If only the name matches we didn't agree, but add it for resolution.
|
||||
if entry.name == current.name {
|
||||
top_entries.insert(i, Some(entry));
|
||||
|
||||
continue;
|
||||
}
|
||||
// We got different entries
|
||||
if entry.name > current.name {
|
||||
continue;
|
||||
}
|
||||
// We got a new, better current.
|
||||
// Clear existing entries.
|
||||
top_entries.clear();
|
||||
agree += 1;
|
||||
top_entries.insert(i, Some(entry.clone()));
|
||||
current = entry;
|
||||
agree += 1;
|
||||
continue;
|
||||
}
|
||||
// If exact match, we agree.
|
||||
if let Ok((_, true)) = current.matches(&entry, true) {
|
||||
top_entries.insert(i, entry);
|
||||
agree += 1;
|
||||
continue;
|
||||
}
|
||||
// If only the name matches we didn't agree, but add it for resolution.
|
||||
if entry.name == current.name {
|
||||
top_entries.insert(i, entry);
|
||||
continue;
|
||||
}
|
||||
// We got different entries
|
||||
if entry.name > current.name {
|
||||
continue;
|
||||
}
|
||||
// We got a new, better current.
|
||||
// Clear existing entries.
|
||||
top_entries.clear();
|
||||
agree += 1;
|
||||
top_entries.insert(i, entry.clone());
|
||||
current = entry;
|
||||
}
|
||||
|
||||
if has_err > 0 && has_err > opts.disks.len() - opts.min_disks {
|
||||
if let Some(finished_fn) = opts.finished.as_ref() {
|
||||
finished_fn(&errs).await;
|
||||
if vnf > 0 && vnf >= (readers.len() - opts.min_disks) {
|
||||
return Err(Error::new(DiskError::VolumeNotFound));
|
||||
}
|
||||
let mut combined_err = Vec::new();
|
||||
errs.iter().zip(opts.disks.iter()).for_each(|(err, disk)| match (err, disk) {
|
||||
(Some(err), Some(disk)) => {
|
||||
combined_err.push(format!("drive {} returned: {}", disk.to_string(), err));
|
||||
|
||||
if fnf > 0 && fnf >= (readers.len() - opts.min_disks) {
|
||||
return Err(Error::new(DiskError::FileNotFound));
|
||||
}
|
||||
|
||||
if has_err > 0 && has_err > opts.disks.len() - opts.min_disks {
|
||||
if let Some(finished_fn) = opts.finished.as_ref() {
|
||||
finished_fn(&errs).await;
|
||||
}
|
||||
(Some(err), None) => {
|
||||
combined_err.push(err.to_string());
|
||||
let mut combined_err = Vec::new();
|
||||
errs.iter().zip(opts.disks.iter()).for_each(|(err, disk)| match (err, disk) {
|
||||
(Some(err), Some(disk)) => {
|
||||
combined_err.push(format!("drive {} returned: {}", disk.to_string(), err));
|
||||
}
|
||||
(Some(err), None) => {
|
||||
combined_err.push(err.to_string());
|
||||
}
|
||||
_ => {}
|
||||
});
|
||||
|
||||
return Err(Error::from_string(combined_err.join(", ")));
|
||||
}
|
||||
|
||||
// Break if all at EOF or error.
|
||||
if at_eof + has_err == readers.len() {
|
||||
if has_err > 0 {
|
||||
if let Some(finished_fn) = opts.finished.as_ref() {
|
||||
if has_err > 0 {
|
||||
finished_fn(&errs).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
});
|
||||
|
||||
return Err(Error::from_string(combined_err.join(", ")));
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// Break if all at EOF or error.
|
||||
if at_eof + has_err == readers.len() && has_err > 0 {
|
||||
if let Some(finished_fn) = opts.finished.as_ref() {
|
||||
finished_fn(&errs).await;
|
||||
if agree == readers.len() {
|
||||
for r in readers.iter_mut() {
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
if let Some(agreed_fn) = opts.agreed.as_ref() {
|
||||
agreed_fn(current).await;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
for (i, r) in readers.iter_mut().enumerate() {
|
||||
if top_entries[i].is_some() {
|
||||
let _ = r.skip(1).await;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
partial_fn(MetaCacheEntries(top_entries), &errs).await;
|
||||
}
|
||||
break;
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
|
||||
if agree == readers.len() {
|
||||
if let Some(agreed_fn) = opts.agreed.as_ref() {
|
||||
agreed_fn(current).await;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
jobs.push(revjob);
|
||||
|
||||
if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
partial_fn(MetaCacheEntries(top_entries), &errs).await;
|
||||
}
|
||||
}
|
||||
let a = join_all(jobs).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user