This commit is contained in:
weisd
2024-12-22 05:04:48 +08:00
parent 13302c3586
commit 34448c9e89
5 changed files with 15 additions and 93 deletions
+2 -15
View File
@@ -6,8 +6,7 @@ use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
use super::{
os, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions,
FileReader, FileWriter, Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, BUCKET_META_PREFIX, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET,
STORAGE_FORMAT_FILE_BACKUP,
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE_BACKUP,
};
use crate::bitrot::bitrot_verify;
use crate::bucket::metadata_sys::{self};
@@ -27,7 +26,6 @@ use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry};
use crate::heal::error::{ERR_IGNORE_FILE_CONTRIB, ERR_SKIP_FILE};
use crate::heal::heal_commands::{HealScanMode, HealingTracker};
use crate::heal::heal_ops::HEALING_TRACKER_FILENAME;
use crate::io::AsyncToSync;
use crate::metacache::writer::MetacacheWriter;
use crate::new_object_layer_fn;
use crate::set_disk::{
@@ -51,7 +49,7 @@ use nix::NixPath;
use path_absolutize::Absolutize;
use std::collections::{HashMap, HashSet};
use std::fmt::Debug;
use std::io::{Cursor, Write};
use std::io::Cursor;
use std::os::unix::fs::MetadataExt;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Arc;
@@ -839,7 +837,6 @@ impl LocalDisk {
}
}
let prefix = "";
let mut dir_stack: Vec<String> = Vec::with_capacity(5);
for entry in entries.iter() {
@@ -2342,16 +2339,6 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> {
#[cfg(test)]
mod test {
use futures::future::join_all;
use tokio::io::BufWriter;
use utils::fs::open_file;
use utils::fs::O_RDWR;
use utils::fs::O_TRUNC;
use crate::io::VecAsyncReader;
use crate::io::VecAsyncWriter;
use crate::metacache::writer::MetacacheReader;
use super::*;
#[tokio::test]
+3 -8
View File
@@ -15,10 +15,7 @@ pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
use crate::{
bucket::{
metadata_sys::get_versioning_config,
versioning::{self, VersioningApi},
},
bucket::{metadata_sys::get_versioning_config, versioning::VersioningApi},
erasure::Writer,
error::{Error, Result},
file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion},
@@ -27,7 +24,6 @@ use crate::{
data_usage_cache::{DataUsageCache, DataUsageEntry},
heal_commands::{HealScanMode, HealingTracker},
},
io,
store_api::{FileInfo, ObjectInfo, RawFileInfo},
};
use endpoint::Endpoint;
@@ -45,7 +41,6 @@ use std::{
cmp::Ordering,
fmt::Debug,
io::{Cursor, SeekFrom},
ops::Index,
path::PathBuf,
sync::Arc,
};
@@ -921,7 +916,7 @@ impl MetaCacheEntriesSorted {
continue;
}
prev_prefix = curr_prefix.clone();
prev_prefix = curr_prefix;
objects.push(ObjectInfo {
is_dir: true,
@@ -954,7 +949,7 @@ impl MetaCacheEntriesSorted {
continue;
}
prev_prefix = curr_prefix.clone();
prev_prefix = curr_prefix;
objects.push(ObjectInfo {
is_dir: true,
+2 -1
View File
@@ -350,7 +350,8 @@ impl DiskAPI for RemoteDisk {
Ok(response.volumes)
}
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<Vec<MetaCacheEntry>> {
// FIXME: TODO: use writer
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, _wr: &mut W) -> Result<Vec<MetaCacheEntry>> {
info!("walk_dir");
let walk_dir_options = serde_json::to_string(&opts)?;
let mut client = node_service_time_out_client(&self.addr)
-1
View File
@@ -7,7 +7,6 @@ use tokio::io::AsyncRead;
use tokio::io::AsyncReadExt;
use tokio::io::AsyncWrite;
use tokio::io::AsyncWriteExt;
use tracing::warn;
// use std::sync::Arc;
// use tokio::sync::mpsc;
// use tokio::sync::mpsc::Sender;
+8 -68
View File
@@ -1,8 +1,6 @@
use crate::cache_value::metacache_set::{list_path_raw, ListPathRawOptions};
use crate::disk::error::{is_all_not_found, is_all_volume_not_found, is_err_eof, DiskError};
use crate::disk::{
DiskInfo, DiskStore, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntry, MetadataResolutionParams, WalkDirOptions,
};
use crate::disk::{DiskInfo, DiskStore, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntry, MetadataResolutionParams};
use crate::error::{Error, Result};
use crate::file_meta::merge_file_meta_versions;
use crate::peer::is_reserved_or_invalid_bucket;
@@ -11,21 +9,20 @@ use crate::store::check_list_objs_args;
use crate::store_api::{ListObjectsInfo, ObjectInfo};
use crate::utils::path::{self, base_dir_from_prefix, SLASH_SEPARATOR};
use crate::{store::ECStore, store_api::ListObjectsV2Info};
use futures::future::{join_all, ok};
use futures::select;
use futures::future::join_all;
use rand::seq::SliceRandom;
use rand::thread_rng;
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::io::ErrorKind;
use std::sync::Arc;
use tokio::sync::broadcast::{self, Receiver as B_Receiver};
use tokio::sync::mpsc::{self, Receiver, Sender};
use tracing::{error, warn};
use tracing::error;
const MAX_OBJECT_LIST: i32 = 1000;
const MAX_DELETE_LIST: i32 = 1000;
const MAX_UPLOADS_LIST: i32 = 10000;
const MAX_PARTS_LIST: i32 = 10000;
// const MAX_DELETE_LIST: i32 = 1000;
// const MAX_UPLOADS_LIST: i32 = 10000;
// const MAX_PARTS_LIST: i32 = 10000;
const METACACHE_SHARE_PREFIX: bool = false;
@@ -188,7 +185,7 @@ impl ECStore {
get_objects.truncate(max_keys as usize);
true
} else {
!err_eof && get_objects.len() > 0
!err_eof && !get_objects.is_empty()
}
};
@@ -388,62 +385,6 @@ impl ECStore {
_ = all_at_eof;
Ok(Vec::new())
// // let mut errs = Vec::new();
// let mut ress = Vec::new();
// let mut uniq = HashSet::new();
// for (disks_ress, _disks_errs) in results {
// for disks_res in disks_ress.iter() {
// if disks_res.is_none() {
// // TODO handle errs
// continue;
// }
// let entrys = disks_res.as_ref().unwrap();
// for entry in entrys {
// // warn!("lst_merged entry---- {}", &entry.name);
// if !opts.prefix.is_empty() && !entry.name.starts_with(&opts.prefix) {
// continue;
// }
// if !uniq.contains(&entry.name) {
// uniq.insert(entry.name.clone());
// // TODO: 过滤
// if opts.limit > 0 && ress.len() as i32 >= opts.limit {
// return Ok(ress);
// }
// if entry.is_object() {
// // if !opts.delimiter.is_empty() {
// // // entry.name.trim_start_matches(pat)
// // }
// let fi = entry.to_fileinfo(&opts.bucket)?;
// if let Some(f) = fi {
// ress.push(f.to_object_info(&opts.bucket, &entry.name, false));
// }
// continue;
// }
// if entry.is_dir() {
// ress.push(ObjectInfo {
// is_dir: true,
// bucket: opts.bucket.clone(),
// name: entry.name.clone(),
// ..Default::default()
// });
// }
// }
// }
// }
// }
// // warn!("list_merged errs {:?}", errs);
// Ok(ress)
}
}
@@ -840,7 +781,6 @@ mod test {
use crate::set_disk::SetDisks;
use crate::store::ECStore;
use crate::store_list_objects::ListPathOptions;
use crate::StorageAPI;
use futures::future::join_all;
use lock::namespace_lock::NsLockMap;
use tokio::sync::broadcast;