diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 8eec78e08..8ab0f6ee0 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -6,7 +6,8 @@ use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ os, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - UpdateMetadataOpts, VolumeInfo, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE_BACKUP, + UpdateMetadataOpts, VolumeInfo, WalkDirOptions, BUCKET_META_PREFIX, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, + STORAGE_FORMAT_FILE_BACKUP, }; use crate::bitrot::bitrot_verify; use crate::bucket::metadata_sys::{self}; @@ -739,9 +740,12 @@ impl LocalDisk { return Ok(()); } + println!("list_dir opts {} {}", &opts.bucket, &opts.base_dir); + let mut entries = match self.list_dir("", &opts.bucket, &opts.base_dir, -1).await { Ok(res) => res, Err(e) => { + println!("list_dir err {:?}", &e); if !DiskError::VolumeNotFound.is(&e) && !is_err_file_not_found(&e) { info!("list_dir err {:?}", &e); } @@ -749,10 +753,13 @@ impl LocalDisk { if opts.report_notfound && is_err_file_not_found(&e) { return Err(e); } + return Ok(()); } }; + println!("list_dir entries {:?}", &entries); + if entries.is_empty() { return Ok(()); } @@ -861,12 +868,71 @@ impl LocalDisk { } } - if let Some(dir) = dir_objes.get(entry) { - // + let mut meta = MetaCacheEntry { + name, + ..Default::default() + }; + + let mut is_dir_obj = false; + + if let Some(_dir) = dir_objes.get(entry) { + is_dir_obj = true; + meta.name + .truncate(meta.name.len() - meta.name.chars().last().unwrap().len_utf8()); + meta.name.push_str(GLOBAL_DIR_SUFFIX_WITH_SLASH); } + + match self + .read_metadata(self.get_object_path(&opts.bucket, format!("{}/{}", &meta.name, FORMAT_CONFIG_FILE).as_str())?) + .await + { + Ok(res) => { + if is_dir_obj { + meta.name = meta.name.trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH).to_owned(); + meta.name.push_str(SLASH_SEPARATOR); + } + + meta.metadata = res; + + out.write_obj(&meta)?; + *objs_returned += 1; + } + Err(err) => { + warn!("scan dir read_metadata err {:?}", err); + if let Some(e) = err.downcast_ref::() { + if os_is_not_exist(e) || is_sys_err_is_dir(e) { + // NOT an object, append to stack (with slash) + // If dirObject, but no metadata (which is unexpected) we skip it. + if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await { + meta.name.push_str(SLASH_SEPARATOR); + dir_stack.push(meta.name); + } + } + } + + continue; + } + }; } - // FIXME:TODO: + while let Some(dir) = dir_stack.pop() { + if opts.limit > 0 && *objs_returned >= opts.limit { + return Ok(()); + } + + out.write_obj(&MetaCacheEntry { + name: dir.clone(), + ..Default::default() + })?; + *objs_returned += 1; + + if opts.recursive { + let mut dir = dir; + if let Err(er) = Box::pin(self.scan_dir(&mut dir, opts, out, objs_returned)).await { + warn!("scan_dir err {:?}", &er); + } + } + } Ok(()) } @@ -1452,6 +1518,7 @@ impl DiskAPI for LocalDisk { } // FIXME: TODO: io.writer TODO cancel + #[tracing::instrument(level = "debug", skip(self, wr))] async fn walk_dir(&self, opts: WalkDirOptions, wr: crate::io::Writer) -> Result> { // warn!("walk_dir opts {:?}", &opts); @@ -2266,6 +2333,9 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> { #[cfg(test)] mod test { + use utils::fs::open_file; + use utils::fs::O_RDWR; + use super::*; #[tokio::test] @@ -2345,4 +2415,34 @@ mod test { let _ = fs::remove_dir_all(&p).await; } + + #[tokio::test] + async fn test_walk_dir() { + let ep = Endpoint::try_from("/Users/weisd/project/weisd/s3-rustfs/target/volume/test").unwrap(); + let disk = match LocalDisk::new(&ep, false).await { + Ok(res) => res, + Err(err) => { + println!("LocalDisk::new err {:?}", err); + return; + } + }; + + let mut f = match open_file("./testfile.txt", O_CREATE | O_RDWR).await { + Ok(res) => res, + Err(err) => { + println!("openfile err {:?}", err); + return; + } + }; + + let opts = WalkDirOptions { + bucket: "dada".to_owned(), + // base_dir: "dada".to_owned(), + recursive: true, + ..Default::default() + }; + if let Err(err) = disk.walk_dir(opts, crate::io::Writer::File(f)).await { + println!("walk_dir err {:?}", err); + } + } } diff --git a/ecstore/src/metacache/writer.rs b/ecstore/src/metacache/writer.rs index cc23cf6ec..0ce58d9a5 100644 --- a/ecstore/src/metacache/writer.rs +++ b/ecstore/src/metacache/writer.rs @@ -54,6 +54,8 @@ impl MetacacheWriter { } pub fn write_obj(&mut self, obj: &MetaCacheEntry) -> Result<()> { + println!("write_obj {:?}", &obj); + self.init()?; rmp::encode::write_bool(&mut self.wr, true)?;