init listobject

ecstore/src/disk/local.rs

@@ -1,7 +1,9 @@
 use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
 use super::{
-    DeleteOptions, DiskAPI, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, VolumeInfo,
+    DeleteOptions, DiskAPI, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions,
+    RenameDataResp, VolumeInfo, WalkDirOptions,
 };
+use crate::disk::STORAGE_FORMAT_FILE;
 use crate::{
     error::{Error, Result},
     file_meta::FileMeta,
@@ -10,13 +12,15 @@ use crate::{
 };
 use bytes::Bytes;
 use path_absolutize::Absolutize;
+use std::sync::Arc;
 use std::{
     fs::Metadata,
     path::{Path, PathBuf},
 };
 use time::OffsetDateTime;
 use tokio::fs::{self, File};
-use tokio::io::ErrorKind;
+use tokio::io::{DuplexStream, ErrorKind};
+use tokio::sync::mpsc;
 use tracing::{debug, warn};
 use uuid::Uuid;
 
@@ -496,7 +500,7 @@ impl DiskAPI for LocalDisk {
 
         // Ok((buffer, bytes_read))
     }
-    async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: usize) -> Result<Vec<String>> {
+    async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
         let p = self.get_bucket_path(volume)?;
 
         let mut entries = fs::read_dir(&p).await?;
@@ -522,8 +526,50 @@ impl DiskAPI for LocalDisk {
 
         Ok(volumes)
     }
-    async fn walk_dir(&self) -> Result<Vec<FileInfo>> {
-        unimplemented!()
+    async fn walk_dir(&self, opts: WalkDirOptions, wr: Arc<DuplexStream>) -> Result<()> {
+        let mut entries = self.list_dir("", &opts.bucket, &opts.base_dir, -1).await?;
+
+        entries.sort();
+
+        // 已读计数
+        let objs_returned = 0;
+
+        let bucket = opts.bucket.as_str();
+
+        // 第一层过滤
+        for entry in entries.iter() {
+            // check limit
+            if opts.limit > 0 && objs_returned >= opts.limit {
+                return Ok(());
+            }
+            // check prefix
+            if !opts.filter_prefix.is_empty() && !entry.starts_with(&opts.filter_prefix) {
+                continue;
+            }
+
+            warn!("walk_dir entry {}", entry);
+
+            let mut meta = MetaCacheEntry {
+                name: entry.clone(),
+                ..Default::default()
+            };
+
+            let fpath = self.get_object_path(bucket, format!("{}/{}", &meta.name, STORAGE_FORMAT_FILE).as_str())?;
+
+            let (fdata, _) = match self.read_metadata_with_dmtime(&fpath).await {
+                Ok(res) => res,
+                Err(e) => {
+                    // TODO: check err
+                    (Vec::new(), OffsetDateTime::UNIX_EPOCH)
+                }
+            };
+
+            meta.metadata = fdata;
+
+            // TODO: FIXME:
+        }
+
+        Ok(())
     }
 
     // #[tracing::instrument(skip(self))]

ecstore/src/disk/mod.rs

@@ -14,6 +14,7 @@ const STORAGE_FORMAT_FILE: &str = "xl.meta";
 use crate::{
     erasure::ReadAt,
     error::Result,
+    file_meta::FileMeta,
     store_api::{FileInfo, RawFileInfo},
 };
 use bytes::Bytes;
@@ -21,7 +22,7 @@ use std::{fmt::Debug, io::SeekFrom, pin::Pin, sync::Arc};
 use time::OffsetDateTime;
 use tokio::{
     fs::File,
-    io::{AsyncReadExt, AsyncSeekExt, AsyncWrite},
+    io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, DuplexStream},
 };
 use uuid::Uuid;
 
@@ -50,9 +51,9 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter>;
     async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader>;
     // 读目录下的所有文件、目录
-    async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: usize) -> Result<Vec<String>>;
-    // 读目录下的所有xl.meta
-    async fn walk_dir(&self) -> Result<Vec<FileInfo>>;
+    async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>>;
+    // 并发边读边写 TODO: wr io.Writer
+    async fn walk_dir(&self, opts: WalkDirOptions, wr: Arc<DuplexStream>) -> Result<()>;
     async fn rename_data(
         &self,
         src_volume: &str,
@@ -81,6 +82,48 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>>;
 }
 
+#[derive(Debug, Default, Clone)]
+pub struct WalkDirOptions {
+    // Bucket to scanner
+    pub bucket: String,
+    // Directory inside the bucket.
+    pub base_dir: String,
+    // Do a full recursive scan.
+    pub recursive: bool,
+
+    // ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
+    pub report_notfound: bool,
+
+    // FilterPrefix will only return results with given prefix within folder.
+    // Should never contain a slash.
+    pub filter_prefix: String,
+
+    // ForwardTo will forward to the given object path.
+    pub forward_to: String,
+
+    // Limit the number of returned objects if > 0.
+    pub limit: i32,
+
+    // DiskID contains the disk ID of the disk.
+    // Leave empty to not check disk ID.
+    pub disk_id: String,
+}
+
+#[derive(Debug, Default)]
+pub struct MetaCacheEntry {
+    // name is the full name of the object including prefixes
+    name: String,
+    // Metadata. If none is present it is not an object but only a prefix.
+    // Entries without metadata will only be present in non-recursive scans.
+    metadata: Vec<u8>,
+
+    // cached contains the metadata if decoded.
+    cached: Option<FileMeta>,
+
+    // Indicates the entry can be reused and only one reference to metadata is expected.
+    reusable: bool,
+}
+
 pub struct DiskOption {
     pub cleanup: bool,
     pub health_check: bool,