merge global_disks

ecstore/src/disk/local.rs

@@ -1,7 +1,7 @@
 use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
 use super::{
-    DeleteOptions, DiskAPI, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions,
-    RenameDataResp, VolumeInfo, WalkDirOptions,
+    DeleteOptions, DiskAPI, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp,
+    ReadOptions, RenameDataResp, VolumeInfo, WalkDirOptions,
 };
 use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE};
 use crate::{
@@ -138,8 +138,34 @@ impl LocalDisk {
         Ok(())
     }
 
+    pub async fn move_to_trash(&self, delete_path: &PathBuf, _recursive: bool, _immediate_purge: bool) -> Result<()> {
+        let trash_path = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
+        // TODO: 清空回收站
+        if let Err(err) = fs::rename(&delete_path, &trash_path).await {
+            match err.kind() {
+                ErrorKind::NotFound => (),
+                _ => {
+                    warn!("delete_file rename {:?} err {:?}", &delete_path, &err);
+                    return Err(Error::from(err));
+                }
+            }
+        }
+
+        // FIXME: 先清空回收站吧，有时间再添加判断逻辑
+        let _ = fs::remove_dir_all(&trash_path).await;
+
+        // TODO: immediate
+        Ok(())
+    }
+
     // #[tracing::instrument(skip(self))]
-    pub async fn delete_file(&self, base_path: &PathBuf, delete_path: &PathBuf, recursive: bool, _immediate: bool) -> Result<()> {
+    pub async fn delete_file(
+        &self,
+        base_path: &PathBuf,
+        delete_path: &PathBuf,
+        recursive: bool,
+        immediate_purge: bool,
+    ) -> Result<()> {
         debug!("delete_file {:?}\n base_path:{:?}", &delete_path, &base_path);
 
         if is_root_path(base_path) || is_root_path(delete_path) {
@@ -153,29 +179,7 @@ impl LocalDisk {
         }
 
         if recursive {
-            let trash_path = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
-
-            if let Some(dir_path) = trash_path.parent() {
-                fs::create_dir_all(dir_path).await?;
-            }
-
-            debug!("delete_file ranme to trash {:?} to {:?}", &delete_path, &trash_path);
-
-            // TODO: 清空回收站
-            if let Err(err) = fs::rename(&delete_path, &trash_path).await {
-                match err.kind() {
-                    ErrorKind::NotFound => (),
-                    _ => {
-                        warn!("delete_file rename {:?} err {:?}", &delete_path, &err);
-                        return Err(Error::from(err));
-                    }
-                }
-            }
-
-            // FIXME: 先清空回收站吧，有时间再添加判断逻辑
-            let _ = fs::remove_dir_all(&trash_path).await;
-
-            // TODO: immediate
+            self.move_to_trash(delete_path, recursive, immediate_purge).await?;
         } else {
             if delete_path.is_dir() {
                 if let Err(err) = fs::remove_dir(&delete_path).await {
@@ -253,6 +257,52 @@ impl LocalDisk {
 
         Ok((data, modtime))
     }
+
+    async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &Vec<FileInfo>) -> Result<()> {
+        let volume_dir = self.get_bucket_path(volume)?;
+        let xlpath = self.get_object_path(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str())?;
+
+        let (data, _) = match self.read_all_data(volume, volume_dir.as_path(), &xlpath).await {
+            Ok(res) => res,
+            Err(_err) => {
+                //  TODO: check if not found return err
+
+                (Vec::new(), OffsetDateTime::UNIX_EPOCH)
+            }
+        };
+
+        if data.is_empty() {
+            return Err(Error::new(DiskError::FileNotFound));
+        }
+
+        let mut fm = FileMeta::default();
+
+        fm.unmarshal_msg(&data)?;
+
+        for fi in fis {
+            let data_dir = fm.delete_version(fi)?;
+
+            if data_dir.is_some() {
+                let dir_path = self.get_object_path(volume, format!("{}/{}", path, data_dir.unwrap().to_string()).as_str())?;
+
+                self.move_to_trash(&dir_path, true, false).await?;
+            }
+        }
+
+        // 没有版本了，删除xl.meta
+        if fm.versions.is_empty() {
+            self.delete_file(&volume_dir, &xlpath, true, false).await?;
+            return Ok(());
+        }
+
+        // 更新xl.meta
+        let buf = fm.marshal_msg()?;
+
+        self.write_all(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str(), buf)
+            .await?;
+
+        Ok(())
+    }
 }
 
 fn is_root_path(path: impl AsRef<Path>) -> bool {
@@ -606,7 +656,7 @@ impl DiskAPI for LocalDisk {
         let (src_data_path, dst_data_path) = {
             let mut data_dir = String::new();
             if !fi.is_remote() {
-                data_dir = utils::path::retain_slash(fi.data_dir.to_string().as_str());
+                data_dir = utils::path::retain_slash(fi.data_dir.unwrap_or(Uuid::nil()).to_string().as_str());
             }
 
             if !data_dir.is_empty() {
@@ -645,10 +695,9 @@ impl DiskAPI for LocalDisk {
         let old_data_dir = meta
             .find_version(fi.version_id)
             .map(|(_, version)| {
-                version.get_data_dir().filter(|data_dir| {
-                    warn!("get data dir {}", &data_dir);
-                    meta.shard_data_dir_count(&fi.version_id, data_dir) == 0
-                })
+                version
+                    .get_data_dir()
+                    .filter(|data_dir| meta.shard_data_dir_count(&fi.version_id, &Some(data_dir.clone())) == 0)
             })
             .unwrap_or_default();
 
@@ -819,6 +868,27 @@ impl DiskAPI for LocalDisk {
 
         Ok(RawFileInfo { buf })
     }
+    async fn delete_versions(
+        &self,
+        volume: &str,
+        versions: Vec<FileInfoVersions>,
+        _opts: DeleteOptions,
+    ) -> Result<Vec<Option<Error>>> {
+        let mut errs = Vec::with_capacity(versions.len());
+        for _ in 0..versions.len() {
+            errs.push(None);
+        }
+
+        for (i, ver) in versions.iter().enumerate() {
+            if let Err(e) = self.delete_versions_internal(volume, ver.name.as_str(), &ver.versions).await {
+                errs[i] = Some(e);
+            } else {
+                errs[i] = None;
+            }
+        }
+
+        Ok(errs)
+    }
     async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
         let mut results = Vec::new();
         let mut found = 0;

ecstore/src/disk/mod.rs

@@ -14,7 +14,7 @@ const STORAGE_FORMAT_FILE: &str = "xl.meta";
 
 use crate::{
     erasure::{ReadAt, Write},
-    error::Result,
+    error::{Error, Result},
     file_meta::FileMeta,
     store_api::{FileInfo, RawFileInfo},
 };
@@ -85,9 +85,31 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
         opts: &ReadOptions,
     ) -> Result<FileInfo>;
     async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo>;
+    async fn delete_versions(
+        &self,
+        volume: &str,
+        versions: Vec<FileInfoVersions>,
+        opts: DeleteOptions,
+    ) -> Result<Vec<Option<Error>>>;
     async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>>;
 }
 
+#[derive(Debug, Default, Clone)]
+pub struct FileInfoVersions {
+    // Name of the volume.
+    pub volume: String,
+
+    // Name of the file.
+    pub name: String,
+
+    // Represents the latest mod time of the
+    // latest version.
+    pub latest_mod_time: Option<OffsetDateTime>,
+
+    pub versions: Vec<FileInfo>,
+    pub free_versions: Vec<FileInfo>,
+}
+
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct WalkDirOptions {
     // Bucket to scanner

ecstore/src/disk/remote.rs

@@ -20,13 +20,14 @@ use tracing::info;
 use uuid::Uuid;
 
 use crate::{
-    error::Result,
+    error::{Error, Result},
     store_api::{FileInfo, RawFileInfo},
 };
 
 use super::{
-    endpoint::Endpoint, DeleteOptions, DiskAPI, DiskOption, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq,
-    ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, VolumeInfo, WalkDirOptions,
+    endpoint::Endpoint, DeleteOptions, DiskAPI, DiskOption, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry,
+    ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, VolumeInfo,
+    WalkDirOptions,
 };
 
 #[derive(Debug)]
@@ -40,36 +41,28 @@ impl RemoteDisk {
     pub async fn new(ep: &Endpoint, _opt: &DiskOption) -> Result<Self> {
         let root = fs::canonicalize(ep.url.path()).await?;
 
-        Ok(Self { 
+        Ok(Self {
             channel: RwLock::new(None),
             url: ep.url.clone(),
-            root
+            root,
         })
     }
 
     fn get_client(&self) -> NodeServiceClient<Timeout<Channel>> {
         let channel = {
             let read_lock = self.channel.read().unwrap();
-            
+
             if let Some(ref channel) = *read_lock {
                 channel.clone()
             } else {
-                let addr = format!(
-                    "{}://{}:{}",
-                    self.url.scheme(),
-                    self.url.host_str().unwrap(),
-                    self.url.port().unwrap()
-                );
+                let addr = format!("{}://{}:{}", self.url.scheme(), self.url.host_str().unwrap(), self.url.port().unwrap());
                 info!("disk url: {:?}", addr);
                 let connector = tonic_Endpoint::from_shared(addr).unwrap();
-                
-                let new_channel = tokio::runtime::Runtime::new()
-                    .unwrap()
-                    .block_on(connector.connect())
-                    .unwrap();
-                
+
+                let new_channel = tokio::runtime::Runtime::new().unwrap().block_on(connector.connect()).unwrap();
+
                 *self.channel.write().unwrap() = Some(new_channel.clone());
-                
+
                 new_channel
             }
         };
@@ -347,6 +340,15 @@ impl DiskAPI for RemoteDisk {
         Ok(raw_file_info)
     }
 
+    async fn delete_versions(
+        &self,
+        _volume: &str,
+        _versions: Vec<FileInfoVersions>,
+        _opts: DeleteOptions,
+    ) -> Result<Vec<Option<Error>>> {
+        unimplemented!()
+    }
+
     async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
         let read_multiple_req = serde_json::to_string(&req)?;
         let mut client = self.get_client();