From 11abe53ef606d9693f7dffbed6d93c55d7a54c05 Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 22 Nov 2024 20:58:01 +0800 Subject: [PATCH] add disk.delete_version --- ecstore/src/disk/local.rs | 92 +++++++++++++++++++++++++++++---- ecstore/src/disk/mod.rs | 7 +-- ecstore/src/disk/remote.rs | 6 +-- ecstore/src/file_meta_inline.rs | 57 ++++++++++++++++++++ 4 files changed, 146 insertions(+), 16 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index c168e4231..ac14fd5ff 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -1,12 +1,12 @@ use super::error::{ is_err_file_not_found, is_sys_err_io, is_sys_err_not_empty, is_sys_err_too_many_files, os_is_not_exist, os_is_permission, }; -use super::os::is_root_disk; +use super::os::{is_root_disk, rename_all}; use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ os, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter, Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, - UpdateMetadataOpts, VolumeInfo, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET, + UpdateMetadataOpts, VolumeInfo, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE_BACKUP, }; use crate::bitrot::bitrot_verify; use crate::bucket::metadata_sys::GLOBAL_BucketMetadataSys; @@ -1760,16 +1760,88 @@ impl DiskAPI for LocalDisk { async fn delete_version( &self, volume: &str, - _path: &str, - _fi: FileInfo, - _force_del_marker: bool, - _opts: DeleteOptions, - ) -> Result { - let _volume_dir = self.get_bucket_path(volume)?; + path: &str, + fi: FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> Result<()> { + if path.starts_with(SLASH_SEPARATOR) { + return self + .delete( + volume, + path, + DeleteOptions { + recursive: false, + immediate: false, + ..Default::default() + }, + ) + .await; + } - // self.read_all_data(bucket, volume_dir, path); + let volume_dir = self.get_bucket_path(volume)?; - unimplemented!() + let file_path = volume_dir.join(Path::new(&path)); + + check_path_length(file_path.to_string_lossy().as_ref())?; + + let xl_path = file_path.join(Path::new(STORAGE_FORMAT_FILE)); + let buf = match self.read_all_data(volume, &volume_dir, &xl_path).await { + Ok(res) => res, + Err(err) => { + // + if !is_err_file_not_found(&err) { + return Err(err); + } + + if fi.deleted && force_del_marker { + return self.write_metadata("", volume, path, fi).await; + } + + if fi.version_id.is_some() { + return Err(Error::new(DiskError::FileVersionNotFound)); + } else { + return Err(Error::new(DiskError::FileNotFound)); + } + } + }; + + let mut meta = FileMeta::load(&buf)?; + let old_dir = meta.delete_version(&fi)?; + + if let Some(uuid) = old_dir { + let vid = fi.version_id.unwrap_or(Uuid::nil()); + let _ = meta.data.remove(vec![vid, uuid])?; + + let old_path = file_path.join(Path::new(uuid.to_string().as_str())); + check_path_length(old_path.to_string_lossy().as_ref())?; + + if let Err(err) = self.move_to_trash(&old_path, true, false).await { + if !is_err_file_not_found(&err) { + return Err(err); + } + } + } + + if !meta.versions.is_empty() { + let buf = meta.marshal_msg()?; + return self + .write_all_meta(volume, format!("{}{}{}", path, SLASH_SEPARATOR, STORAGE_FORMAT_FILE).as_str(), &buf, true) + .await; + } + + // opts.undo_write && opts.old_data_dir.is_some_and(f) + if let Some(old_data_dir) = opts.old_data_dir { + if opts.undo_write { + let src_path = file_path.join(Path::new( + format!("{}{}{}", old_data_dir.to_string(), SLASH_SEPARATOR, STORAGE_FORMAT_FILE_BACKUP).as_str(), + )); + let dst_path = file_path.join(Path::new(format!("{}{}{}", path, SLASH_SEPARATOR, STORAGE_FORMAT_FILE).as_str())); + return rename_all(src_path, dst_path, file_path).await; + } + } + + self.delete_file(&volume_dir, &xl_path, true, false).await } async fn delete_versions( &self, diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index ec393a1a2..10ceeafb7 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -11,7 +11,8 @@ pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp"; pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash"; pub const BUCKET_META_PREFIX: &str = "buckets"; pub const FORMAT_CONFIG_FILE: &str = "format.json"; -const STORAGE_FORMAT_FILE: &str = "xl.meta"; +pub const STORAGE_FORMAT_FILE: &str = "xl.meta"; +pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp"; use crate::{ erasure::Writer, @@ -303,7 +304,7 @@ impl DiskAPI for Disk { fi: FileInfo, force_del_marker: bool, opts: DeleteOptions, - ) -> Result { + ) -> Result<()> { match self { Disk::Local(local_disk) => local_disk.delete_version(volume, path, fi, force_del_marker, opts).await, Disk::Remote(remote_disk) => remote_disk.delete_version(volume, path, fi, force_del_marker, opts).await, @@ -409,7 +410,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { fi: FileInfo, force_del_marker: bool, opts: DeleteOptions, - ) -> Result; + ) -> Result<()>; async fn delete_versions( &self, volume: &str, diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index ed69af25b..618365f6d 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -616,7 +616,7 @@ impl DiskAPI for RemoteDisk { fi: FileInfo, force_del_marker: bool, opts: DeleteOptions, - ) -> Result { + ) -> Result<()> { info!("delete_version"); let file_info = serde_json::to_string(&fi)?; let opts = serde_json::to_string(&opts)?; @@ -639,9 +639,9 @@ impl DiskAPI for RemoteDisk { return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); } - let raw_file_info = serde_json::from_str::(&response.raw_file_info)?; + // let raw_file_info = serde_json::from_str::(&response.raw_file_info)?; - Ok(raw_file_info) + Ok(()) } async fn delete_versions( &self, diff --git a/ecstore/src/file_meta_inline.rs b/ecstore/src/file_meta_inline.rs index a85813553..005e18a9f 100644 --- a/ecstore/src/file_meta_inline.rs +++ b/ecstore/src/file_meta_inline.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use uuid::Uuid; use crate::error::{Error, Result}; use std::io::{Cursor, Read}; @@ -157,7 +158,63 @@ impl InlineData { self.serialize(keys, values) } + pub fn remove(&mut self, remove_keys: Vec) -> Result { + let buf = self.after_version(); + let mut cur = Cursor::new(buf); + let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize; + let mut keys = Vec::with_capacity(fields_len + 1); + let mut values = Vec::with_capacity(fields_len + 1); + + let remove_key = |found_key: &str| { + for key in remove_keys.iter() { + if key.to_string().as_str() == found_key { + return true; + } + } + false + }; + + let mut found = false; + + while fields_len > 0 { + fields_len -= 1; + + let str_len = rmp::decode::read_str_len(&mut cur)?; + + let mut field_buff = vec![0u8; str_len as usize]; + + cur.read_exact(&mut field_buff)?; + + let find_key = String::from_utf8(field_buff)?; + + let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; + let start = cur.position() as usize; + let end = start + bin_len; + cur.set_position(end as u64); + + let find_value = &buf[start..end]; + + if !remove_key(&find_key) { + values.push(find_value.to_vec()); + keys.push(find_key); + } else { + found = true; + } + } + + if !found { + return Ok(false); + } + + if keys.is_empty() { + self.0 = Vec::new(); + return Ok(true); + } + + self.serialize(keys, values)?; + Ok(true) + } fn serialize(&mut self, keys: Vec, values: Vec>) -> Result<()> { assert_eq!(keys.len(), values.len(), "InlineData serialize: keys/values not match");