From 8b28e0869c4d4943df601db103bf35eb9f0ab6c5 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 19 Feb 2025 13:51:40 +0800 Subject: [PATCH] sync s3s, fix #224 --- Cargo.lock | 8 ++++---- Cargo.toml | 4 ++-- ecstore/src/disk/local.rs | 6 ++---- ecstore/src/disk/mod.rs | 3 ++- ecstore/src/file_meta.rs | 7 ++++--- ecstore/src/set_disk.rs | 8 ++++++-- rustfs/src/storage/ecfs.rs | 14 +++++++++++--- 7 files changed, 31 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d1dc017d0..f55eac6a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4126,9 +4126,9 @@ dependencies = [ [[package]] name = "numeric_cast" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6699ec124a5322dd30184abe24b2b9a92237ed0fe8e038f1f410bf7cccfe8b33" +checksum = "a3c00a0c9600379bd32f8972de90676a7672cba3bf4886986bc05902afc1e093" [[package]] name = "objc" @@ -5515,7 +5515,7 @@ checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "s3s" version = "0.11.0-dev" -source = "git+https://github.com/Nugine/s3s.git?rev=529c8933a11528c506d5fbf7c4c2ab155db37dfe#529c8933a11528c506d5fbf7c4c2ab155db37dfe" +source = "git+https://github.com/Nugine/s3s.git?rev=7c78bd75f818783f1263041b61ab08b8182519b4#7c78bd75f818783f1263041b61ab08b8182519b4" dependencies = [ "arrayvec", "async-trait", @@ -5562,7 +5562,7 @@ dependencies = [ [[package]] name = "s3s-policy" version = "0.11.0-dev" -source = "git+https://github.com/Nugine/s3s.git?rev=529c8933a11528c506d5fbf7c4c2ab155db37dfe#529c8933a11528c506d5fbf7c4c2ab155db37dfe" +source = "git+https://github.com/Nugine/s3s.git?rev=7c78bd75f818783f1263041b61ab08b8182519b4#7c78bd75f818783f1263041b61ab08b8182519b4" dependencies = [ "indexmap 2.7.1", "serde", diff --git a/Cargo.toml b/Cargo.toml index eed309921..2eed73bb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,10 +71,10 @@ reqwest = { version = "0.12.12", default-features = false, features = ["rustls-t rfd = "0.15.2" rmp = "0.8.14" rmp-serde = "1.3.0" -s3s = { git = "https://github.com/Nugine/s3s.git", rev = "529c8933a11528c506d5fbf7c4c2ab155db37dfe", default-features = true, features = [ +s3s = { git = "https://github.com/Nugine/s3s.git", rev = "7c78bd75f818783f1263041b61ab08b8182519b4", default-features = true, features = [ "tower", ] } -s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "529c8933a11528c506d5fbf7c4c2ab155db37dfe" } +s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "7c78bd75f818783f1263041b61ab08b8182519b4" } serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.138" tempfile = "3.16.0" diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index cc58de3ce..8a559477e 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -19,7 +19,7 @@ use crate::disk::error::{ use crate::disk::os::{check_path_length, is_empty_dir}; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; use crate::error::{Error, Result}; -use crate::file_meta::read_xl_meta_no_data; +use crate::file_meta::{get_file_info, read_xl_meta_no_data, FileInfoOpts}; use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; use crate::heal::data_scanner::{has_active_rules, scan_data_folder, ScannerItem, ShouldSleepFn, SizeSummary}; use crate::heal::data_scanner_metric::{ScannerMetric, ScannerMetrics}; @@ -1995,10 +1995,8 @@ impl DiskAPI for LocalDisk { let (data, _) = self.read_raw(volume, file_dir, file_path, read_data).await?; - let mut meta = FileMeta::default(); - meta.unmarshal_msg(&data)?; + let fi = get_file_info(&data, volume, path, version_id, FileInfoOpts { data: read_data }).await?; - let fi = meta.into_fileinfo(volume, path, version_id, read_data, true)?; Ok(fi) } async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 32bc2ca5b..0b7aba4cb 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -1208,8 +1208,9 @@ pub struct VolumeInfo { pub created: Option, } -#[derive(Deserialize, Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug, Default)] pub struct ReadOptions { + pub incl_free_versions: bool, pub read_data: bool, pub healing: bool, } diff --git a/ecstore/src/file_meta.rs b/ecstore/src/file_meta.rs index e130ada40..e8caced50 100644 --- a/ecstore/src/file_meta.rs +++ b/ecstore/src/file_meta.rs @@ -554,7 +554,7 @@ impl FileMeta { let header = &ver.header; if let Some(vid) = has_vid { - if header.version_id == Some(vid) { + if header.version_id != Some(vid) { is_latest = false; succ_mod_time = header.mod_time; continue; @@ -2045,7 +2045,7 @@ pub struct FileInfoOpts { pub data: bool, } -async fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result { +pub async fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result { let vid = { if version_id.is_empty() { None @@ -2067,8 +2067,9 @@ async fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, o }); } + warn!("get_file_info vid {:?}", &vid); let fi = meta.into_fileinfo(volume, path, version_id, opts.data, true)?; - + warn!("get_file_info {:?}", &fi); Ok(fi) } diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index 5d457bf22..4691b7e24 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -1088,7 +1088,11 @@ impl SetDisks { let mut errors = Vec::with_capacity(disks.len()); for disk in disks.iter() { - let opts = ReadOptions { read_data, healing }; + let opts = ReadOptions { + read_data, + healing, + ..Default::default() + }; futures.push(async move { if let Some(disk) = disk { if version_id.is_empty() { @@ -3634,7 +3638,7 @@ impl ObjectIO for SetDisks { Ok(reader) } - #[tracing::instrument(level = "debug", skip(self, data))] + #[tracing::instrument(level = "debug", skip(self, data,))] async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result { let disks = self.disks.read().await; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 422a1d2c8..ee699d98e 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -47,6 +47,7 @@ use s3s::S3; use s3s::{S3Request, S3Response}; use std::fmt::Debug; use std::str::FromStr; +use tracing::debug; use tracing::error; use tracing::info; use transform_stream::AsyncTryStream; @@ -751,6 +752,7 @@ impl S3 for FS { content_length, tagging, metadata, + version_id, .. } = input; @@ -784,10 +786,12 @@ impl S3 for FS { metadata.insert(xhttp::AMZ_OBJECT_TAGGING.to_owned(), tags); } - let opts: ObjectOptions = put_opts(&bucket, &key, None, &req.headers, Some(metadata)) + let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(metadata)) .await .map_err(to_s3_error)?; + debug!("put_object opts {:?}", &opts); + let obj_info = store .put_object(&bucket, &key, &mut reader, &opts) .await @@ -810,7 +814,11 @@ impl S3 for FS { req: S3Request, ) -> S3Result> { let CreateMultipartUploadInput { - bucket, key, tagging, .. + bucket, + key, + tagging, + version_id, + .. } = req.input; // mc cp step 3 @@ -827,7 +835,7 @@ impl S3 for FS { metadata.insert(xhttp::AMZ_OBJECT_TAGGING.to_owned(), tags); } - let opts: ObjectOptions = put_opts(&bucket, &key, None, &req.headers, Some(metadata)) + let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(metadata)) .await .map_err(to_s3_error)?;