gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-19 02:56:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into bool-api

Browse Source
This commit is contained in:
weisd
2024-11-12 16:18:11 +08:00
parent 3edaefe168 e50530bc44
commit 47ade0f135
22 changed files with 568 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/bucket/metadata.rs
+1 -28
View File
@@ -22,6 +22,7 @@ use crate::error::{Error, Result};
use crate::disk::BUCKET_META_PREFIX;
use crate::store::ECStore;
use crate::utils::xml::deserialize;
pub const BUCKET_METADATA_FILE: &str = ".metadata.bin";
pub const BUCKET_METADATA_FORMAT: u16 = 1;
@@ -427,31 +428,3 @@ mod test {
assert_eq!(bm.name, new.name);
}
}
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
where
T: for<'xml> xml::Deserialize<'xml>,
{
let mut d = xml::Deserializer::new(input);
let ans = T::deserialize(&mut d)?;
d.expect_eof()?;
Ok(ans)
}
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
let mut buf = Vec::with_capacity(256);
{
let mut ser = xml::Serializer::new(&mut buf);
val.serialize_content(&mut ser)?;
}
Ok(String::from_utf8(buf).unwrap())
}
pub fn serialize<T: xml::Serialize>(val: &T) -> xml::SerResult<Vec<u8>> {
let mut buf = Vec::with_capacity(256);
{
let mut ser = xml::Serializer::new(&mut buf);
val.serialize(&mut ser)?;
}
Ok(buf)
}
ecstore/src/bucket/metadata_sys.rs
+2 -1
View File
@@ -10,6 +10,7 @@ use crate::disk::error::DiskError;
use crate::error::{Error, Result};
use crate::global::{is_dist_erasure, is_erasure, new_object_layer_fn, GLOBAL_Endpoints};
use crate::store::ECStore;
use crate::utils::xml::deserialize;
use futures::future::join_all;
use s3s::dto::{
BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,
@@ -19,7 +20,7 @@ use time::OffsetDateTime;
use tokio::sync::RwLock;
use tracing::{error, warn};
use super::metadata::{deserialize, load_bucket_metadata, BucketMetadata};
use super::metadata::{load_bucket_metadata, BucketMetadata};
use super::policy::bucket_policy::BucketPolicy;
use super::quota::BucketQuota;
use super::target::BucketTargets;
ecstore/src/config/storageclass.rs
+22 -25
View File
@@ -53,31 +53,28 @@ pub static DEFAULT_INLINE_BLOCK: usize = 128 * 1024;
lazy_static! {
pub static ref DefaultKVS: KVS = {
let mut kvs = Vec::new();
kvs.push(KV {
key: CLASS_STANDARD.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
});
kvs.push(KV {
key: CLASS_RRS.to_owned(),
value: "EC:1".to_owned(),
hidden_if_empty: false,
});
kvs.push(KV {
key: OPTIMIZE.to_owned(),
value: "availability".to_owned(),
hidden_if_empty: false,
});
kvs.push(KV {
key: INLINE_BLOCK.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
});
let kvs = vec![
KV {
key: CLASS_STANDARD.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: CLASS_RRS.to_owned(),
value: "EC:1".to_owned(),
hidden_if_empty: false,
},
KV {
key: OPTIMIZE.to_owned(),
value: "availability".to_owned(),
hidden_if_empty: false,
},
KV {
key: INLINE_BLOCK.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
];
KVS(kvs)
};
ecstore/src/erasure.rs
+33 -24
View File
@@ -67,34 +67,36 @@ impl Erasure {
);
let mut total: usize = 0;
loop {
let new_len = {
let remain = total_size - total;
if remain > self.block_size {
self.block_size
} else {
remain
}
};
if new_len == 0 {
break;
}
self.buf.resize(new_len, 0u8);
match reader.read_exact(&mut self.buf).await {
Ok(res) => res,
Err(e) => {
if let ErrorKind::UnexpectedEof = e.kind() {
break;
if total_size > 0 {
let new_len = {
let remain = total_size - total;
if remain > self.block_size {
self.block_size
} else {
return Err(Error::new(e));
remain
}
}
};
};
total += self.buf.len();
if new_len == 0 && total > 0 {
break;
}
self.buf.resize(new_len, 0u8);
match reader.read_exact(&mut self.buf).await {
Ok(res) => res,
Err(e) => {
if let ErrorKind::UnexpectedEof = e.kind() {
break;
} else {
return Err(Error::new(e));
}
}
};
total += self.buf.len();
}
let blocks = self.encode_data(&self.buf)?;
let mut errs = Vec::new();
@@ -112,6 +114,9 @@ impl Erasure {
let none_count = errs.iter().filter(|&x| x.is_none()).count();
if none_count >= write_quorum {
if total_size == 0 {
break;
}
continue;
}
@@ -119,6 +124,10 @@ impl Erasure {
warn!("Erasure encode errs {:?}", &errs);
return Err(err);
}
if total_size == 0 {
break;
}
}
Ok(total)
ecstore/src/lib.rs
+1 -1
View File
@@ -17,7 +17,7 @@ mod sets;
pub mod store;
pub mod store_api;
mod store_init;
mod utils;
pub mod utils;
pub mod bucket;
pub mod file_meta_inline;
ecstore/src/options.rs
+1 -1
View File
@@ -64,7 +64,7 @@ pub fn put_opts_from_headers(
}
fn get_default_opts(
headers: &HeaderMap<HeaderValue>,
_headers: &HeaderMap<HeaderValue>,
metadata: HashMap<String, String>,
_copy_source: bool,
) -> Result<ObjectOptions> {
ecstore/src/store.rs
+4 -3
View File
@@ -22,6 +22,7 @@ use crate::store_err::{
use crate::store_init::ec_drives_no_config;
use crate::utils::crypto::base64_decode;
use crate::utils::path::{base_dir_from_prefix, decode_dir_object, encode_dir_object, SLASH_SEPARATOR};
use crate::utils::xml;
use crate::{
bucket::metadata::BucketMetadata,
disk::{error::DiskError, new_disk, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
@@ -954,12 +955,12 @@ impl StorageAPI for ECStore {
}
if opts.lock_enabled {
meta.object_lock_config_xml = metadata::serialize::<ObjectLockConfiguration>(&enableObjcetLockConfig)?;
meta.versioning_config_xml = metadata::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
meta.object_lock_config_xml = xml::serialize::<ObjectLockConfiguration>(&enableObjcetLockConfig)?;
meta.versioning_config_xml = xml::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
}
if opts.versioning_enabled {
meta.versioning_config_xml = metadata::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
meta.versioning_config_xml = xml::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
}
meta.save(self).await?;
ecstore/src/utils/mod.rs
+1
View File
@@ -6,3 +6,4 @@ pub mod net;
pub mod os;
pub mod path;
pub mod wildcard;
pub mod xml;
ecstore/src/utils/path.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use std::path::{Path, PathBuf};
use std::path::PathBuf;
const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__";
ecstore/src/utils/xml.rs
+29
View File
@@ -0,0 +1,29 @@
use s3s::xml;
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
where
T: for<'xml> xml::Deserialize<'xml>,
{
let mut d = xml::Deserializer::new(input);
let ans = T::deserialize(&mut d)?;
d.expect_eof()?;
Ok(ans)
}
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
let mut buf = Vec::with_capacity(256);
{
let mut ser = xml::Serializer::new(&mut buf);
val.serialize_content(&mut ser)?;
}
Ok(String::from_utf8(buf).unwrap())
}
pub fn serialize<T: xml::Serialize>(val: &T) -> xml::SerResult<Vec<u8>> {
let mut buf = Vec::with_capacity(256);
{
let mut ser = xml::Serializer::new(&mut buf);
val.serialize(&mut ser)?;
}
Ok(buf)
}
ecstore/src/xhttp.rs
+1
View File
@@ -1,2 +1,3 @@
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
pub const AMZ_STORAGE_CLASS: &str = "x-amz-storage-class";
pub const AMZ_DECODED_CONTENT_LENGTH: &str = "X-Amz-Decoded-Content-Length";