gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-28 09:08:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix: #38

Browse Source 
Squashed commit of the following:

commit ff72105e1ec81b5679401b021fb44e8135444de8
Author: weisd <weishidavip@163.com>
Date:   Sat Oct 5 09:55:09 2024 +0800

    base bucketmetadata_sys done

commit 69d0c7725ae1065d25755c71aa54e42c4f3cdb50
Author: weisd <weishidavip@163.com>
Date:   Sat Oct 5 01:41:19 2024 +0800

    update bucket tagging op use bucketmetadata_sys

commit 39902c73d6f7a817041245af84a23d14115c70dc
Author: weisd <weishidavip@163.com>
Date:   Fri Oct 4 23:32:07 2024 +0800

    test bucketmetadata_sys

commit 15f1dfc765dc383ab26d71aca7c64dd0917b83f3
Author: weisd <weishidavip@163.com>
Date:   Fri Oct 4 23:21:58 2024 +0800

    init bucketmetadata_sys

commit 8c3c5e0f7385fa881b1dc4811d92afe8b86e9a6f
Author: weisd <weishidavip@163.com>
Date:   Tue Oct 1 23:06:09 2024 +0800

    init bucketmetadata_sys

commit e2746d154aa68be9dcf8add0241b38c72ef89b9c
Author: weisd <weishidavip@163.com>
Date:   Tue Oct 1 04:35:25 2024 +0800

    stash
This commit is contained in:
weisd
2024-10-05 10:04:59 +08:00
parent 3a608e5554
commit 10c63effd4
45 changed files with 974 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/main.rs
+19 -3
View File
@@ -6,8 +6,12 @@ mod storage;
use clap::Parser;
use common::error::{Error, Result};
use ecstore::{
bucket::init_bucket_metadata_sys,
endpoints::EndpointServerPools,
store::{init_local_disks, update_erasure_type, ECStore},
set_global_endpoints,
store::{init_local_disks, ECStore},
store_api::{BucketOptions, StorageAPI},
update_erasure_type,
};
use grpc::make_server;
use hyper_util::{
@@ -89,7 +93,7 @@ async fn run(opt: config::Opt) -> Result<()> {
// 用于rpc
let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(opt.address.clone().as_str(), opt.volumes.clone())
.map_err(|err| Error::from_string(err.to_string()))?;
set_global_endpoints(endpoint_pools.as_ref().clone()).await;
update_erasure_type(setup_type).await;
// 初始化本地磁盘
@@ -179,11 +183,23 @@ async fn run(opt: config::Opt) -> Result<()> {
});
// init store
ECStore::new(opt.address.clone(), endpoint_pools.clone())
let store = ECStore::new(opt.address.clone(), endpoint_pools.clone())
.await
.map_err(|err| Error::from_string(err.to_string()))?;
info!(" init store success!");
let buckets_list = store
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.map_err(|err| Error::from_string(err.to_string()))?;
let buckets = buckets_list.iter().map(|v| v.name.clone()).collect();
init_bucket_metadata_sys(store.clone(), buckets).await;
tokio::select! {
_ = tokio::signal::ctrl_c() => {
rustfs/src/storage/ecfs.rs
+39 -127
View File
@@ -1,15 +1,19 @@
use bytes::BufMut;
use bytes::Bytes;
use ecstore::bucket::get_bucket_metadata_sys;
use ecstore::bucket::metadata::BUCKET_TAGGING_CONFIG;
use ecstore::bucket::tags::Tags;
use ecstore::bucket_meta::BucketMetadata;
use ecstore::disk::error::DiskError;
use ecstore::disk::RUSTFS_META_BUCKET;
use ecstore::store::new_object_layer_fn;
use ecstore::new_object_layer_fn;
use ecstore::store_api::BucketOptions;
use ecstore::store_api::CompletePart;
use ecstore::store_api::DeleteBucketOptions;
use ecstore::store_api::HTTPRangeSpec;
use ecstore::store_api::MakeBucketOptions;
use ecstore::store_api::MultipartUploadResult;
use ecstore::store_api::ObjectIO;
use ecstore::store_api::ObjectOptions;
use ecstore::store_api::ObjectToDelete;
use ecstore::store_api::PutObjReader;
@@ -17,6 +21,7 @@ use ecstore::store_api::StorageAPI;
use futures::pin_mut;
use futures::{Stream, StreamExt};
use http::HeaderMap;
use log::warn;
use s3s::dto::*;
use s3s::s3_error;
use s3s::Body;
@@ -25,6 +30,7 @@ use s3s::S3ErrorCode;
use s3s::S3Result;
use s3s::S3;
use s3s::{S3Request, S3Response};
use std::collections::HashMap;
use std::fmt::Debug;
use std::str::FromStr;
use transform_stream::AsyncTryStream;
@@ -248,7 +254,7 @@ impl S3 for FS {
None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
};
if let Err(e) = store.get_bucket_info(&input.bucket, &BucketOptions {}).await {
if let Err(e) = store.get_bucket_info(&input.bucket, &BucketOptions::default()).await {
if DiskError::VolumeNotFound.is(&e) {
return Err(s3_error!(NoSuchBucket));
} else {
@@ -322,7 +328,7 @@ impl S3 for FS {
None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
};
if let Err(e) = store.get_bucket_info(&input.bucket, &BucketOptions {}).await {
if let Err(e) = store.get_bucket_info(&input.bucket, &BucketOptions::default()).await {
if DiskError::VolumeNotFound.is(&e) {
return Err(s3_error!(NoSuchBucket));
} else {
@@ -373,7 +379,7 @@ impl S3 for FS {
None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
};
let bucket_infos = try_!(store.list_bucket(&BucketOptions {}).await);
let bucket_infos = try_!(store.list_bucket(&BucketOptions::default()).await);
let buckets: Vec<Bucket> = bucket_infos
.iter()
@@ -701,53 +707,19 @@ impl S3 for FS {
}))
.await?;
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = lock
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
let meta_obj = try_!(
store
.get_object_reader(
RUSTFS_META_BUCKET,
BucketMetadata::new(bucket.as_str()).save_file_path().as_str(),
HTTPRangeSpec::nil(),
Default::default(),
&ObjectOptions::default(),
)
.await
);
let stream = meta_obj.stream;
let mut data = vec![];
pin_mut!(stream);
while let Some(x) = stream.next().await {
let x = try_!(x);
data.put_slice(&x[..]);
let mut tag_map = HashMap::new();
for tag in tagging.tag_set.iter() {
tag_map.insert(tag.key.clone(), tag.value.clone());
}
let mut meta = try_!(BucketMetadata::unmarshal_from(&data[..]));
if tagging.tag_set.is_empty() {
meta.tagging = None;
} else {
meta.tagging = Some(tagging.tag_set.into_iter().map(|x| (x.key, x.value)).collect())
}
let tags = Tags::new(tag_map, false);
let data = try_!(meta.marshal_msg());
let len = data.len();
try_!(
store
.put_object(
RUSTFS_META_BUCKET,
BucketMetadata::new(bucket.as_str()).save_file_path().as_str(),
PutObjReader::new(StreamingBlob::from(Body::from(data)), len),
&ObjectOptions::default(),
)
.await
);
let data = try_!(tags.marshal_msg());
let _updated = try_!(bucket_meta_sys.update(&bucket, BUCKET_TAGGING_CONFIG, data).await);
Ok(S3Response::new(Default::default()))
}
@@ -763,51 +735,23 @@ impl S3 for FS {
}))
.await?;
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = lock
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
let meta_obj = try_!(
store
.get_object_reader(
RUSTFS_META_BUCKET,
BucketMetadata::new(bucket.as_str()).save_file_path().as_str(),
HTTPRangeSpec::nil(),
Default::default(),
&ObjectOptions::default(),
)
.await
);
let stream = meta_obj.stream;
let mut data = vec![];
pin_mut!(stream);
while let Some(x) = stream.next().await {
let x = try_!(x);
data.put_slice(&x[..]);
}
let meta = try_!(BucketMetadata::unmarshal_from(&data[..]));
if meta.tagging.is_none() {
return Err({
let mut err = S3Error::with_message(S3ErrorCode::Custom("NoSuchTagSet".into()), "The TagSet does not exist");
err.set_status_code("404".try_into().unwrap());
err
});
}
Ok(S3Response::new(GetBucketTaggingOutput {
tag_set: meta
.tagging
.unwrap()
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
let tag_set: Vec<Tag> = match bucket_meta_sys.get_tagging_config(&bucket).await {
Ok((tags, _)) => tags
.tag_set
.tag_map
.into_iter()
.map(|(key, value)| Tag { key, value })
.collect(),
}))
Err(err) => {
warn!("get_tagging_config err {:?}", &err);
// TODO: check not found
Vec::new()
}
};
Ok(S3Response::new(GetBucketTaggingOutput { tag_set }))
}
#[tracing::instrument(level = "debug", skip(self))]
@@ -824,48 +768,16 @@ impl S3 for FS {
}))
.await?;
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = lock
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
let meta_obj = try_!(
store
.get_object_reader(
RUSTFS_META_BUCKET,
BucketMetadata::new(bucket.as_str()).save_file_path().as_str(),
HTTPRangeSpec::nil(),
Default::default(),
&ObjectOptions::default(),
)
.await
);
let tag_map = HashMap::new();
let stream = meta_obj.stream;
let tags = Tags::new(tag_map, false);
let mut data = vec![];
pin_mut!(stream);
let data = try_!(tags.marshal_msg());
while let Some(x) = stream.next().await {
let x = try_!(x);
data.put_slice(&x[..]);
}
let mut meta = try_!(BucketMetadata::unmarshal_from(&data[..]));
meta.tagging = None;
let data = try_!(meta.marshal_msg());
let len = data.len();
try_!(
store
.put_object(
RUSTFS_META_BUCKET,
BucketMetadata::new(bucket.as_str()).save_file_path().as_str(),
PutObjReader::new(StreamingBlob::from(Body::from(data)), len),
&ObjectOptions::default(),
)
.await
);
let _updated = try_!(bucket_meta_sys.update(&bucket, BUCKET_TAGGING_CONFIG, data).await);
Ok(S3Response::new(DeleteBucketTaggingOutput {}))
}