merge versioning, fix bug todo

This commit is contained in:
weisd
2024-11-02 00:21:10 +08:00
parent 28dc7379a6
commit 09ea11c13d
65 changed files with 5187 additions and 1966 deletions
+1
View File
@@ -41,6 +41,7 @@ tokio = { workspace = true, features = [
"net",
"signal",
] }
lazy_static.workspace = true
tokio-stream.workspace = true
tonic = { version = "0.12.3", features = ["gzip"] }
tonic-reflection.workspace = true
+22 -5
View File
@@ -16,7 +16,18 @@ use lock::{lock_args::LockArgs, Locker, GLOBAL_LOCAL_SERVER};
use protos::{
models::{PingBody, PingBodyBuilder},
proto_gen::node_service::{
node_service_server::NodeService as Node, CheckPartsRequest, CheckPartsResponse, DeleteBucketRequest, DeleteBucketResponse, DeletePathsRequest, DeletePathsResponse, DeleteRequest, DeleteResponse, DeleteVersionRequest, DeleteVersionResponse, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DeleteVolumeResponse, DiskInfoRequest, DiskInfoResponse, GenerallyLockRequest, GenerallyLockResponse, GetBucketInfoRequest, GetBucketInfoResponse, ListBucketRequest, ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse, ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, RenameDataRequest, RenameDataResponse, RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse, StatVolumeRequest, StatVolumeResponse, UpdateMetadataRequest, UpdateMetadataResponse, VerifyFileRequest, VerifyFileResponse, WalkDirRequest, WalkDirResponse, WriteAllRequest, WriteAllResponse, WriteMetadataRequest, WriteMetadataResponse, WriteRequest, WriteResponse
node_service_server::NodeService as Node, CheckPartsRequest, CheckPartsResponse, DeleteBucketRequest,
DeleteBucketResponse, DeletePathsRequest, DeletePathsResponse, DeleteRequest, DeleteResponse, DeleteVersionRequest,
DeleteVersionResponse, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DeleteVolumeResponse,
DiskInfoRequest, DiskInfoResponse, GenerallyLockRequest, GenerallyLockResponse, GetBucketInfoRequest,
GetBucketInfoResponse, ListBucketRequest, ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest,
ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest,
MakeVolumesResponse, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse,
ReadMultipleRequest, ReadMultipleResponse, ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse,
RenameDataRequest, RenameDataResponse, RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse,
StatVolumeRequest, StatVolumeResponse, UpdateMetadataRequest, UpdateMetadataResponse, VerifyFileRequest,
VerifyFileResponse, WalkDirRequest, WalkDirResponse, WriteAllRequest, WriteAllResponse, WriteMetadataRequest,
WriteMetadataResponse, WriteRequest, WriteResponse,
},
};
use tokio::sync::mpsc;
@@ -205,7 +216,13 @@ impl Node for NodeService {
let request = request.into_inner();
match self
.local_peer
.delete_bucket(&request.bucket, &DeleteBucketOptions { force: false })
.delete_bucket(
&request.bucket,
&DeleteBucketOptions {
force: false,
..Default::default()
},
)
.await
{
Ok(_) => Ok(tonic::Response::new(DeleteBucketResponse {
@@ -326,7 +343,7 @@ impl Node for NodeService {
check_parts_resp,
error_info: None,
}))
},
}
Err(err) => Ok(tonic::Response::new(VerifyFileResponse {
success: false,
check_parts_resp: "".to_string(),
@@ -372,7 +389,7 @@ impl Node for NodeService {
check_parts_resp,
error_info: None,
}))
},
}
Err(err) => Ok(tonic::Response::new(CheckPartsResponse {
success: false,
check_parts_resp: "".to_string(),
@@ -914,7 +931,7 @@ impl Node for NodeService {
}
};
match disk.update_metadata(&request.volume, &request.path, file_info, opts).await {
match disk.update_metadata(&request.volume, &request.path, file_info, &opts).await {
Ok(_) => Ok(tonic::Response::new(UpdateMetadataResponse {
success: true,
error_info: None,
+6 -30
View File
@@ -6,11 +6,10 @@ mod storage;
use clap::Parser;
use common::error::{Error, Result};
use ecstore::{
bucket::metadata_sys::init_bucket_metadata_sys,
config::GLOBAL_ConfigSys,
endpoints::EndpointServerPools,
set_global_endpoints,
store::{init_local_disks, ECStore},
store_api::{BucketOptions, StorageAPI},
update_erasure_type,
};
use grpc::make_server;
@@ -74,22 +73,6 @@ async fn run(opt: config::Opt) -> Result<()> {
//获取监听地址
let local_addr: SocketAddr = listener.local_addr()?;
// let mut domain_name = {
// netif::up()?
// .map(|x| x.address().to_owned())
// .filter(|v| v.is_ipv4())
// // .filter(|v| v.is_ipv4() && !v.is_loopback() && !v.is_unspecified())
// .map(|v| format!("{}", v))
// .next()
// .and_then(|ip| {
// if let SocketAddr::V4(ipv4) = local_addr {
// Some(format!("{}:{}", ip, ipv4.port()))
// } else {
// None
// }
// })
// };
// 用于rpc
let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(opt.address.clone().as_str(), opt.volumes.clone())
.map_err(|err| Error::from_string(err.to_string()))?;
@@ -112,8 +95,9 @@ async fn run(opt: config::Opt) -> Result<()> {
// Setup S3 service
// 本项目使用s3s库来实现s3服务
let service = {
let store = storage::ecfs::FS::new();
// let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(opt.address.clone(), endpoint_pools).await?);
let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new());
let mut b = S3ServiceBuilder::new(store.clone());
//设置AK和SK
//其中部份内容从config配置文件中读取
let mut access_key = String::from_str(config::DEFAULT_ACCESS_KEY).unwrap();
@@ -128,6 +112,8 @@ async fn run(opt: config::Opt) -> Result<()> {
info!("authentication is enabled {}, {}", &access_key, &secret_key);
b.set_auth(SimpleAuth::from_single(access_key, secret_key));
b.set_access(store.clone());
// // Enable parsing virtual-hosted-style requests
// if let Some(dm) = opt.domain_name {
// info!("virtual-hosted-style requests are enabled use domain_name {}", &dm);
@@ -197,17 +183,7 @@ async fn run(opt: config::Opt) -> Result<()> {
.await
.map_err(|err| Error::from_string(err.to_string()))?;
let buckets_list = store
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.map_err(|err| Error::from_string(err.to_string()))?;
let buckets = buckets_list.iter().map(|v| v.name.clone()).collect();
init_bucket_metadata_sys(store.clone(), buckets).await;
store.init().await.map_err(|err| Error::from_string(err.to_string()))?;
tokio::select! {
_ = tokio::signal::ctrl_c() => {
+779
View File
@@ -0,0 +1,779 @@
use super::ecfs::FS;
use ecstore::bucket::policy::action::Action;
use http::HeaderMap;
use s3s::access::{S3Access, S3AccessContext};
use s3s::auth::Credentials;
use s3s::{dto::*, s3_error, S3Request, S3Result};
use uuid::Uuid;
#[derive(Debug, Default, Clone)]
struct ReqInfo {
pub card: Option<Credentials>,
pub action: Option<Action>,
pub bucket: Option<String>,
pub object: Option<String>,
pub version_id: Option<Uuid>,
}
async fn authorize_request(_req: &ReqInfo, _hs: &HeaderMap, _action: Action) -> S3Result<()> {
// TODO: globalIAMSys.IsAllowed
Ok(())
}
#[async_trait::async_trait]
impl S3Access for FS {
// /// Checks whether the current request has accesses to the resources.
// ///
// /// This method is called before deserializing the operation input.
// ///
// /// By default, this method rejects all anonymous requests
// /// and returns [`AccessDenied`](crate::S3ErrorCode::AccessDenied) error.
// ///
// /// An access control provider can override this method to implement custom logic.
// ///
// /// Common fields in the context:
// /// + [`cx.credentials()`](S3AccessContext::credentials)
// /// + [`cx.s3_path()`](S3AccessContext::s3_path)
// /// + [`cx.s3_op().name()`](crate::S3Operation::name)
// /// + [`cx.extensions_mut()`](S3AccessContext::extensions_mut)
async fn check(&self, cx: &mut S3AccessContext<'_>) -> S3Result<()> {
// 上层验证了 ak/sk
// warn!("check s3_op {:?} cred {:?}", cx.s3_op().name(), cx.credentials());
let action = Action::from_str(format!("s3:{}", cx.s3_op().name()).as_str());
let req_info = ReqInfo {
card: cx.credentials().cloned(),
action: action,
..Default::default()
};
// warn!("req_info {:?}", req_info);
let ext = cx.extensions_mut();
ext.insert(req_info);
// 统一在这验证?还是在下面各自验证?
Ok(())
}
/// Checks whether the CreateBucket request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn create_bucket(&self, req: &mut S3Request<CreateBucketInput>) -> S3Result<()> {
if let Some(req_info) = req.extensions.get_mut::<ReqInfo>() {
let CreateBucketInput { bucket, .. } = &req.input;
req_info.bucket = Some(bucket.to_owned());
authorize_request(req_info, &req.headers, Action::CreateBucket).await
} else {
Err(s3_error!(AccessDenied, "AccessDenied"))
}
}
/// Checks whether the AbortMultipartUpload request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn abort_multipart_upload(&self, _req: &mut S3Request<AbortMultipartUploadInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the CompleteMultipartUpload request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn complete_multipart_upload(&self, _req: &mut S3Request<CompleteMultipartUploadInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the CopyObject request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn copy_object(&self, _req: &mut S3Request<CopyObjectInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the CreateMultipartUpload request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn create_multipart_upload(&self, _req: &mut S3Request<CreateMultipartUploadInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucket request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket(&self, _req: &mut S3Request<DeleteBucketInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucketAnalyticsConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket_analytics_configuration(
&self,
_req: &mut S3Request<DeleteBucketAnalyticsConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucketCors request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket_cors(&self, _req: &mut S3Request<DeleteBucketCorsInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucketEncryption request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket_encryption(&self, _req: &mut S3Request<DeleteBucketEncryptionInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucketIntelligentTieringConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket_intelligent_tiering_configuration(
&self,
_req: &mut S3Request<DeleteBucketIntelligentTieringConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucketInventoryConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket_inventory_configuration(
&self,
_req: &mut S3Request<DeleteBucketInventoryConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucketLifecycle request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket_lifecycle(&self, _req: &mut S3Request<DeleteBucketLifecycleInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucketMetricsConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket_metrics_configuration(
&self,
_req: &mut S3Request<DeleteBucketMetricsConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucketOwnershipControls request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket_ownership_controls(&self, _req: &mut S3Request<DeleteBucketOwnershipControlsInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucketPolicy request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket_policy(&self, _req: &mut S3Request<DeleteBucketPolicyInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucketReplication request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket_replication(&self, _req: &mut S3Request<DeleteBucketReplicationInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucketTagging request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket_tagging(&self, _req: &mut S3Request<DeleteBucketTaggingInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteBucketWebsite request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_bucket_website(&self, _req: &mut S3Request<DeleteBucketWebsiteInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteObject request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_object(&self, _req: &mut S3Request<DeleteObjectInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteObjectTagging request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_object_tagging(&self, _req: &mut S3Request<DeleteObjectTaggingInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeleteObjects request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_objects(&self, _req: &mut S3Request<DeleteObjectsInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the DeletePublicAccessBlock request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn delete_public_access_block(&self, _req: &mut S3Request<DeletePublicAccessBlockInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketAccelerateConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_accelerate_configuration(
&self,
_req: &mut S3Request<GetBucketAccelerateConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketAcl request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_acl(&self, _req: &mut S3Request<GetBucketAclInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketAnalyticsConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_analytics_configuration(
&self,
_req: &mut S3Request<GetBucketAnalyticsConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketCors request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_cors(&self, _req: &mut S3Request<GetBucketCorsInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketEncryption request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_encryption(&self, _req: &mut S3Request<GetBucketEncryptionInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketIntelligentTieringConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_intelligent_tiering_configuration(
&self,
_req: &mut S3Request<GetBucketIntelligentTieringConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketInventoryConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_inventory_configuration(
&self,
_req: &mut S3Request<GetBucketInventoryConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketLifecycleConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_lifecycle_configuration(
&self,
_req: &mut S3Request<GetBucketLifecycleConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketLocation request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_location(&self, _req: &mut S3Request<GetBucketLocationInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketLogging request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_logging(&self, _req: &mut S3Request<GetBucketLoggingInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketMetricsConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_metrics_configuration(&self, _req: &mut S3Request<GetBucketMetricsConfigurationInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketNotificationConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_notification_configuration(
&self,
_req: &mut S3Request<GetBucketNotificationConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketOwnershipControls request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_ownership_controls(&self, _req: &mut S3Request<GetBucketOwnershipControlsInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketPolicy request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_policy(&self, _req: &mut S3Request<GetBucketPolicyInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketPolicyStatus request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_policy_status(&self, _req: &mut S3Request<GetBucketPolicyStatusInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketReplication request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_replication(&self, _req: &mut S3Request<GetBucketReplicationInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketRequestPayment request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_request_payment(&self, _req: &mut S3Request<GetBucketRequestPaymentInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketTagging request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_tagging(&self, _req: &mut S3Request<GetBucketTaggingInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketVersioning request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_versioning(&self, _req: &mut S3Request<GetBucketVersioningInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetBucketWebsite request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_bucket_website(&self, _req: &mut S3Request<GetBucketWebsiteInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetObject request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_object(&self, _req: &mut S3Request<GetObjectInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetObjectAcl request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_object_acl(&self, _req: &mut S3Request<GetObjectAclInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetObjectAttributes request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_object_attributes(&self, _req: &mut S3Request<GetObjectAttributesInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetObjectLegalHold request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_object_legal_hold(&self, _req: &mut S3Request<GetObjectLegalHoldInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetObjectLockConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_object_lock_configuration(&self, _req: &mut S3Request<GetObjectLockConfigurationInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetObjectRetention request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_object_retention(&self, _req: &mut S3Request<GetObjectRetentionInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetObjectTagging request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_object_tagging(&self, _req: &mut S3Request<GetObjectTaggingInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetObjectTorrent request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_object_torrent(&self, _req: &mut S3Request<GetObjectTorrentInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the GetPublicAccessBlock request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn get_public_access_block(&self, _req: &mut S3Request<GetPublicAccessBlockInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the HeadBucket request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn head_bucket(&self, _req: &mut S3Request<HeadBucketInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the HeadObject request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn head_object(&self, _req: &mut S3Request<HeadObjectInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the ListBucketAnalyticsConfigurations request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn list_bucket_analytics_configurations(
&self,
_req: &mut S3Request<ListBucketAnalyticsConfigurationsInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the ListBucketIntelligentTieringConfigurations request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn list_bucket_intelligent_tiering_configurations(
&self,
_req: &mut S3Request<ListBucketIntelligentTieringConfigurationsInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the ListBucketInventoryConfigurations request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn list_bucket_inventory_configurations(
&self,
_req: &mut S3Request<ListBucketInventoryConfigurationsInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the ListBucketMetricsConfigurations request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn list_bucket_metrics_configurations(
&self,
_req: &mut S3Request<ListBucketMetricsConfigurationsInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the ListBuckets request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn list_buckets(&self, _req: &mut S3Request<ListBucketsInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the ListMultipartUploads request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn list_multipart_uploads(&self, _req: &mut S3Request<ListMultipartUploadsInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the ListObjectVersions request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn list_object_versions(&self, _req: &mut S3Request<ListObjectVersionsInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the ListObjects request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn list_objects(&self, _req: &mut S3Request<ListObjectsInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the ListObjectsV2 request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn list_objects_v2(&self, _req: &mut S3Request<ListObjectsV2Input>) -> S3Result<()> {
Ok(())
}
/// Checks whether the ListParts request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn list_parts(&self, _req: &mut S3Request<ListPartsInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketAccelerateConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_accelerate_configuration(
&self,
_req: &mut S3Request<PutBucketAccelerateConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketAcl request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_acl(&self, _req: &mut S3Request<PutBucketAclInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketAnalyticsConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_analytics_configuration(
&self,
_req: &mut S3Request<PutBucketAnalyticsConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketCors request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_cors(&self, _req: &mut S3Request<PutBucketCorsInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketEncryption request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_encryption(&self, _req: &mut S3Request<PutBucketEncryptionInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketIntelligentTieringConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_intelligent_tiering_configuration(
&self,
_req: &mut S3Request<PutBucketIntelligentTieringConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketInventoryConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_inventory_configuration(
&self,
_req: &mut S3Request<PutBucketInventoryConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketLifecycleConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_lifecycle_configuration(
&self,
_req: &mut S3Request<PutBucketLifecycleConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketLogging request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_logging(&self, _req: &mut S3Request<PutBucketLoggingInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketMetricsConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_metrics_configuration(&self, _req: &mut S3Request<PutBucketMetricsConfigurationInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketNotificationConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_notification_configuration(
&self,
_req: &mut S3Request<PutBucketNotificationConfigurationInput>,
) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketOwnershipControls request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_ownership_controls(&self, _req: &mut S3Request<PutBucketOwnershipControlsInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketPolicy request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_policy(&self, _req: &mut S3Request<PutBucketPolicyInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketReplication request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_replication(&self, _req: &mut S3Request<PutBucketReplicationInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketRequestPayment request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_request_payment(&self, _req: &mut S3Request<PutBucketRequestPaymentInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketTagging request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_tagging(&self, _req: &mut S3Request<PutBucketTaggingInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketVersioning request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_versioning(&self, _req: &mut S3Request<PutBucketVersioningInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutBucketWebsite request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_bucket_website(&self, _req: &mut S3Request<PutBucketWebsiteInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutObject request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_object(&self, _req: &mut S3Request<PutObjectInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutObjectAcl request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_object_acl(&self, _req: &mut S3Request<PutObjectAclInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutObjectLegalHold request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_object_legal_hold(&self, _req: &mut S3Request<PutObjectLegalHoldInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutObjectLockConfiguration request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_object_lock_configuration(&self, _req: &mut S3Request<PutObjectLockConfigurationInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutObjectRetention request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_object_retention(&self, _req: &mut S3Request<PutObjectRetentionInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutObjectTagging request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_object_tagging(&self, _req: &mut S3Request<PutObjectTaggingInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the PutPublicAccessBlock request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn put_public_access_block(&self, _req: &mut S3Request<PutPublicAccessBlockInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the RestoreObject request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn restore_object(&self, _req: &mut S3Request<RestoreObjectInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the SelectObjectContent request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn select_object_content(&self, _req: &mut S3Request<SelectObjectContentInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the UploadPart request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn upload_part(&self, _req: &mut S3Request<UploadPartInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the UploadPartCopy request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn upload_part_copy(&self, _req: &mut S3Request<UploadPartCopyInput>) -> S3Result<()> {
Ok(())
}
/// Checks whether the WriteGetObjectResponse request has accesses to the resources.
///
/// This method returns `Ok(())` by default.
async fn write_get_object_response(&self, _req: &mut S3Request<WriteGetObjectResponseInput>) -> S3Result<()> {
Ok(())
}
}
+234 -32
View File
@@ -1,4 +1,5 @@
use bytes::Bytes;
use common::error::Result;
use ecstore::bucket::error::BucketMetadataError;
use ecstore::bucket::metadata;
use ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
@@ -12,9 +13,12 @@ use ecstore::bucket::metadata::OBJECT_LOCK_CONFIG;
use ecstore::bucket::metadata_sys;
use ecstore::bucket::policy::bucket_policy::BucketPolicy;
use ecstore::bucket::policy_sys::PolicySys;
use ecstore::bucket::tagging::decode_tags;
use ecstore::bucket::tagging::encode_tags;
use ecstore::bucket::versioning_sys::BucketVersioningSys;
use ecstore::disk::error::DiskError;
use ecstore::new_object_layer_fn;
use ecstore::options::put_opts;
use ecstore::store_api::BucketOptions;
use ecstore::store_api::CompletePart;
use ecstore::store_api::DeleteBucketOptions;
@@ -29,6 +33,7 @@ use ecstore::store_api::StorageAPI;
use futures::pin_mut;
use futures::{Stream, StreamExt};
use http::HeaderMap;
use lazy_static::lazy_static;
use log::warn;
use s3s::dto::*;
use s3s::s3_error;
@@ -39,13 +44,11 @@ use s3s::S3;
use s3s::{S3Request, S3Response};
use std::fmt::Debug;
use std::str::FromStr;
use tracing::debug;
use tracing::info;
use transform_stream::AsyncTryStream;
use uuid::Uuid;
use common::error::Result;
use tracing::debug;
macro_rules! try_ {
($result:expr) => {
match $result {
@@ -57,7 +60,13 @@ macro_rules! try_ {
};
}
#[derive(Debug)]
lazy_static! {
static ref RUSTFS_OWNER: Owner = Owner {
display_name: Some("rustfs".to_owned()),
id: Some("c19050dbcee97fda828689dda99097a6321af2248fa760517237346e5d9c8a66".to_owned()),
};
}
#[derive(Debug, Clone)]
pub struct FS {
// pub store: ECStore,
}
@@ -130,7 +139,13 @@ impl S3 for FS {
};
try_!(
store
.delete_bucket(&input.bucket, &DeleteBucketOptions { force: false })
.delete_bucket(
&input.bucket,
&DeleteBucketOptions {
force: false,
..Default::default()
}
)
.await
);
@@ -399,6 +414,7 @@ impl S3 for FS {
let output = ListBucketsOutput {
buckets: Some(buckets),
owner: Some(RUSTFS_OWNER.to_owned()),
..Default::default()
};
Ok(S3Response::new(output))
@@ -513,16 +529,17 @@ impl S3 for FS {
key,
metadata,
content_length,
content_type,
checksum_sha256,
content_md5,
..
} = input;
debug!("put_object metadata {:?}", metadata);
let Some(body) = body else { return Err(s3_error!(IncompleteBody)) };
let Some(content_length) = content_length else { return Err(s3_error!(IncompleteBody)) };
let reader = PutObjReader::new(body, content_length as usize);
let mut reader = PutObjReader::new(body, content_length as usize);
let layer = new_object_layer_fn();
let lock = layer.read().await;
@@ -531,11 +548,18 @@ impl S3 for FS {
None => return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())),
};
try_!(store.put_object(&bucket, &key, reader, &ObjectOptions::default()).await);
let opts: ObjectOptions = try_!(put_opts(&bucket, &key, None, &req.headers, metadata).await);
let obj_info = try_!(store.put_object(&bucket, &key, &mut reader, &opts).await);
let e_tag = obj_info.etag;
// store.put_object(bucket, object, data, opts);
let output = PutObjectOutput { ..Default::default() };
let output = PutObjectOutput {
e_tag,
..Default::default()
};
Ok(S3Response::new(output))
}
@@ -592,7 +616,7 @@ impl S3 for FS {
let content_length = content_length.ok_or_else(|| s3_error!(IncompleteBody))?;
// mc cp step 4
let data = PutObjReader::new(body, content_length as usize);
let mut data = PutObjReader::new(body, content_length as usize);
let opts = ObjectOptions::default();
let layer = new_object_layer_fn();
@@ -602,9 +626,16 @@ impl S3 for FS {
None => return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())),
};
try_!(store.put_object_part(&bucket, &key, &upload_id, part_id, data, &opts).await);
let info = try_!(
store
.put_object_part(&bucket, &key, &upload_id, part_id, &mut data, &opts)
.await
);
let output = UploadPartOutput { ..Default::default() };
let output = UploadPartOutput {
e_tag: info.etag,
..Default::default()
};
Ok(S3Response::new(output))
}
@@ -766,7 +797,7 @@ impl S3 for FS {
Ok(S3Response::new(DeleteBucketTaggingOutput {}))
}
#[tracing::instrument(level = "debug", skip(self))]
#[tracing::instrument(level = "debug", skip(self, req))]
async fn put_object_tagging(&self, req: S3Request<PutObjectTaggingInput>) -> S3Result<S3Response<PutObjectTaggingOutput>> {
let PutObjectTaggingInput {
bucket,
@@ -781,12 +812,15 @@ impl S3 for FS {
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
let mut object_info = try_!(store.get_object_info(&bucket, &object, &ObjectOptions::default()).await);
object_info.tags = Some(tagging.tag_set.into_iter().map(|Tag { key, value }| (key, value)).collect());
// let mut object_info = try_!(store.get_object_info(&bucket, &object, &ObjectOptions::default()).await);
let tags = encode_tags(tagging.tag_set);
// TODO: getOpts
// TODO: Replicate
try_!(
store
.put_object_info(&bucket, &object, object_info, &ObjectOptions::default())
.put_object_tags(&bucket, &object, &tags, &ObjectOptions::default())
.await
);
@@ -803,13 +837,13 @@ impl S3 for FS {
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
let object_info = try_!(store.get_object_info(&bucket, &object, &ObjectOptions::default()).await);
// TODO: version
let tags = try_!(store.get_object_tags(&bucket, &object, &ObjectOptions::default()).await);
let tag_set = decode_tags(tags.as_str());
Ok(S3Response::new(GetObjectTaggingOutput {
tag_set: object_info
.tags
.map(|tags| tags.into_iter().map(|(key, value)| Tag { key, value }).collect())
.unwrap_or_else(Vec::new),
tag_set,
version_id: None,
}))
}
@@ -827,18 +861,21 @@ impl S3 for FS {
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
let mut object_info = try_!(store.get_object_info(&bucket, &object, &ObjectOptions::default()).await);
object_info.tags = None;
try_!(
store
.put_object_info(&bucket, &object, object_info, &ObjectOptions::default())
.await
);
// TODO: Replicate
// TODO: version
try_!(store.delete_object_tags(&bucket, &object, &ObjectOptions::default()).await);
Ok(S3Response::new(DeleteObjectTaggingOutput { version_id: None }))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn list_object_versions(
&self,
_req: S3Request<ListObjectVersionsInput>,
) -> S3Result<S3Response<ListObjectVersionsOutput>> {
Err(s3_error!(NotImplemented, "ListObjectVersions is not implemented yet"))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn get_bucket_versioning(
&self,
@@ -949,8 +986,12 @@ impl S3 for FS {
}
}
// warn!("input policy {}", &policy);
let cfg = try_!(BucketPolicy::unmarshal(policy.as_bytes()));
// warn!("parse policy {:?}", &cfg);
if let Err(err) = cfg.validate(&bucket) {
warn!("put_bucket_policy err input {:?}, {:?}", &policy, err);
return Err(s3_error!(InvalidPolicyDocument));
@@ -1037,7 +1078,7 @@ impl S3 for FS {
..
} = req.input;
warn!("lifecycle_configuration {:?}", &lifecycle_configuration);
// warn!("lifecycle_configuration {:?}", &lifecycle_configuration);
// TODO: objcetLock
@@ -1183,7 +1224,7 @@ impl S3 for FS {
}
};
warn!("object_lock_configuration {:?}", &object_lock_configuration);
// warn!("object_lock_configuration {:?}", &object_lock_configuration);
Ok(S3Response::new(GetObjectLockConfigurationOutput {
object_lock_configuration,
@@ -1395,6 +1436,167 @@ impl S3 for FS {
Ok(S3Response::new(PutBucketNotificationConfigurationOutput::default()))
}
async fn get_bucket_acl(&self, req: S3Request<GetBucketAclInput>) -> S3Result<S3Response<GetBucketAclOutput>> {
let GetBucketAclInput { bucket, .. } = req.input;
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = lock
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
if let Err(e) = store.get_bucket_info(&bucket, &BucketOptions::default()).await {
if DiskError::VolumeNotFound.is(&e) {
return Err(s3_error!(NoSuchBucket));
} else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e)));
}
}
let mut grants = Vec::new();
grants.push(Grant {
grantee: Some(Grantee {
type_: Type::from_static(Type::CANONICAL_USER),
display_name: None,
email_address: None,
id: None,
uri: None,
}),
permission: Some(Permission::from_static(Permission::FULL_CONTROL)),
});
Ok(S3Response::new(GetBucketAclOutput {
grants: Some(grants),
owner: Some(RUSTFS_OWNER.to_owned()),
..Default::default()
}))
}
async fn put_bucket_acl(&self, req: S3Request<PutBucketAclInput>) -> S3Result<S3Response<PutBucketAclOutput>> {
let PutBucketAclInput {
bucket,
acl,
access_control_policy,
..
} = req.input;
// TODO:checkRequestAuthType
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = lock
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
if let Err(e) = store.get_bucket_info(&bucket, &BucketOptions::default()).await {
if DiskError::VolumeNotFound.is(&e) {
return Err(s3_error!(NoSuchBucket));
} else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e)));
}
}
if let Some(canned_acl) = acl {
if canned_acl.as_str() != BucketCannedACL::PRIVATE {
return Err(s3_error!(NotImplemented));
}
} else {
let is_full_control = access_control_policy.is_some_and(|v| {
v.grants.is_some_and(|gs| {
//
!gs.is_empty()
&& gs.get(0).is_some_and(|g| {
g.to_owned()
.permission
.is_some_and(|p| p.as_str() == Permission::FULL_CONTROL)
})
})
});
if !is_full_control {
return Err(s3_error!(NotImplemented));
}
}
Ok(S3Response::new(PutBucketAclOutput::default()))
}
async fn get_object_acl(&self, req: S3Request<GetObjectAclInput>) -> S3Result<S3Response<GetObjectAclOutput>> {
let GetObjectAclInput { bucket, key, .. } = req.input;
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = lock
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
if let Err(e) = store.get_object_info(&bucket, &key, &ObjectOptions::default()).await {
return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e)));
}
let mut grants = Vec::new();
grants.push(Grant {
grantee: Some(Grantee {
type_: Type::from_static(Type::CANONICAL_USER),
display_name: None,
email_address: None,
id: None,
uri: None,
}),
permission: Some(Permission::from_static(Permission::FULL_CONTROL)),
});
Ok(S3Response::new(GetObjectAclOutput {
grants: Some(grants),
owner: Some(RUSTFS_OWNER.to_owned()),
..Default::default()
}))
}
async fn put_object_acl(&self, req: S3Request<PutObjectAclInput>) -> S3Result<S3Response<PutObjectAclOutput>> {
let PutObjectAclInput {
bucket,
key,
acl,
access_control_policy,
..
} = req.input;
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = lock
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
if let Err(e) = store.get_object_info(&bucket, &key, &ObjectOptions::default()).await {
return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e)));
}
if let Some(canned_acl) = acl {
if canned_acl.as_str() != BucketCannedACL::PRIVATE {
return Err(s3_error!(NotImplemented));
}
} else {
let is_full_control = access_control_policy.is_some_and(|v| {
v.grants.is_some_and(|gs| {
//
!gs.is_empty()
&& gs.get(0).is_some_and(|g| {
g.to_owned()
.permission
.is_some_and(|p| p.as_str() == Permission::FULL_CONTROL)
})
})
});
if !is_full_control {
return Err(s3_error!(NotImplemented));
}
}
Ok(S3Response::new(PutObjectAclOutput::default()))
}
}
#[allow(dead_code)]
+1
View File
@@ -1 +1,2 @@
pub mod acess;
pub mod ecfs;