diff --git a/Cargo.lock b/Cargo.lock index 507f9d4f7..926af7b0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2107,6 +2107,7 @@ version = "0.0.1" dependencies = [ "async-trait", "common", + "ecstore", "hyper", "matchit 0.8.5", "pin-project-lite", @@ -2115,6 +2116,7 @@ dependencies = [ "serde", "serde-xml-rs", "serde_urlencoded", + "time", "tracing", ] @@ -2254,7 +2256,7 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "s3s" version = "0.11.0-dev" -source = "git+https://github.com/Nugine/s3s.git?rev=c41ac1d30a0ae3ded6a18cc1b5bbb98f8b7f35c2#c41ac1d30a0ae3ded6a18cc1b5bbb98f8b7f35c2" +source = "git+https://github.com/Nugine/s3s.git?rev=b4ee0f7297d5ed23772fa8ba6e760c14f481990d#b4ee0f7297d5ed23772fa8ba6e760c14f481990d" dependencies = [ "arrayvec", "async-trait", @@ -2300,7 +2302,7 @@ dependencies = [ [[package]] name = "s3s-policy" version = "0.11.0-dev" -source = "git+https://github.com/Nugine/s3s.git?rev=c41ac1d30a0ae3ded6a18cc1b5bbb98f8b7f35c2#c41ac1d30a0ae3ded6a18cc1b5bbb98f8b7f35c2" +source = "git+https://github.com/Nugine/s3s.git?rev=b4ee0f7297d5ed23772fa8ba6e760c14f481990d#b4ee0f7297d5ed23772fa8ba6e760c14f481990d" dependencies = [ "indexmap 2.6.0", "serde", diff --git a/Cargo.toml b/Cargo.toml index a8e1f3dc0..47e76c891 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,10 +50,10 @@ prost-types = "0.13.3" protobuf = "3.7" protos = { path = "./common/protos" } rand = "0.8.5" -s3s = { git = "https://github.com/Nugine/s3s.git", rev = "c41ac1d30a0ae3ded6a18cc1b5bbb98f8b7f35c2", default-features = true, features = [ +s3s = { git = "https://github.com/Nugine/s3s.git", rev = "b4ee0f7297d5ed23772fa8ba6e760c14f481990d", default-features = true, features = [ "tower", ] } -s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "c41ac1d30a0ae3ded6a18cc1b5bbb98f8b7f35c2" } +s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "b4ee0f7297d5ed23772fa8ba6e760c14f481990d" } serde = { version = "1.0.214", features = ["derive"] } serde_json = "1.0.132" tempfile = "3.13.0" diff --git a/ecstore/src/bucket/metadata.rs b/ecstore/src/bucket/metadata.rs index 01283bfbe..ac0134d86 100644 --- a/ecstore/src/bucket/metadata.rs +++ b/ecstore/src/bucket/metadata.rs @@ -22,6 +22,7 @@ use crate::error::{Error, Result}; use crate::disk::BUCKET_META_PREFIX; use crate::store::ECStore; +use crate::utils::xml::deserialize; pub const BUCKET_METADATA_FILE: &str = ".metadata.bin"; pub const BUCKET_METADATA_FORMAT: u16 = 1; @@ -427,31 +428,3 @@ mod test { assert_eq!(bm.name, new.name); } } - -pub fn deserialize(input: &[u8]) -> xml::DeResult -where - T: for<'xml> xml::Deserialize<'xml>, -{ - let mut d = xml::Deserializer::new(input); - let ans = T::deserialize(&mut d)?; - d.expect_eof()?; - Ok(ans) -} - -pub fn serialize_content(val: &T) -> xml::SerResult { - let mut buf = Vec::with_capacity(256); - { - let mut ser = xml::Serializer::new(&mut buf); - val.serialize_content(&mut ser)?; - } - Ok(String::from_utf8(buf).unwrap()) -} - -pub fn serialize(val: &T) -> xml::SerResult> { - let mut buf = Vec::with_capacity(256); - { - let mut ser = xml::Serializer::new(&mut buf); - val.serialize(&mut ser)?; - } - Ok(buf) -} diff --git a/ecstore/src/bucket/metadata_sys.rs b/ecstore/src/bucket/metadata_sys.rs index 72888b1df..764a57b50 100644 --- a/ecstore/src/bucket/metadata_sys.rs +++ b/ecstore/src/bucket/metadata_sys.rs @@ -10,6 +10,7 @@ use crate::disk::error::DiskError; use crate::error::{Error, Result}; use crate::global::{is_dist_erasure, is_erasure, new_object_layer_fn, GLOBAL_Endpoints}; use crate::store::ECStore; +use crate::utils::xml::deserialize; use futures::future::join_all; use s3s::dto::{ BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration, @@ -19,7 +20,7 @@ use time::OffsetDateTime; use tokio::sync::RwLock; use tracing::{error, warn}; -use super::metadata::{deserialize, load_bucket_metadata, BucketMetadata}; +use super::metadata::{load_bucket_metadata, BucketMetadata}; use super::policy::bucket_policy::BucketPolicy; use super::quota::BucketQuota; use super::target::BucketTargets; diff --git a/ecstore/src/config/storageclass.rs b/ecstore/src/config/storageclass.rs index 411e532b7..6e8917250 100644 --- a/ecstore/src/config/storageclass.rs +++ b/ecstore/src/config/storageclass.rs @@ -53,31 +53,28 @@ pub static DEFAULT_INLINE_BLOCK: usize = 128 * 1024; lazy_static! { pub static ref DefaultKVS: KVS = { - let mut kvs = Vec::new(); - - kvs.push(KV { - key: CLASS_STANDARD.to_owned(), - value: "".to_owned(), - hidden_if_empty: false, - }); - - kvs.push(KV { - key: CLASS_RRS.to_owned(), - value: "EC:1".to_owned(), - hidden_if_empty: false, - }); - - kvs.push(KV { - key: OPTIMIZE.to_owned(), - value: "availability".to_owned(), - hidden_if_empty: false, - }); - - kvs.push(KV { - key: INLINE_BLOCK.to_owned(), - value: "".to_owned(), - hidden_if_empty: true, - }); + let kvs = vec![ + KV { + key: CLASS_STANDARD.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: CLASS_RRS.to_owned(), + value: "EC:1".to_owned(), + hidden_if_empty: false, + }, + KV { + key: OPTIMIZE.to_owned(), + value: "availability".to_owned(), + hidden_if_empty: false, + }, + KV { + key: INLINE_BLOCK.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + ]; KVS(kvs) }; diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index 2f53377e3..07551a343 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -17,7 +17,7 @@ mod sets; pub mod store; pub mod store_api; mod store_init; -mod utils; +pub mod utils; pub mod bucket; pub mod file_meta_inline; diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index fab2f8a9a..9f361d482 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -20,6 +20,7 @@ use crate::store_err::{ use crate::store_init::ec_drives_no_config; use crate::utils::crypto::base64_decode; use crate::utils::path::{base_dir_from_prefix, decode_dir_object, encode_dir_object, SLASH_SEPARATOR}; +use crate::utils::xml; use crate::{ bucket::metadata::BucketMetadata, disk::{error::DiskError, new_disk, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, @@ -885,12 +886,12 @@ impl StorageAPI for ECStore { } if opts.lock_enabled { - meta.object_lock_config_xml = metadata::serialize::(&enableObjcetLockConfig)?; - meta.versioning_config_xml = metadata::serialize::(&enableVersioningConfig)?; + meta.object_lock_config_xml = xml::serialize::(&enableObjcetLockConfig)?; + meta.versioning_config_xml = xml::serialize::(&enableVersioningConfig)?; } if opts.versioning_enabled { - meta.versioning_config_xml = metadata::serialize::(&enableVersioningConfig)?; + meta.versioning_config_xml = xml::serialize::(&enableVersioningConfig)?; } meta.save(self).await?; diff --git a/ecstore/src/utils/mod.rs b/ecstore/src/utils/mod.rs index 7d56bd2ce..daff92a6c 100644 --- a/ecstore/src/utils/mod.rs +++ b/ecstore/src/utils/mod.rs @@ -6,3 +6,4 @@ pub mod net; pub mod os; pub mod path; pub mod wildcard; +pub mod xml; diff --git a/ecstore/src/utils/xml.rs b/ecstore/src/utils/xml.rs new file mode 100644 index 000000000..b298d40bd --- /dev/null +++ b/ecstore/src/utils/xml.rs @@ -0,0 +1,29 @@ +use s3s::xml; + +pub fn deserialize(input: &[u8]) -> xml::DeResult +where + T: for<'xml> xml::Deserialize<'xml>, +{ + let mut d = xml::Deserializer::new(input); + let ans = T::deserialize(&mut d)?; + d.expect_eof()?; + Ok(ans) +} + +pub fn serialize_content(val: &T) -> xml::SerResult { + let mut buf = Vec::with_capacity(256); + { + let mut ser = xml::Serializer::new(&mut buf); + val.serialize_content(&mut ser)?; + } + Ok(String::from_utf8(buf).unwrap()) +} + +pub fn serialize(val: &T) -> xml::SerResult> { + let mut buf = Vec::with_capacity(256); + { + let mut ser = xml::Serializer::new(&mut buf); + val.serialize(&mut ser)?; + } + Ok(buf) +} diff --git a/router/Cargo.toml b/router/Cargo.toml index 422643924..27dd7e412 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -19,3 +19,5 @@ serde_urlencoded = "0.7.1" serde.workspace = true quick-xml = "0.37.0" serde-xml-rs = "0.6.0" +ecstore.workspace = true +time.workspace = true diff --git a/router/src/handlers.rs b/router/src/handlers.rs index 84345eb18..111cacf08 100644 --- a/router/src/handlers.rs +++ b/router/src/handlers.rs @@ -1,9 +1,14 @@ use crate::router::Operation; +use ecstore::utils::xml; use hyper::StatusCode; use matchit::Params; -use s3s::{s3_error, Body, S3Request, S3Response, S3Result}; -use serde::{Deserialize, Serialize}; +use s3s::{ + dto::{AssumeRoleOutput, Credentials, Timestamp}, + s3_error, Body, S3Request, S3Response, S3Result, +}; +use serde::Deserialize; use serde_urlencoded::from_bytes; +use time::{Duration, OffsetDateTime}; use tracing::warn; #[derive(Deserialize, Debug, Default)] @@ -18,31 +23,31 @@ pub struct AssumeRoleRequest { pub external_id: String, } -#[derive(Debug, Serialize, Default)] -#[serde(rename_all = "PascalCase", default)] -pub struct AssumeRoleResponse { - #[serde(rename = "AssumeRoleResult")] - pub result: AssumeRoleResult, -} +// #[derive(Debug, Serialize, Default)] +// #[serde(rename_all = "PascalCase", default)] +// pub struct AssumeRoleResponse { +// #[serde(rename = "AssumeRoleResult")] +// pub result: AssumeRoleResult, +// } -#[derive(Debug, Serialize, Default)] -#[serde(rename_all = "PascalCase", default)] -pub struct AssumeRoleResult { - pub credentials: Credentials, -} +// #[derive(Debug, Serialize, Default)] +// #[serde(rename_all = "PascalCase", default)] +// pub struct AssumeRoleResult { +// pub credentials: Credentials, +// } -#[derive(Debug, Serialize, Default)] -#[serde(rename_all = "PascalCase", default)] -pub struct Credentials { - #[serde(rename = "AccessKeyId")] - pub access_key: String, - #[serde(rename = "SecretAccessKey")] - pub secret_key: String, - pub status: String, - pub expiration: usize, - pub session_token: String, - pub parent_user: String, -} +// #[derive(Debug, Serialize, Default)] +// #[serde(rename_all = "PascalCase", default)] +// pub struct Credentials { +// #[serde(rename = "AccessKeyId")] +// pub access_key: String, +// #[serde(rename = "SecretAccessKey")] +// pub secret_key: String, +// pub status: String, +// pub expiration: usize, +// pub session_token: String, +// pub parent_user: String, +// } pub struct AssumeRoleHandle {} #[async_trait::async_trait] @@ -50,9 +55,7 @@ impl Operation for AssumeRoleHandle { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { warn!("handle AssumeRoleHandle"); - let Some(cred) = req.credentials else { return Err(s3_error!(InvalidRequest, "get body failed")) }; - - warn!("AssumeRole get cred {:?}", cred); + let Some(cred) = req.credentials else { return Err(s3_error!(InvalidRequest, "get cred failed")) }; let mut input = req.input; @@ -63,22 +66,21 @@ impl Operation for AssumeRoleHandle { warn!("AssumeRole get body {:?}", body); - let resp = AssumeRoleResponse { - result: AssumeRoleResult { - credentials: Credentials { - access_key: "test".to_owned(), - secret_key: "test".to_owned(), - status: "on".to_owned(), - expiration: 0, - session_token: "sdf".to_owned(), - parent_user: cred.access_key, - }, - }, - }; - // getAssumeRoleCredentials - let output = serde_xml_rs::to_string(&resp).unwrap(); + let exp = OffsetDateTime::now_utc().saturating_add(Duration::days(1)); - warn!("output {:?}", output); + // TODO: create tmp access_key + let resp = AssumeRoleOutput { + credentials: Some(Credentials { + access_key_id: cred.access_key, + expiration: Timestamp::from(exp), + secret_access_key: cred.secret_key.expose().to_string(), + session_token: "sdfsdf".to_owned(), + }), + ..Default::default() + }; + + // getAssumeRoleCredentials + let output = xml::serialize::(&resp).unwrap(); Ok(S3Response::new((StatusCode::OK, Body::from(output)))) @@ -86,6 +88,20 @@ impl Operation for AssumeRoleHandle { } } +pub struct AccountInfoHandler {} +#[async_trait::async_trait] +impl Operation for AccountInfoHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + warn!("handle AccountInfoHandler"); + + let Some(cred) = req.credentials else { return Err(s3_error!(InvalidRequest, "get cred failed")) }; + + warn!("AccountInfoHandler cread {:?}", &cred); + + return Err(s3_error!(NotImplemented)); + } +} + pub struct ServiceHandle {} #[async_trait::async_trait] impl Operation for ServiceHandle { diff --git a/router/src/lib.rs b/router/src/lib.rs index d111ad750..673bc1d4c 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -12,7 +12,11 @@ pub fn make_admin_route() -> Result { let mut r = S3Router::new(); r.insert(Method::POST, "/", AdminOperation(&handlers::AssumeRoleHandle {}))?; - + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/accountinfo").as_str(), + AdminOperation(&handlers::AccountInfoHandler {}), + )?; r.insert( Method::POST, format!("{}{}", ADMIN_PREFIX, "/v3/service").as_str(), diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index 4e8b00636..c17f67e67 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -5,7 +5,7 @@ use ecstore::{ DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, Reader, UpdateMetadataOpts, WalkDirOptions, }, - erasure::{ReadAt, Writer}, + erasure::Writer, peer::{LocalPeerS3Client, PeerS3Client}, store::{all_local_disk_path, find_local_disk}, store_api::{BucketOptions, DeleteBucketOptions, FileInfo, MakeBucketOptions}, diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 50da57f06..4b1f10902 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -1,7 +1,6 @@ use bytes::Bytes; use common::error::Result; use ecstore::bucket::error::BucketMetadataError; -use ecstore::bucket::metadata; use ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG; use ecstore::bucket::metadata::BUCKET_NOTIFICATION_CONFIG; use ecstore::bucket::metadata::BUCKET_POLICY_CONFIG; @@ -33,6 +32,7 @@ use ecstore::store_api::ObjectOptions; use ecstore::store_api::ObjectToDelete; use ecstore::store_api::PutObjReader; use ecstore::store_api::StorageAPI; +use ecstore::utils::xml; use ecstore::xhttp; use futures::pin_mut; use futures::{Stream, StreamExt}; @@ -868,7 +868,7 @@ impl S3 for FS { } } - let data = try_!(metadata::serialize(&tagging)); + let data = try_!(xml::serialize(&tagging)); try_!(metadata_sys::update(&bucket, BUCKET_TAGGING_CONFIG, data).await); @@ -1003,7 +1003,7 @@ impl S3 for FS { // check bucket object lock enable // check replication suspended - let data = try_!(metadata::serialize(&versioning_configuration)); + let data = try_!(xml::serialize(&versioning_configuration)); try_!(metadata_sys::update(&bucket, BUCKET_VERSIONING_CONFIG, data).await); @@ -1166,7 +1166,7 @@ impl S3 for FS { let Some(input_cfg) = lifecycle_configuration else { return Err(s3_error!(InvalidArgument)) }; - let data = try_!(metadata::serialize(&input_cfg)); + let data = try_!(xml::serialize(&input_cfg)); try_!(metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, data).await); Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default())) @@ -1262,7 +1262,7 @@ impl S3 for FS { // TODO: check kms - let data = try_!(metadata::serialize(&server_side_encryption_configuration)); + let data = try_!(xml::serialize(&server_side_encryption_configuration)); try_!(metadata_sys::update(&bucket, BUCKET_SSECONFIG, data).await); Ok(S3Response::new(PutBucketEncryptionOutput::default())) } @@ -1340,7 +1340,7 @@ impl S3 for FS { } } - let data = try_!(metadata::serialize(&input_cfg)); + let data = try_!(xml::serialize(&input_cfg)); try_!(metadata_sys::update(&bucket, OBJECT_LOCK_CONFIG, data).await); @@ -1405,7 +1405,7 @@ impl S3 for FS { } // TODO: check enable, versioning enable - let data = try_!(metadata::serialize(&replication_configuration)); + let data = try_!(xml::serialize(&replication_configuration)); try_!(metadata_sys::update(&bucket, BUCKET_REPLICATION_CONFIG, data).await); @@ -1510,7 +1510,7 @@ impl S3 for FS { } } - let data = try_!(metadata::serialize(¬ification_configuration)); + let data = try_!(xml::serialize(¬ification_configuration)); try_!(metadata_sys::update(&bucket, BUCKET_NOTIFICATION_CONFIG, data).await);