Merge branch 'main' of github.com:rustfs/s3-rustfs into feature/logger

# Conflicts:
#	Cargo.toml
This commit is contained in:
houseme
2025-02-27 19:03:32 +08:00
36 changed files with 629 additions and 416 deletions
+5 -1
View File
@@ -7,6 +7,9 @@ repository.workspace = true
rust-version.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[[bin]]
name = "rustfs"
path = "src/main.rs"
[lints]
workspace = true
@@ -69,7 +72,8 @@ crypto = { path = "../crypto" }
iam = { path = "../iam" }
jsonwebtoken = "9.3.0"
tower-http = { version = "0.6.2", features = ["cors"] }
include_dir = "0.7.4"
mime_guess = "2.0.5"
rust-embed = { version = "8.5.0", features = ["interpolate-folder-path"] }
[build-dependencies]
prost-build.workspace = true
+3 -138
View File
@@ -18,12 +18,11 @@ use ecstore::store::is_valid_object_prefix;
use ecstore::store_api::StorageAPI;
use ecstore::utils::crypto::base64_encode;
use ecstore::utils::path::path_join;
use ecstore::utils::xml;
use ecstore::GLOBAL_Endpoints;
use futures::{Stream, StreamExt};
use http::{HeaderMap, Uri};
use hyper::StatusCode;
use iam::auth::{get_claims_from_token_with_secret, get_new_credentials_with_metadata};
use iam::auth::get_claims_from_token_with_secret;
use iam::error::Error as IamError;
use iam::policy::Policy;
use iam::sys::SESSION_POLICY_NAME;
@@ -33,10 +32,7 @@ use madmin::utils::parse_duration;
use matchit::Params;
use s3s::header::CONTENT_TYPE;
use s3s::stream::{ByteStream, DynByteStream};
use s3s::{
dto::{AssumeRoleOutput, Credentials, Timestamp},
s3_error, Body, S3Error, S3Request, S3Response, S3Result,
};
use s3s::{s3_error, Body, S3Error, S3Request, S3Response, S3Result};
use s3s::{S3ErrorCode, StdError};
use serde::{Deserialize, Serialize};
use serde_json::Value;
@@ -47,7 +43,6 @@ use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Duration as std_Duration;
use time::{Duration, OffsetDateTime};
use tokio::sync::mpsc::{self};
use tokio::time::interval;
use tokio::{select, spawn};
@@ -57,32 +52,10 @@ use tracing::{error, info, warn};
pub mod group;
pub mod policy;
pub mod service_account;
pub mod sts;
pub mod trace;
pub mod user;
const ASSUME_ROLE_ACTION: &str = "AssumeRole";
const ASSUME_ROLE_VERSION: &str = "2011-06-15";
#[derive(Deserialize, Debug, Default)]
#[serde(rename_all = "PascalCase", default)]
pub struct AssumeRoleRequest {
pub action: String,
pub duration_seconds: usize,
pub version: String,
pub role_arn: String,
pub role_session_name: String,
pub policy: String,
pub external_id: String,
}
fn get_token_signing_key() -> Option<String> {
if let Some(s) = get_global_action_cred() {
Some(s.secret_key.clone())
} else {
None
}
}
// check_key_valid get auth.cred
pub async fn check_key_valid(security_token: Option<String>, ak: &str) -> S3Result<(auth::Credentials, bool)> {
let Some(mut cred) = get_global_action_cred() else {
@@ -216,114 +189,6 @@ pub fn populate_session_policy(claims: &mut HashMap<String, Value>, policy: &str
Ok(())
}
pub struct AssumeRoleHandle {}
#[async_trait::async_trait]
impl Operation for AssumeRoleHandle {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle AssumeRoleHandle");
let Some(user) = req.credentials else { return Err(s3_error!(InvalidRequest, "get cred failed")) };
let session_token = get_session_token(&req.headers);
if session_token.is_some() {
return Err(s3_error!(InvalidRequest, "AccessDenied1"));
}
let (cred, _owner) = check_key_valid(session_token, &user.access_key).await?;
// // TODO: 判断权限, 不允许sts访问
if cred.is_temp() || cred.is_service_account() {
return Err(s3_error!(InvalidRequest, "AccessDenied"));
}
let mut input = req.input;
let bytes = match input.store_all_unlimited().await {
Ok(b) => b,
Err(e) => {
warn!("get body failed, e: {:?}", e);
return Err(s3_error!(InvalidRequest, "get body failed"));
}
};
let body: AssumeRoleRequest = from_bytes(&bytes).map_err(|_e| s3_error!(InvalidRequest, "get body failed"))?;
if body.action.as_str() != ASSUME_ROLE_ACTION {
return Err(s3_error!(InvalidArgument, "not suport action"));
}
if body.version.as_str() != ASSUME_ROLE_VERSION {
return Err(s3_error!(InvalidArgument, "not suport version"));
}
let mut claims = cred.claims.unwrap_or_default();
populate_session_policy(&mut claims, &body.policy)?;
let exp = {
if body.duration_seconds > 0 {
body.duration_seconds
} else {
3600
}
};
claims.insert(
"exp".to_string(),
serde_json::Value::Number(serde_json::Number::from(OffsetDateTime::now_utc().unix_timestamp() + exp as i64)),
);
claims.insert("parent".to_string(), serde_json::Value::String(cred.access_key.clone()));
// warn!("AssumeRole get cred {:?}", &user);
// warn!("AssumeRole get body {:?}", &body);
let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) };
if let Err(_err) = iam_store.policy_db_get(&cred.access_key, &cred.groups).await {
return Err(s3_error!(InvalidArgument, "invalid policy arg"));
}
let Some(secret) = get_token_signing_key() else {
return Err(s3_error!(InvalidArgument, "global active sk not init"));
};
info!("AssumeRole get claims {:?}", &claims);
let mut new_cred = get_new_credentials_with_metadata(&claims, &secret)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("get new cred failed {}", e)))?;
new_cred.parent_user = cred.access_key.clone();
info!("AssumeRole get new_cred {:?}", &new_cred);
if let Err(_err) = iam_store.set_temp_user(&new_cred.access_key, &new_cred, None).await {
return Err(s3_error!(InternalError, "set_temp_user failed"));
}
// TODO: globalSiteReplicationSys
let resp = AssumeRoleOutput {
credentials: Some(Credentials {
access_key_id: new_cred.access_key,
expiration: Timestamp::from(
new_cred
.expiration
.unwrap_or(OffsetDateTime::now_utc().saturating_add(Duration::seconds(3600))),
),
secret_access_key: new_cred.secret_key,
session_token: new_cred.session_token,
}),
..Default::default()
};
// getAssumeRoleCredentials
let output = xml::serialize::<AssumeRoleOutput>(&resp).unwrap();
Ok(S3Response::new((StatusCode::OK, Body::from(output))))
}
}
#[derive(Debug, Serialize, Default)]
#[serde(rename_all = "PascalCase", default)]
pub struct AccountInfo {
+5 -4
View File
@@ -40,9 +40,10 @@ impl Operation for AddServiceAccount {
}
};
let mut create_req: AddServiceAccountReq =
let create_req: AddServiceAccountReq =
serde_json::from_slice(&body[..]).map_err(|e| s3_error!(InvalidRequest, "unmarshal body failed, e: {:?}", e))?;
create_req.expiration = create_req.expiration.and_then(|expire| expire.replace_millisecond(0).ok());
// create_req.expiration = create_req.expiration.and_then(|expire| expire.replace_millisecond(0).ok());
if has_space_be(&create_req.access_key) {
return Err(s3_error!(InvalidRequest, "access key has spaces"));
@@ -71,7 +72,7 @@ impl Operation for AddServiceAccount {
let req_groups = cred.groups.clone();
let mut req_is_derived_cred = false;
if cred.is_owner() || cred.is_service_account() {
if cred.is_service_account() || cred.is_temp() {
req_parent_user = cred.parent_user.clone();
req_is_derived_cred = true;
}
@@ -128,7 +129,7 @@ impl Operation for AddServiceAccount {
.await
.map_err(|e| {
debug!("create service account failed, e: {:?}", e);
s3_error!(InternalError, "create service account failed")
s3_error!(InternalError, "create service account failed, e: {:?}", e)
})?;
let resp = AddServiceAccountResp {
+139
View File
@@ -0,0 +1,139 @@
use crate::admin::{
handlers::{check_key_valid, get_session_token, populate_session_policy},
router::Operation,
};
use ecstore::utils::xml;
use http::StatusCode;
use iam::{auth::get_new_credentials_with_metadata, manager::get_token_signing_key};
use matchit::Params;
use s3s::{
dto::{AssumeRoleOutput, Credentials, Timestamp},
s3_error, Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result,
};
use serde::Deserialize;
use serde_urlencoded::from_bytes;
use time::{Duration, OffsetDateTime};
use tracing::{info, warn};
const ASSUME_ROLE_ACTION: &str = "AssumeRole";
const ASSUME_ROLE_VERSION: &str = "2011-06-15";
#[derive(Deserialize, Debug, Default)]
#[serde(rename_all = "PascalCase", default)]
pub struct AssumeRoleRequest {
pub action: String,
pub duration_seconds: usize,
pub version: String,
pub role_arn: String,
pub role_session_name: String,
pub policy: String,
pub external_id: String,
}
pub struct AssumeRoleHandle {}
#[async_trait::async_trait]
impl Operation for AssumeRoleHandle {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
warn!("handle AssumeRoleHandle");
let Some(user) = req.credentials else { return Err(s3_error!(InvalidRequest, "get cred failed")) };
let session_token = get_session_token(&req.headers);
if session_token.is_some() {
return Err(s3_error!(InvalidRequest, "AccessDenied1"));
}
let (cred, _owner) = check_key_valid(session_token, &user.access_key).await?;
// // TODO: 判断权限, 不允许sts访问
if cred.is_temp() || cred.is_service_account() {
return Err(s3_error!(InvalidRequest, "AccessDenied"));
}
let mut input = req.input;
let bytes = match input.store_all_unlimited().await {
Ok(b) => b,
Err(e) => {
warn!("get body failed, e: {:?}", e);
return Err(s3_error!(InvalidRequest, "get body failed"));
}
};
let body: AssumeRoleRequest = from_bytes(&bytes).map_err(|_e| s3_error!(InvalidRequest, "get body failed"))?;
if body.action.as_str() != ASSUME_ROLE_ACTION {
return Err(s3_error!(InvalidArgument, "not suport action"));
}
if body.version.as_str() != ASSUME_ROLE_VERSION {
return Err(s3_error!(InvalidArgument, "not suport version"));
}
let mut claims = cred.claims.unwrap_or_default();
populate_session_policy(&mut claims, &body.policy)?;
let exp = {
if body.duration_seconds > 0 {
body.duration_seconds
} else {
3600
}
};
claims.insert(
"exp".to_string(),
serde_json::Value::Number(serde_json::Number::from(OffsetDateTime::now_utc().unix_timestamp() + exp as i64)),
);
claims.insert("parent".to_string(), serde_json::Value::String(cred.access_key.clone()));
// warn!("AssumeRole get cred {:?}", &user);
// warn!("AssumeRole get body {:?}", &body);
let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) };
if let Err(_err) = iam_store.policy_db_get(&cred.access_key, &cred.groups).await {
return Err(s3_error!(InvalidArgument, "invalid policy arg"));
}
let Some(secret) = get_token_signing_key() else {
return Err(s3_error!(InvalidArgument, "global active sk not init"));
};
info!("AssumeRole get claims {:?}", &claims);
let mut new_cred = get_new_credentials_with_metadata(&claims, &secret)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("get new cred failed {}", e)))?;
new_cred.parent_user = cred.access_key.clone();
info!("AssumeRole get new_cred {:?}", &new_cred);
if let Err(_err) = iam_store.set_temp_user(&new_cred.access_key, &new_cred, None).await {
return Err(s3_error!(InternalError, "set_temp_user failed"));
}
// TODO: globalSiteReplicationSys
let resp = AssumeRoleOutput {
credentials: Some(Credentials {
access_key_id: new_cred.access_key,
expiration: Timestamp::from(
new_cred
.expiration
.unwrap_or(OffsetDateTime::now_utc().saturating_add(Duration::seconds(3600))),
),
secret_access_key: new_cred.secret_key,
session_token: new_cred.session_token,
}),
..Default::default()
};
// getAssumeRoleCredentials
let output = xml::serialize::<AssumeRoleOutput>(&resp).unwrap();
Ok(S3Response::new((StatusCode::OK, Body::from(output))))
}
}
+2 -2
View File
@@ -7,7 +7,7 @@ use common::error::Result;
use handlers::{
group, policy,
service_account::{AddServiceAccount, DeleteServiceAccount, InfoServiceAccount, ListServiceAccount, UpdateServiceAccount},
user,
sts, user,
};
use hyper::Method;
use router::{AdminOperation, S3Router};
@@ -19,7 +19,7 @@ pub fn make_admin_route() -> Result<impl S3Route> {
let mut r: S3Router<AdminOperation> = S3Router::new();
// 1
r.insert(Method::POST, "/", AdminOperation(&handlers::AssumeRoleHandle {}))?;
r.insert(Method::POST, "/", AdminOperation(&sts::AssumeRoleHandle {}))?;
regist_user_route(&mut r)?;
+2 -6
View File
@@ -1,4 +1,3 @@
use log::warn;
use s3s::auth::S3Auth;
use s3s::auth::SecretKey;
use s3s::auth::SimpleAuth;
@@ -20,22 +19,19 @@ impl IAMAuth {
impl S3Auth for IAMAuth {
async fn get_secret_key(&self, access_key: &str) -> S3Result<SecretKey> {
if access_key.is_empty() {
return Err(s3_error!(NotSignedUp, "Your account is not signed up"));
return Err(s3_error!(UnauthorizedAccess, "Your account is not signed up"));
}
if let Ok(key) = self.simple_auth.get_secret_key(access_key).await {
return Ok(key);
}
warn!("Failed to get secret key from simple auth");
if let Ok(iam_store) = iam::get() {
if let Some(id) = iam_store.get_user(access_key).await {
warn!("get cred {:?}", id.credentials);
return Ok(SecretKey::from(id.credentials.secret_key.clone()));
}
}
Err(s3_error!(NotSignedUp, "Your account is not signed up2"))
Err(s3_error!(UnauthorizedAccess, "Your account is not signed up2"))
}
}
+4 -1
View File
@@ -35,13 +35,16 @@ const LONG_VERSION: &str = concat!(
#[command(version = SHORT_VERSION, long_version = LONG_VERSION)]
pub struct Opt {
/// DIR points to a directory on a filesystem.
#[arg(required = true)]
#[arg(required = true, env = "RUSTFS_VOLUMES")]
pub volumes: Vec<String>,
/// bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname
#[arg(long, default_value_t = format!("0.0.0.0:{}", DEFAULT_PORT), env = "RUSTFS_ADDRESS")]
pub address: String,
#[arg(long, env = "RUSTFS_SERVER_DOMAINS")]
pub server_domains: Vec<String>,
/// Access key used for authentication.
#[arg(long, default_value_t = DEFAULT_ACCESS_KEY.to_string(), env = "RUSTFS_ACCESS_KEY")]
pub access_key: String,
+94 -8
View File
@@ -6,16 +6,32 @@ use axum::{
Router,
};
use include_dir::{include_dir, Dir};
use mime_guess::from_path;
use rust_embed::RustEmbed;
use serde::Serialize;
static STATIC_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/static");
#[derive(RustEmbed)]
#[folder = "$CARGO_MANIFEST_DIR/static"]
struct StaticFiles;
async fn static_handler(uri: axum::http::Uri) -> impl IntoResponse {
let path = uri.path().trim_start_matches('/');
if let Some(file) = STATIC_DIR.get_file(path) {
let mut path = uri.path().trim_start_matches('/');
if path.is_empty() {
path = "index.html"
}
if let Some(file) = StaticFiles::get(path) {
let mime_type = from_path(path).first_or_octet_stream();
Response::builder()
.status(StatusCode::OK)
.body(Body::from(file.contents()))
.header("Content-Type", mime_type.to_string())
.body(Body::from(file.data))
.unwrap()
} else if let Some(file) = StaticFiles::get("index.html") {
let mime_type = from_path("index.html").first_or_octet_stream();
Response::builder()
.status(StatusCode::OK)
.header("Content-Type", mime_type.to_string())
.body(Body::from(file.data))
.unwrap()
} else {
Response::builder()
@@ -25,13 +41,83 @@ async fn static_handler(uri: axum::http::Uri) -> impl IntoResponse {
}
}
pub async fn start_static_file_server(addrs: &str) {
#[derive(Debug, Serialize)]
struct Config {
api: Api,
s3: S3,
release: Release,
license: License,
}
impl Config {
fn new(url: &str, version: &str, date: &str) -> Self {
Config {
api: Api {
base_url: format!("{}/rustfs/admin/v3", url),
},
s3: S3 {
endpoint: url.to_owned(),
region: "cn-east-1".to_owned(),
},
release: Release {
version: version.to_string(),
date: date.to_string(),
},
license: License {
name: "Apache-2.0".to_string(),
url: "https://www.apache.org/licenses/LICENSE-2.0".to_string(),
},
}
}
fn to_json(&self) -> String {
serde_json::to_string(self).unwrap_or_default()
}
}
#[derive(Debug, Serialize)]
struct Api {
#[serde(rename = "baseURL")]
base_url: String,
}
#[derive(Debug, Serialize)]
struct S3 {
endpoint: String,
region: String,
}
#[derive(Debug, Serialize)]
struct Release {
version: String,
date: String,
}
#[derive(Debug, Serialize)]
struct License {
name: String,
url: String,
}
async fn config_handler(axum::extract::Extension(fs_addr): axum::extract::Extension<String>) -> impl IntoResponse {
let cfg = Config::new(&fs_addr, "v0.0.1", "2025-01-01").to_json();
Response::builder()
.header("content-type", "application/json")
.status(StatusCode::OK)
.body(Body::from(cfg))
.unwrap()
}
pub async fn start_static_file_server(addrs: &str, fs_addr: &str) {
// 创建路由
let app = Router::new().route("/*file", get(static_handler));
let app = Router::new()
.route("/config.json", get(config_handler).layer(axum::extract::Extension(fs_addr.to_owned())))
.nest_service("/", get(static_handler));
let listener = tokio::net::TcpListener::bind(addrs).await.unwrap();
println!("console listening on: {}", listener.local_addr().unwrap());
println!("console running on: http://{} with s3 api {}", listener.local_addr().unwrap(), fs_addr);
axum::serve(listener, app).await.unwrap();
}
+1 -4
View File
@@ -67,10 +67,7 @@ fn match_for_io_error(err_status: &Status) -> Option<&std::io::Error> {
}
}
err = match err.source() {
Some(err) => err,
None => return None,
};
err = err.source()?;
}
}
+13 -2
View File
@@ -31,7 +31,7 @@ use hyper_util::{
};
use iam::init_iam_sys;
use protos::proto_gen::node_service::node_service_server::NodeServiceServer;
use s3s::service::S3ServiceBuilder;
use s3s::{host::MultiDomain, service::S3ServiceBuilder};
use service::hybrid;
use std::{io::IsTerminal, net::SocketAddr};
use tokio::net::TcpListener;
@@ -139,6 +139,11 @@ async fn run(opt: config::Opt) -> Result<()> {
b.set_route(admin::make_admin_route()?);
if !opt.server_domains.is_empty() {
info!("virtual-hosted-style requests are enabled use domain_name {:?}", &opt.server_domains);
b.set_host(MultiDomain::new(&opt.server_domains)?);
}
// // Enable parsing virtual-hosted-style requests
// if let Some(dm) = opt.domain_name {
// info!("virtual-hosted-style requests are enabled use domain_name {}", &dm);
@@ -236,7 +241,13 @@ async fn run(opt: config::Opt) -> Result<()> {
if opt.console_enable {
info!("console is enabled");
tokio::spawn(async move {
console::start_static_file_server(&opt.console_address).await;
let ep = if !opt.server_domains.is_empty() {
format!("http://{}", opt.server_domains[0].clone())
} else {
format!("http://127.0.0.1:{}", server_port)
};
console::start_static_file_server(&opt.console_address, &ep).await;
});
}
+1 -1
View File
@@ -52,7 +52,7 @@ impl S3Access for FS {
);
if cx.credentials().is_none() {
return Err(s3_error!(AccessDenied, "Signature is required"));
return Err(s3_error!(UnauthorizedAccess, "Signature is required"));
};
// TODO: FIXME: check auth
+14 -4
View File
@@ -47,6 +47,7 @@ use s3s::S3;
use s3s::{S3Request, S3Response};
use std::fmt::Debug;
use std::str::FromStr;
use tracing::debug;
use tracing::error;
use tracing::info;
use transform_stream::AsyncTryStream;
@@ -549,6 +550,7 @@ impl S3 for FS {
.map(|v| Bucket {
creation_date: v.created.map(Timestamp::from),
name: Some(v.name.clone()),
..Default::default()
})
.collect();
@@ -751,6 +753,7 @@ impl S3 for FS {
content_length,
tagging,
metadata,
version_id,
..
} = input;
@@ -784,10 +787,12 @@ impl S3 for FS {
metadata.insert(xhttp::AMZ_OBJECT_TAGGING.to_owned(), tags);
}
let opts: ObjectOptions = put_opts(&bucket, &key, None, &req.headers, Some(metadata))
let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(metadata))
.await
.map_err(to_s3_error)?;
debug!("put_object opts {:?}", &opts);
let obj_info = store
.put_object(&bucket, &key, &mut reader, &opts)
.await
@@ -810,7 +815,11 @@ impl S3 for FS {
req: S3Request<CreateMultipartUploadInput>,
) -> S3Result<S3Response<CreateMultipartUploadOutput>> {
let CreateMultipartUploadInput {
bucket, key, tagging, ..
bucket,
key,
tagging,
version_id,
..
} = req.input;
// mc cp step 3
@@ -827,7 +836,7 @@ impl S3 for FS {
metadata.insert(xhttp::AMZ_OBJECT_TAGGING.to_owned(), tags);
}
let opts: ObjectOptions = put_opts(&bucket, &key, None, &req.headers, Some(metadata))
let opts: ObjectOptions = put_opts(&bucket, &key, version_id, &req.headers, Some(metadata))
.await
.map_err(to_s3_error)?;
@@ -952,7 +961,7 @@ impl S3 for FS {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
store
let oi = store
.complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, opts)
.await
.map_err(to_s3_error)?;
@@ -960,6 +969,7 @@ impl S3 for FS {
let output = CompleteMultipartUploadOutput {
bucket: Some(bucket),
key: Some(key),
e_tag: oi.etag,
..Default::default()
};
Ok(S3Response::new(output))
-1
View File
@@ -1 +0,0 @@
static index
+1
View File
@@ -0,0 +1 @@
console static path, do not delete