feat: improve legacy metadata and admin compatibility (#2202)

This commit is contained in:
weisd
2026-03-18 21:05:09 +08:00
committed by GitHub
parent 84077adf17
commit b9b7d86ae4
133 changed files with 11707 additions and 1945 deletions
+2
View File
@@ -49,6 +49,7 @@ rustfs-appauth = { workspace = true }
rustfs-audit = { workspace = true }
rustfs-common = { workspace = true }
rustfs-config = { workspace = true, features = ["constants", "notify"] }
rustfs-crypto = { workspace = true }
rustfs-credentials = { workspace = true }
rustfs-ecstore = { workspace = true }
rustfs-filemeta.workspace = true
@@ -168,6 +169,7 @@ aws-sdk-s3 = { workspace = true }
aws-config = { workspace = true }
anyhow = { workspace = true }
tokio = { workspace = true, features = ["test-util"] }
temp-env = { workspace = true }
[build-dependencies]
http.workspace = true
+41 -2
View File
@@ -142,7 +142,7 @@ impl Config {
Config {
port,
api: Api {
base_url: format!("{http_prefix}{local_ip}:{port}/{RUSTFS_ADMIN_PREFIX}"),
base_url: build_console_api_base_url(&format!("{http_prefix}{local_ip}:{port}")),
},
s3: S3 {
endpoint: format!("{http_prefix}{local_ip}:{port}"),
@@ -192,6 +192,10 @@ impl Config {
}
}
fn build_console_api_base_url(base_url: &str) -> String {
format!("{base_url}{RUSTFS_ADMIN_PREFIX}")
}
#[derive(Debug, Serialize, Clone)]
struct Api {
#[serde(rename = "baseURL")]
@@ -350,7 +354,7 @@ async fn config_handler(uri: Uri, headers: HeaderMap) -> impl IntoResponse {
};
let url = format!("{}://{}:{}", scheme, host_for_url, cfg.port);
cfg.api.base_url = format!("{url}{RUSTFS_ADMIN_PREFIX}");
cfg.api.base_url = build_console_api_base_url(&url);
cfg.s3.endpoint = url;
Response::builder()
@@ -646,3 +650,38 @@ pub(crate) fn make_console_server() -> Router {
// Build console router with enhanced middleware stack using tower-http features
setup_console_middleware_stack(cors_layer, rate_limit_enable, rate_limit_rpm, auth_timeout)
}
#[cfg(test)]
mod tests {
use super::*;
use std::net::{IpAddr, Ipv4Addr};
#[test]
fn console_api_base_url_keeps_rustfs_admin_prefix() {
let cfg = Config::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001, "test", "2026-03-16T00:00:00Z");
assert!(
cfg.api.base_url.ends_with("/rustfs/admin/v3"),
"console baseURL must keep using the RustFS admin prefix"
);
assert!(
!cfg.api.base_url.ends_with("/minio/admin/v3"),
"console baseURL must not switch to the MinIO admin prefix by default"
);
}
#[test]
fn console_api_base_url_builder_preserves_existing_console_contract() {
assert_eq!(
build_console_api_base_url("http://127.0.0.1:9001"),
"http://127.0.0.1:9001/rustfs/admin/v3"
);
}
#[test]
fn external_admin_paths_are_not_console_paths() {
assert!(is_console_path("/rustfs/console/"));
assert!(!is_console_path("/minio/admin/v3/info"));
assert!(!is_console_path("/rustfs/admin/v3/info"));
}
}
-1
View File
@@ -15,7 +15,6 @@
#[cfg(test)]
mod tests {
use crate::config::Opt;
use clap::Parser;
use serial_test::serial;
#[tokio::test]
+577 -5
View File
@@ -16,7 +16,7 @@ use crate::{
admin::{
auth::validate_admin_request,
router::{AdminOperation, Operation, S3Router},
utils::has_space_be,
utils::{encode_compatible_admin_payload, has_space_be, read_compatible_admin_body},
},
auth::{check_key_valid, get_session_token},
server::{ADMIN_PREFIX, RemoteAddr},
@@ -28,6 +28,7 @@ use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
use rustfs_credentials::get_global_action_cred;
use rustfs_iam::error::is_err_no_such_user;
use rustfs_iam::store::MappedPolicy;
use rustfs_madmin::{GroupPolicyEntities, PolicyEntities, PolicyEntitiesResult, UserPolicyEntities};
use rustfs_policy::policy::{
Policy,
action::{Action, AdminAction},
@@ -37,10 +38,12 @@ use s3s::{
header::{CONTENT_LENGTH, CONTENT_TYPE},
s3_error,
};
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use serde_urlencoded::from_bytes;
use std::collections::HashMap;
use time::OffsetDateTime;
use tracing::warn;
use url::form_urlencoded;
pub fn register_iam_policy_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
@@ -72,6 +75,26 @@ pub fn register_iam_policy_route(r: &mut S3Router<AdminOperation>) -> std::io::R
format!("{}{}", ADMIN_PREFIX, "/v3/set-user-or-group-policy").as_str(),
AdminOperation(&SetPolicyForUserOrGroup {}),
)?;
r.insert(
Method::PUT,
format!("{}{}", ADMIN_PREFIX, "/v3/set-policy").as_str(),
AdminOperation(&SetPolicyForUserOrGroup {}),
)?;
r.insert(
Method::POST,
format!("{}{}", ADMIN_PREFIX, "/v3/idp/builtin/policy/attach").as_str(),
AdminOperation(&AttachPolicyBuiltin {}),
)?;
r.insert(
Method::POST,
format!("{}{}", ADMIN_PREFIX, "/v3/idp/builtin/policy/detach").as_str(),
AdminOperation(&DetachPolicyBuiltin {}),
)?;
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/v3/idp/builtin/policy-entities").as_str(),
AdminOperation(&ListPolicyEntitiesBuiltin {}),
)?;
Ok(())
}
@@ -323,11 +346,11 @@ impl Operation for RemoveCannedPolicy {
#[derive(Debug, Deserialize, Default)]
pub struct SetPolicyForUserOrGroupQuery {
#[serde(rename = "policyName")]
#[serde(rename = "policyName", alias = "policy")]
pub policy_name: String,
#[serde(rename = "userOrGroup")]
#[serde(rename = "userOrGroup", alias = "user-or-group")]
pub user_or_group: String,
#[serde(rename = "isGroup")]
#[serde(rename = "isGroup", alias = "is-group")]
pub is_group: bool,
}
@@ -419,3 +442,552 @@ impl Operation for SetPolicyForUserOrGroup {
Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header))
}
}
#[derive(Debug, Deserialize, Default)]
struct PolicyAssociationReq {
#[serde(default)]
policies: Vec<String>,
#[serde(default)]
user: String,
#[serde(default)]
group: String,
}
#[derive(Debug, Serialize)]
struct PolicyAssociationResp {
#[serde(rename = "policiesAttached", skip_serializing_if = "Vec::is_empty")]
policies_attached: Vec<String>,
#[serde(rename = "policiesDetached", skip_serializing_if = "Vec::is_empty")]
policies_detached: Vec<String>,
#[serde(rename = "updatedAt", with = "time::serde::rfc3339")]
updated_at: OffsetDateTime,
}
#[derive(Debug, Deserialize, Default)]
struct PolicyEntitiesQuery {
users: Vec<String>,
groups: Vec<String>,
policies: Vec<String>,
}
fn split_policy_names(policy_names: &str) -> Vec<String> {
policy_names
.split(',')
.map(str::trim)
.filter(|name| !name.is_empty())
.map(ToOwned::to_owned)
.collect()
}
fn attach_policy_names(existing: &[String], requested: &[String]) -> (Vec<String>, Vec<String>) {
let mut updated = existing.to_vec();
let mut attached = Vec::new();
for policy in requested {
if updated.iter().any(|current| current == policy) {
continue;
}
updated.push(policy.clone());
attached.push(policy.clone());
}
(updated, attached)
}
fn detach_policy_names(existing: &[String], requested: &[String]) -> (Vec<String>, Vec<String>) {
let mut detached = Vec::new();
let updated = existing
.iter()
.filter(|policy| {
let should_detach = requested.iter().any(|requested_policy| requested_policy == *policy);
if should_detach {
detached.push((*policy).clone());
}
!should_detach
})
.cloned()
.collect();
(updated, detached)
}
fn validate_policy_association_req(req: &PolicyAssociationReq) -> S3Result<()> {
if req.policies.is_empty() {
return Err(s3_error!(InvalidArgument, "no policy names were given"));
}
if req.policies.iter().any(|policy| policy.is_empty()) {
return Err(s3_error!(InvalidArgument, "an empty policy name was given"));
}
let has_user = !req.user.is_empty();
let has_group = !req.group.is_empty();
match (has_user, has_group) {
(false, false) => Err(s3_error!(InvalidArgument, "no user or group association was given")),
(true, true) => Err(s3_error!(InvalidArgument, "either a group or a user association must be given, not both")),
_ => Ok(()),
}
}
fn parse_policy_entities_query(query: Option<&str>) -> PolicyEntitiesQuery {
let mut parsed = PolicyEntitiesQuery::default();
let Some(query) = query else {
return parsed;
};
for (key, value) in form_urlencoded::parse(query.as_bytes()) {
match key.as_ref() {
"user" => parsed.users.push(value.into_owned()),
"group" => parsed.groups.push(value.into_owned()),
"policy" => parsed.policies.push(value.into_owned()),
_ => {}
}
}
parsed
}
fn split_policy_list(policy_names: Option<&String>) -> Vec<String> {
policy_names.map_or_else(Vec::new, |names| split_policy_names(names))
}
fn direct_user_policy_names(user_info: &rustfs_madmin::UserInfo) -> Vec<String> {
split_policy_list(user_info.policy_name.as_ref())
}
fn sorted_group_policy_entities(mut entities: Vec<GroupPolicyEntities>) -> Vec<GroupPolicyEntities> {
entities.sort_by(|left, right| left.group.cmp(&right.group));
entities
}
fn build_policy_mappings(
user_mappings: &[UserPolicyEntities],
group_mappings: &[GroupPolicyEntities],
requested_policies: &[String],
) -> Vec<PolicyEntities> {
let mut policy_map: HashMap<String, PolicyEntities> = HashMap::new();
for user_mapping in user_mappings {
for policy in &user_mapping.policies {
let entry = policy_map.entry(policy.clone()).or_insert_with(|| PolicyEntities {
policy: policy.clone(),
..Default::default()
});
entry.users.push(user_mapping.user.clone());
}
}
for group_mapping in group_mappings {
for policy in &group_mapping.policies {
let entry = policy_map.entry(policy.clone()).or_insert_with(|| PolicyEntities {
policy: policy.clone(),
..Default::default()
});
entry.groups.push(group_mapping.group.clone());
}
}
let mut results: Vec<PolicyEntities> = policy_map
.into_iter()
.filter_map(|(_, mut mapping)| {
if !requested_policies.is_empty() && !requested_policies.iter().any(|policy| policy == &mapping.policy) {
return None;
}
mapping.users.sort();
mapping.users.dedup();
mapping.groups.sort();
mapping.groups.dedup();
Some(mapping)
})
.collect();
results.sort_by(|left, right| left.policy.cmp(&right.policy));
results
}
async fn collect_group_policy_mappings(
iam_store: &std::sync::Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>>,
requested_groups: &[String],
) -> S3Result<HashMap<String, GroupPolicyEntities>> {
let mut mappings = HashMap::new();
let groups = if requested_groups.is_empty() {
iam_store
.list_groups_load()
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?
} else {
requested_groups.to_vec()
};
for group in groups {
let group_desc = iam_store.get_group_description(&group).await.map_err(|e| {
warn!("get group description failed, e: {:?}", e);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?;
let policies = split_policy_names(&group_desc.policy);
if policies.is_empty() {
continue;
}
mappings.insert(group.clone(), GroupPolicyEntities { group, policies });
}
Ok(mappings)
}
async fn handle_builtin_policy_entities(req: S3Request<Body>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
};
let (cred, owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
validate_admin_request(
&req.headers,
&cred,
owner,
false,
vec![
Action::AdminAction(AdminAction::ListGroupsAdminAction),
Action::AdminAction(AdminAction::ListUsersAdminAction),
Action::AdminAction(AdminAction::ListUserPoliciesAdminAction),
],
req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
)
.await?;
let query = parse_policy_entities_query(req.uri.query());
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
let all_group_policy_mappings = collect_group_policy_mappings(&iam_store, &[]).await?;
let users = iam_store.list_users().await.map_err(|e| {
warn!("list users failed, e: {:?}", e);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?;
let all_user_policy_mappings = users
.iter()
.filter_map(|(user, user_info)| {
let policies = split_policy_list(user_info.policy_name.as_ref());
if policies.is_empty() {
return None;
}
Some(UserPolicyEntities {
user: user.clone(),
policies,
member_of_mappings: Vec::new(),
})
})
.collect::<Vec<_>>();
let user_mappings = if query.users.is_empty() {
Vec::new()
} else {
let mut mappings = Vec::new();
for user in &query.users {
let Some(user_info) = users.get(user) else {
continue;
};
let mut member_of_mappings = user_info
.member_of
.as_ref()
.map(|groups| {
groups
.iter()
.filter_map(|group| all_group_policy_mappings.get(group).cloned())
.collect::<Vec<_>>()
})
.unwrap_or_default();
member_of_mappings = sorted_group_policy_entities(member_of_mappings);
let policies = split_policy_list(user_info.policy_name.as_ref());
if policies.is_empty() && member_of_mappings.is_empty() {
continue;
}
mappings.push(UserPolicyEntities {
user: user.clone(),
policies,
member_of_mappings,
});
}
mappings.sort_by(|left, right| left.user.cmp(&right.user));
mappings
};
let mut group_mappings = if query.groups.is_empty() {
Vec::new()
} else {
query
.groups
.iter()
.filter_map(|group| all_group_policy_mappings.get(group).cloned())
.collect::<Vec<_>>()
};
group_mappings = sorted_group_policy_entities(group_mappings);
let policy_mappings = if query.users.is_empty() && query.groups.is_empty() && query.policies.is_empty() {
build_policy_mappings(
&all_user_policy_mappings,
&all_group_policy_mappings.values().cloned().collect::<Vec<_>>(),
&[],
)
} else if !query.policies.is_empty() {
build_policy_mappings(
&all_user_policy_mappings,
&all_group_policy_mappings.values().cloned().collect::<Vec<_>>(),
&query.policies,
)
} else {
Vec::new()
};
let resp = PolicyEntitiesResult {
timestamp: OffsetDateTime::now_utc(),
user_mappings,
group_mappings,
policy_mappings,
};
let req_path = req.uri.path().to_string();
let body =
serde_json::to_vec(&resp).map_err(|e| s3_error!(InternalError, "marshal policy entities body failed, e: {:?}", e))?;
let (body, content_type) = encode_compatible_admin_payload(&req_path, &cred.secret_key, body)?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, content_type.parse().unwrap());
header.insert(CONTENT_LENGTH, body.len().to_string().parse().unwrap());
Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), header))
}
async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(input_cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "get cred failed"));
};
let (cred, owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
validate_admin_request(
&req.headers,
&cred,
owner,
false,
vec![Action::AdminAction(AdminAction::AttachPolicyAdminAction)],
req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
)
.await?;
let req_path = req.uri.path().to_string();
let body = read_compatible_admin_body(req.input, MAX_ADMIN_REQUEST_BODY_SIZE, &req_path, &cred.secret_key).await?;
let assoc_req: PolicyAssociationReq = serde_json::from_slice(&body)
.map_err(|e| s3_error!(InvalidRequest, "unmarshal policy association body failed, e: {:?}", e))?;
validate_policy_association_req(&assoc_req)?;
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
let (target_name, is_group, existing_policies) = if !assoc_req.user.is_empty() {
match iam_store.is_temp_user(&assoc_req.user).await {
Ok((true, _)) => return Err(s3_error!(InvalidArgument, "temp user can't set policy")),
Ok((false, _)) => {}
Err(err) => {
if !is_err_no_such_user(&err) {
warn!("is temp user failed, e: {:?}", err);
return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string()));
}
}
}
let Some(sys_cred) = get_global_action_cred() else {
return Err(s3_error!(InternalError, "get global action cred failed"));
};
if assoc_req.user == sys_cred.access_key {
return Err(s3_error!(InvalidArgument, "can't set policy for system user"));
}
if iam_store.get_user(&assoc_req.user).await.is_none() {
return Err(s3_error!(InvalidArgument, "user not exist"));
}
let user_info = iam_store.get_user_info(&assoc_req.user).await.map_err(|e| {
warn!("get user info failed, e: {:?}", e);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?;
(assoc_req.user, false, direct_user_policy_names(&user_info))
} else {
let group_desc = iam_store.get_group_description(&assoc_req.group).await.map_err(|e| {
warn!("get group description failed, e: {:?}", e);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?;
(assoc_req.group, true, split_policy_names(&group_desc.policy))
};
let (updated_policies, changed_policies) = if is_attach {
attach_policy_names(&existing_policies, &assoc_req.policies)
} else {
detach_policy_names(&existing_policies, &assoc_req.policies)
};
let updated_at = iam_store
.policy_db_set(&target_name, rustfs_iam::store::UserType::Reg, is_group, &updated_policies.join(","))
.await
.map_err(|e| {
warn!("policy db set failed, e: {:?}", e);
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
})?;
let policies_attached = if is_attach { changed_policies.clone() } else { Vec::new() };
let policies_detached = if is_attach { Vec::new() } else { changed_policies };
let resp = PolicyAssociationResp {
policies_attached,
policies_detached,
updated_at,
};
let body = serde_json::to_vec(&resp).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?;
let (body, content_type) = encode_compatible_admin_payload(&req_path, &cred.secret_key, body)?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, content_type.parse().unwrap());
header.insert(CONTENT_LENGTH, body.len().to_string().parse().unwrap());
Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), header))
}
pub struct AttachPolicyBuiltin {}
#[async_trait::async_trait]
impl Operation for AttachPolicyBuiltin {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
handle_builtin_policy_association(req, true).await
}
}
pub struct DetachPolicyBuiltin {}
#[async_trait::async_trait]
impl Operation for DetachPolicyBuiltin {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
handle_builtin_policy_association(req, false).await
}
}
pub struct ListPolicyEntitiesBuiltin {}
#[async_trait::async_trait]
impl Operation for ListPolicyEntitiesBuiltin {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
handle_builtin_policy_entities(req).await
}
}
#[cfg(test)]
mod tests {
use super::{
GroupPolicyEntities, PolicyAssociationReq, SetPolicyForUserOrGroupQuery, UserPolicyEntities, attach_policy_names,
build_policy_mappings, detach_policy_names, direct_user_policy_names, parse_policy_entities_query,
validate_policy_association_req,
};
use rustfs_madmin::UserInfo;
#[test]
fn set_policy_query_supports_external_parameter_names() {
let query: SetPolicyForUserOrGroupQuery =
serde_urlencoded::from_str("policy=readwrite&user-or-group=test-user&is-group=true").expect("query should parse");
assert_eq!(query.policy_name, "readwrite");
assert_eq!(query.user_or_group, "test-user");
assert!(query.is_group);
}
#[test]
fn set_policy_query_supports_rustfs_parameter_names() {
let query: SetPolicyForUserOrGroupQuery =
serde_urlencoded::from_str("policyName=readwrite&userOrGroup=test-user&isGroup=false").expect("query should parse");
assert_eq!(query.policy_name, "readwrite");
assert_eq!(query.user_or_group, "test-user");
assert!(!query.is_group);
}
#[test]
fn policy_association_req_requires_exactly_one_target() {
let err = validate_policy_association_req(&PolicyAssociationReq {
policies: vec!["readonly".to_string()],
user: "user-a".to_string(),
group: "group-a".to_string(),
})
.expect_err("request should be invalid");
assert_eq!(*err.code(), s3s::S3ErrorCode::InvalidArgument);
}
#[test]
fn attach_policy_names_appends_only_missing_values() {
let existing = vec!["readonly".to_string()];
let requested = vec!["readonly".to_string(), "writeonly".to_string()];
let (updated, attached) = attach_policy_names(&existing, &requested);
assert_eq!(updated, vec!["readonly".to_string(), "writeonly".to_string()]);
assert_eq!(attached, vec!["writeonly".to_string()]);
}
#[test]
fn detach_policy_names_removes_only_requested_values() {
let existing = vec!["readonly".to_string(), "writeonly".to_string()];
let requested = vec!["writeonly".to_string(), "missing".to_string()];
let (updated, detached) = detach_policy_names(&existing, &requested);
assert_eq!(updated, vec!["readonly".to_string()]);
assert_eq!(detached, vec!["writeonly".to_string()]);
}
#[test]
fn policy_entities_query_supports_repeated_external_parameters() {
let query = parse_policy_entities_query(Some("user=alice&user=bob&group=ops&policy=readonly&policy=writeonly"));
assert_eq!(query.users, vec!["alice".to_string(), "bob".to_string()]);
assert_eq!(query.groups, vec!["ops".to_string()]);
assert_eq!(query.policies, vec!["readonly".to_string(), "writeonly".to_string()]);
}
#[test]
fn build_policy_mappings_indexes_users_and_groups() {
let user_mappings = vec![UserPolicyEntities {
user: "alice".to_string(),
policies: vec!["readonly".to_string()],
member_of_mappings: Vec::new(),
}];
let group_mappings = vec![GroupPolicyEntities {
group: "ops".to_string(),
policies: vec!["readonly".to_string(), "writeonly".to_string()],
}];
let mappings = build_policy_mappings(&user_mappings, &group_mappings, &[]);
assert_eq!(mappings.len(), 2);
assert_eq!(mappings[0].policy, "readonly");
assert_eq!(mappings[0].users, vec!["alice".to_string()]);
assert_eq!(mappings[0].groups, vec!["ops".to_string()]);
assert_eq!(mappings[1].policy, "writeonly");
assert_eq!(mappings[1].groups, vec!["ops".to_string()]);
}
#[test]
fn direct_user_policy_names_only_reads_direct_mapping() {
let user_info = UserInfo {
policy_name: Some("readonly,writeonly".to_string()),
member_of: Some(vec!["disabled-group".to_string()]),
..Default::default()
};
assert_eq!(
direct_user_policy_names(&user_info),
vec!["readonly".to_string(), "writeonly".to_string()]
);
}
}
+6 -1
View File
@@ -220,7 +220,12 @@ impl Operation for StartDecommission {
));
}
// TODO: check IsRebalanceStarted
if store.is_rebalance_started().await {
return Err(S3Error::with_message(
S3ErrorCode::OperationAborted,
"Decommission cannot be started, rebalance is already in progress".to_string(),
));
}
let query = {
if let Some(query) = req.uri.query() {
+143 -26
View File
@@ -31,6 +31,7 @@ use serde_json;
use std::sync::Arc;
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
use url::form_urlencoded;
#[derive(Debug, Deserialize)]
pub struct SetBucketQuotaRequest {
@@ -39,10 +40,86 @@ pub struct SetBucketQuotaRequest {
pub quota_type: String,
}
#[derive(Debug, Deserialize)]
struct CompatibleBucketQuotaRequest {
#[serde(default)]
quota: Option<u64>,
#[serde(default)]
size: Option<u64>,
#[serde(default, alias = "quotatype")]
quota_type: Option<String>,
}
#[derive(Debug, Serialize)]
struct CompatibleBucketQuotaResponse {
quota: u64,
size: u64,
rate: u64,
requests: u64,
#[serde(skip_serializing_if = "String::is_empty")]
quotatype: String,
}
fn default_quota_type() -> String {
rustfs_config::QUOTA_TYPE_HARD.to_string()
}
fn is_compat_set_bucket_quota_path(path: &str) -> bool {
path.ends_with("/v3/set-bucket-quota")
}
fn is_compat_get_bucket_quota_path(path: &str) -> bool {
path.ends_with("/v3/get-bucket-quota")
}
fn bucket_from_params_or_query(params: &Params<'_, '_>, uri: &hyper::Uri) -> String {
if let Some(bucket) = params.get("bucket") {
return bucket.to_string();
}
if let Some(query) = uri.query() {
for (key, value) in form_urlencoded::parse(query.as_bytes()) {
if key == "bucket" {
return value.into_owned();
}
}
}
String::new()
}
fn parse_set_bucket_quota_request(body: &[u8]) -> Result<SetBucketQuotaRequest, s3s::S3Error> {
if body.is_empty() {
return Ok(SetBucketQuotaRequest {
quota: None,
quota_type: default_quota_type(),
});
}
let request: CompatibleBucketQuotaRequest =
serde_json::from_slice(body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))?;
Ok(SetBucketQuotaRequest {
quota: request
.size
.filter(|quota| *quota > 0)
.or(request.quota.filter(|quota| *quota > 0)),
quota_type: request.quota_type.unwrap_or_else(default_quota_type),
})
}
fn compat_bucket_quota_response(quota: &BucketQuota) -> CompatibleBucketQuotaResponse {
let size = quota.quota.unwrap_or(0);
CompatibleBucketQuotaResponse {
quota: size,
size,
rate: 0,
requests: 0,
quotatype: if size > 0 { "hard".to_string() } else { String::new() },
}
}
#[derive(Debug, Serialize)]
pub struct BucketQuotaResponse {
pub bucket: String,
@@ -105,6 +182,18 @@ async fn current_usage_from_context(bucket: &str) -> u64 {
}
pub fn register_quota_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
Method::PUT,
format!("{}{}", ADMIN_PREFIX, "/v3/set-bucket-quota").as_str(),
AdminOperation(&SetBucketQuotaHandler {}),
)?;
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/v3/get-bucket-quota").as_str(),
AdminOperation(&GetBucketQuotaHandler {}),
)?;
r.insert(
Method::PUT,
format!("{}{}", ADMIN_PREFIX, "/v3/quota/{bucket}").as_str(),
@@ -161,7 +250,7 @@ impl Operation for SetBucketQuotaHandler {
)
.await?;
let bucket = params.get("bucket").unwrap_or("").to_string();
let bucket = bucket_from_params_or_query(&params, &req.uri);
if bucket.is_empty() {
return Err(s3_error!(InvalidRequest, "bucket name is required"));
}
@@ -172,14 +261,7 @@ impl Operation for SetBucketQuotaHandler {
.await
.map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
let request: SetBucketQuotaRequest = if body.is_empty() {
SetBucketQuotaRequest {
quota: None,
quota_type: default_quota_type(),
}
} else {
serde_json::from_slice(&body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))?
};
let request = parse_set_bucket_quota_request(&body)?;
if request.quota_type.to_uppercase() != rustfs_config::QUOTA_TYPE_HARD {
return Err(s3_error!(InvalidArgument, "{}", rustfs_config::QUOTA_INVALID_TYPE_ERROR_MSG));
@@ -199,15 +281,18 @@ impl Operation for SetBucketQuotaHandler {
// Get real-time usage from data usage system
let current_usage = current_usage_from_context(&bucket).await;
let response = BucketQuotaResponse {
bucket,
quota: quota.quota,
size: current_usage,
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
};
let json = if is_compat_set_bucket_quota_path(req.uri.path()) {
String::new()
} else {
let response = BucketQuotaResponse {
bucket,
quota: quota.quota,
size: current_usage,
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
};
let json =
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?;
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?
};
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
}
@@ -226,7 +311,7 @@ impl Operation for GetBucketQuotaHandler {
let (cred, owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?;
let bucket = params.get("bucket").unwrap_or("").to_string();
let bucket = bucket_from_params_or_query(&params, &req.uri);
if bucket.is_empty() {
return Err(s3_error!(InvalidRequest, "bucket name is required"));
}
@@ -254,15 +339,19 @@ impl Operation for GetBucketQuotaHandler {
_ => s3_error!(InternalError, "Failed to get quota: {}", e),
})?;
let response = BucketQuotaResponse {
bucket,
quota: quota.quota,
size: current_usage.unwrap_or(0),
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
};
let json = if is_compat_get_bucket_quota_path(req.uri.path()) {
serde_json::to_string(&compat_bucket_quota_response(&quota))
.map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?
} else {
let response = BucketQuotaResponse {
bucket,
quota: quota.quota,
size: current_usage.unwrap_or(0),
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
};
let json =
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?;
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?
};
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
}
@@ -486,6 +575,34 @@ mod tests {
assert_eq!(default_quota_type(), "HARD");
}
#[test]
fn parse_set_bucket_quota_request_accepts_compat_shape() {
let request = parse_set_bucket_quota_request(br#"{"quota":1073741824,"size":1073741824,"quotatype":"hard"}"#)
.expect("parse quota request");
assert_eq!(request.quota, Some(1073741824));
assert_eq!(request.quota_type, "hard");
}
#[test]
fn parse_set_bucket_quota_request_prefers_non_zero_quota_over_zero_size() {
let request =
parse_set_bucket_quota_request(br#"{"quota":1073741824,"size":0,"quotatype":"hard"}"#).expect("parse quota request");
assert_eq!(request.quota, Some(1073741824));
assert_eq!(request.quota_type, "hard");
}
#[test]
fn compat_bucket_quota_response_uses_external_field_names() {
let quota = BucketQuota::new(Some(1024));
let json = serde_json::to_string(&compat_bucket_quota_response(&quota)).expect("serialize");
assert!(json.contains("\"quota\":1024"));
assert!(json.contains("\"size\":1024"));
assert!(json.contains("\"quotatype\":\"hard\""));
}
#[test]
fn test_quota_operation_parsing() {
let parse_operation = |operation: &str| match operation.to_uppercase().as_str() {
File diff suppressed because it is too large Load Diff
+106 -16
View File
@@ -17,7 +17,7 @@ use crate::{
admin::{
auth::validate_admin_request,
router::{AdminOperation, Operation, S3Router},
utils::has_space_be,
utils::{encode_compatible_admin_payload, has_space_be, read_compatible_admin_body},
},
auth::{check_key_valid, constant_time_eq, get_session_token},
server::RemoteAddr,
@@ -28,7 +28,7 @@ use rustfs_config::{MAX_ADMIN_REQUEST_BODY_SIZE, MAX_IAM_IMPORT_SIZE};
use rustfs_credentials::{Credentials, get_global_action_cred};
use rustfs_iam::{
store::{GroupInfo, MappedPolicy, UserType},
sys::NewServiceAccountOpts,
sys::{NewServiceAccountOpts, UpdateServiceAccountOpts},
};
use rustfs_madmin::{
AccountStatus, AddOrUpdateUserReq, IAMEntities, IAMErrEntities, IAMErrEntity, IAMErrPolicyEntity,
@@ -50,7 +50,7 @@ use zip::{ZipArchive, ZipWriter, result::ZipError, write::SimpleFileOptions};
#[derive(Debug, Deserialize, Default)]
pub struct AddUserQuery {
#[serde(rename = "accessKey")]
#[serde(rename = "accessKey", alias = "access-key")]
pub access_key: Option<String>,
pub status: Option<String>,
}
@@ -73,6 +73,30 @@ fn should_check_deny_only(target_access_key: &str, requester: &Credentials) -> b
&& !requester.is_service_account()
}
fn should_reject_group_import_name(group_name: &str, group_lookup: &rustfs_iam::error::Error) -> bool {
has_space_be(group_name) || !matches!(group_lookup, rustfs_iam::error::Error::NoSuchGroup(_))
}
fn should_restore_group_as_disabled(status: &str) -> bool {
status.eq_ignore_ascii_case(rustfs_iam::sys::STATUS_DISABLED)
}
fn imported_service_account_status(status: &str) -> Option<String> {
if status.eq_ignore_ascii_case(rustfs_policy::auth::ACCOUNT_OFF)
|| status.eq_ignore_ascii_case(rustfs_madmin::AccountStatus::Disabled.as_ref())
{
return Some(rustfs_policy::auth::ACCOUNT_OFF.to_string());
}
if status.eq_ignore_ascii_case(rustfs_policy::auth::ACCOUNT_ON)
|| status.eq_ignore_ascii_case(rustfs_madmin::AccountStatus::Enabled.as_ref())
{
return Some(rustfs_policy::auth::ACCOUNT_ON.to_string());
}
None
}
pub struct AddUser {}
#[async_trait::async_trait]
impl Operation for AddUser {
@@ -100,14 +124,7 @@ impl Operation for AddUser {
return Err(s3_error!(InvalidArgument, "access key is empty"));
}
let mut input = req.input;
let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await {
Ok(b) => b,
Err(e) => {
warn!("get body failed, e: {:?}", e);
return Err(s3_error!(InvalidRequest, "get body failed"));
}
};
let body = read_compatible_admin_body(req.input, MAX_ADMIN_REQUEST_BODY_SIZE, req.uri.path(), &cred.secret_key).await?;
// let body_bytes = decrypt_data(input_cred.secret_key.expose().as_bytes(), &body)
// .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, format!("decrypt_data err {}", e)))?;
@@ -282,9 +299,10 @@ impl Operation for ListUsers {
let data = serde_json::to_vec(&users)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal users err {e}")))?;
let (data, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, data)?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
header.insert(CONTENT_TYPE, content_type.parse().unwrap());
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
}
@@ -435,6 +453,7 @@ const ALL_SVC_ACCTS_FILE: &str = "svcaccts.json";
const USER_POLICY_MAPPINGS_FILE: &str = "user_mappings.json";
const GROUP_POLICY_MAPPINGS_FILE: &str = "group_mappings.json";
const STS_USER_POLICY_MAPPINGS_FILE: &str = "stsuser_mappings.json";
const GROUP_POLICY_MAPPING_USER_TYPE: UserType = UserType::Reg;
const IAM_ASSETS_DIR: &str = "iam-assets";
@@ -619,7 +638,7 @@ impl Operation for ExportIam {
GROUP_POLICY_MAPPINGS_FILE => {
let mut group_policy_mappings = HashMap::new();
iam_store
.load_mapped_policies(UserType::Reg, true, &mut group_policy_mappings)
.load_mapped_policies(GROUP_POLICY_MAPPING_USER_TYPE, true, &mut group_policy_mappings)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?;
@@ -805,7 +824,7 @@ impl Operation for ImportIam {
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?;
for (group_name, group_info) in groups {
if let Err(e) = iam_store.get_group_description(&group_name).await
&& (matches!(e, rustfs_iam::error::Error::NoSuchGroup(_)) || has_space_be(&group_name))
&& should_reject_group_import_name(&group_name, &e)
{
return Err(s3_error!(InvalidArgument, "group not found or has space be"));
}
@@ -816,6 +835,14 @@ impl Operation for ImportIam {
error: e.to_string(),
});
} else {
if should_restore_group_as_disabled(&group_info.status) {
iam_store.set_group_status(&group_name, false).await.map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("set group status failed, name: {group_name}, err: {e}"),
)
})?;
}
added.groups.push(group_name.clone());
}
}
@@ -893,6 +920,29 @@ impl Operation for ImportIam {
error: e.to_string(),
});
} else {
if let Some(status) = imported_service_account_status(&req.status)
&& status == rustfs_policy::auth::ACCOUNT_OFF
{
iam_store
.update_service_account(
&ak,
UpdateServiceAccountOpts {
session_policy: None,
secret_key: None,
name: None,
description: None,
expiration: None,
status: Some(status),
},
)
.await
.map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("update service account status failed, name: {ak}, err: {e}"),
)
})?;
}
added.service_accounts.push(ak.clone());
}
}
@@ -973,7 +1023,7 @@ impl Operation for ImportIam {
}
if let Err(e) = iam_store
.policy_db_set(&group_name, UserType::None, true, &policies.policies)
.policy_db_set(&group_name, GROUP_POLICY_MAPPING_USER_TYPE, true, &policies.policies)
.await
{
failed.group_policies.push(IAMErrPolicyEntity {
@@ -1064,8 +1114,12 @@ impl Operation for ImportIam {
#[cfg(test)]
mod tests {
use super::should_check_deny_only;
use super::{
GROUP_POLICY_MAPPING_USER_TYPE, imported_service_account_status, should_check_deny_only, should_reject_group_import_name,
should_restore_group_as_disabled,
};
use rustfs_credentials::{Credentials, IAM_POLICY_CLAIM_NAME_SA};
use rustfs_iam::error::Error as IamError;
use serde_json::Value;
use std::collections::HashMap;
@@ -1119,4 +1173,40 @@ mod tests {
};
assert!(!should_check_deny_only("alice", &cred));
}
#[test]
fn test_group_import_allows_missing_group_without_spaces() {
assert!(!should_reject_group_import_name(
"new-group",
&IamError::NoSuchGroup("new-group".to_string())
));
}
#[test]
fn test_group_import_rejects_group_names_with_spaces() {
assert!(should_reject_group_import_name(
" bad-group",
&IamError::NoSuchGroup(" bad-group".to_string())
));
}
#[test]
fn test_group_import_restores_disabled_status_only_when_needed() {
assert!(should_restore_group_as_disabled("disabled"));
assert!(!should_restore_group_as_disabled("enabled"));
}
#[test]
fn test_imported_service_account_status_maps_on_and_off() {
assert_eq!(imported_service_account_status("off").as_deref(), Some("off"));
assert_eq!(imported_service_account_status("on").as_deref(), Some("on"));
assert_eq!(imported_service_account_status("disabled").as_deref(), Some("off"));
assert_eq!(imported_service_account_status("enabled").as_deref(), Some("on"));
assert!(imported_service_account_status("unknown").is_none());
}
#[test]
fn test_group_policy_mappings_use_regular_user_type() {
assert_eq!(GROUP_POLICY_MAPPING_USER_TYPE, rustfs_iam::store::UserType::Reg);
}
}
+53 -1
View File
@@ -17,13 +17,17 @@ use crate::admin::{
router::{AdminOperation, S3Router},
rpc,
};
use crate::server::{ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH};
use crate::server::{ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH};
use hyper::Method;
fn admin_path(path: &str) -> String {
format!("{}{}", ADMIN_PREFIX, path)
}
fn compat_admin_alias_path(path: &str) -> String {
format!("{}{}", MINIO_ADMIN_PREFIX, path)
}
fn assert_route(router: &S3Router<AdminOperation>, method: Method, path: &str) {
assert!(
router.contains_route(method.clone(), path),
@@ -69,9 +73,17 @@ fn test_register_routes_cover_representative_admin_paths() {
assert_route(&router, Method::DELETE, &admin_path("/v3/group/test-group"));
assert_route(&router, Method::PUT, &admin_path("/v3/update-group-members"));
assert_route(&router, Method::PUT, &admin_path("/v3/add-service-accounts"));
assert_route(&router, Method::PUT, &admin_path("/v3/add-service-account"));
assert_route(&router, Method::GET, &admin_path("/v3/temporary-account-info"));
assert_route(&router, Method::GET, &admin_path("/v3/info-access-key"));
assert_route(&router, Method::GET, &admin_path("/v3/list-access-keys-bulk"));
assert_route(&router, Method::GET, &admin_path("/v3/export-iam"));
assert_route(&router, Method::PUT, &admin_path("/v3/import-iam"));
assert_route(&router, Method::GET, &admin_path("/v3/list-canned-policies"));
assert_route(&router, Method::PUT, &admin_path("/v3/set-policy"));
assert_route(&router, Method::POST, &admin_path("/v3/idp/builtin/policy/attach"));
assert_route(&router, Method::POST, &admin_path("/v3/idp/builtin/policy/detach"));
assert_route(&router, Method::GET, &admin_path("/v3/idp/builtin/policy-entities"));
assert_route(&router, Method::GET, &admin_path("/v3/target/list"));
assert_route(&router, Method::GET, &admin_path("/v3/accountinfo"));
@@ -89,6 +101,8 @@ fn test_register_routes_cover_representative_admin_paths() {
assert_route(&router, Method::GET, &admin_path("/v3/tier"));
assert_route(&router, Method::POST, &admin_path("/v3/tier/clear"));
assert_route(&router, Method::PUT, &admin_path("/v3/set-bucket-quota"));
assert_route(&router, Method::GET, &admin_path("/v3/get-bucket-quota"));
assert_route(&router, Method::PUT, &admin_path("/v3/quota/test-bucket"));
assert_route(&router, Method::GET, &admin_path("/v3/quota-stats/test-bucket"));
@@ -107,6 +121,44 @@ fn test_register_routes_cover_representative_admin_paths() {
assert_route(&router, Method::HEAD, "/rustfs/rpc/read_file_stream");
}
#[test]
fn test_admin_alias_paths_match_existing_admin_routes() {
let mut router: S3Router<AdminOperation> = S3Router::new(false);
health::register_health_route(&mut router).expect("register health route");
sts::register_admin_auth_route(&mut router).expect("register sts route");
user::register_user_route(&mut router).expect("register user route");
system::register_system_route(&mut router).expect("register system route");
pools::register_pool_route(&mut router).expect("register pool route");
rebalance::register_rebalance_route(&mut router).expect("register rebalance route");
quota::register_quota_route(&mut router).expect("register quota route");
for (method, path) in [
(Method::GET, compat_admin_alias_path("/v3/is-admin")),
(Method::GET, compat_admin_alias_path("/v3/info")),
(Method::GET, compat_admin_alias_path("/v3/storageinfo")),
(Method::GET, compat_admin_alias_path("/v3/pools/list")),
(Method::PUT, compat_admin_alias_path("/v3/add-service-account")),
(Method::GET, compat_admin_alias_path("/v3/temporary-account-info")),
(Method::GET, compat_admin_alias_path("/v3/info-access-key")),
(Method::GET, compat_admin_alias_path("/v3/list-access-keys-bulk")),
(Method::PUT, compat_admin_alias_path("/v3/set-policy")),
(Method::PUT, compat_admin_alias_path("/v3/set-bucket-quota")),
(Method::GET, compat_admin_alias_path("/v3/get-bucket-quota")),
(Method::POST, compat_admin_alias_path("/v3/idp/builtin/policy/attach")),
(Method::POST, compat_admin_alias_path("/v3/idp/builtin/policy/detach")),
(Method::GET, compat_admin_alias_path("/v3/idp/builtin/policy-entities")),
(Method::POST, compat_admin_alias_path("/v3/rebalance/start")),
] {
assert!(
router.contains_compatible_route(method.clone(), &path),
"expected MinIO admin alias path to match: {} {}",
method.as_str(),
path
);
}
}
#[test]
fn test_phase5_admin_info_and_rpc_read_file_contract() {
let system_src = include_str!("handlers/system.rs");
+42 -3
View File
@@ -14,7 +14,9 @@
use crate::admin::console::{is_console_path, make_console_server};
use crate::admin::handlers::oidc::is_oidc_path;
use crate::server::{ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX};
use crate::server::{
ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX,
};
use hyper::HeaderMap;
use hyper::Method;
use hyper::StatusCode;
@@ -43,6 +45,18 @@ fn is_public_health_path(path: &str) -> bool {
path == HEALTH_PREFIX || path == HEALTH_READY_PATH
}
fn is_admin_path(path: &str) -> bool {
path.starts_with(ADMIN_PREFIX) || path.starts_with(MINIO_ADMIN_PREFIX)
}
fn canonicalize_admin_path(path: &str) -> std::borrow::Cow<'_, str> {
if let Some(suffix) = path.strip_prefix(MINIO_ADMIN_PREFIX) {
return std::borrow::Cow::Owned(format!("{ADMIN_PREFIX}{suffix}"));
}
std::borrow::Cow::Borrowed(path)
}
impl<T: Operation> S3Router<T> {
pub fn new(console_enabled: bool) -> Self {
let router = Router::new();
@@ -81,6 +95,12 @@ impl<T: Operation> S3Router<T> {
let route = Self::make_route_str(method, path);
self.router.at(&route).is_ok()
}
pub(crate) fn contains_compatible_route(&self, method: Method, path: &str) -> bool {
let canonical_path = canonicalize_admin_path(path);
let route = Self::make_route_str(method, canonical_path.as_ref());
self.router.at(&route).is_ok()
}
}
impl<T: Operation> Default for S3Router<T> {
@@ -120,7 +140,7 @@ where
return true;
}
path.starts_with(ADMIN_PREFIX) || path.starts_with(RPC_PREFIX) || is_console_path(path)
is_admin_path(path) || path.starts_with(RPC_PREFIX) || is_console_path(path)
}
// check_access before call
@@ -207,7 +227,8 @@ where
return Err(s3_error!(InternalError, "console is not enabled"));
}
let uri = format!("{}|{}", &req.method, req.uri.path());
let canonical_path = canonicalize_admin_path(req.uri.path());
let uri = format!("{}|{}", &req.method, canonical_path.as_ref());
if let Ok(mat) = self.router.at(&uri) {
let op: &T = mat.value;
@@ -238,6 +259,24 @@ impl Operation for AdminOperation {
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn canonicalize_admin_path_maps_compat_prefix_to_rustfs_prefix() {
assert_eq!(canonicalize_admin_path("/minio/admin/v3/info").as_ref(), "/rustfs/admin/v3/info");
assert_eq!(canonicalize_admin_path("/rustfs/admin/v3/info").as_ref(), "/rustfs/admin/v3/info");
}
#[test]
fn is_admin_path_accepts_rustfs_and_compat_prefixes() {
assert!(is_admin_path("/rustfs/admin/v3/info"));
assert!(is_admin_path("/minio/admin/v3/info"));
assert!(!is_admin_path("/bucket/object"));
}
}
#[allow(dead_code)]
#[derive(Debug, Clone)]
pub struct Extra {
+84
View File
@@ -12,6 +12,90 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::server::MINIO_ADMIN_PREFIX;
use rustfs_crypto::{decrypt_data, decrypt_stream_io, encrypt_stream_io};
use s3s::{Body, S3Result, s3_error};
pub(crate) fn has_space_be(s: &str) -> bool {
s.trim().len() != s.len()
}
pub(crate) fn is_compat_admin_request(path: &str) -> bool {
path.starts_with(MINIO_ADMIN_PREFIX)
}
pub(crate) async fn read_compatible_admin_body(
mut input: Body,
max_len: usize,
path: &str,
secret_key: &str,
) -> S3Result<Vec<u8>> {
let body = input
.store_all_limited(max_len)
.await
.map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
if is_compat_admin_request(path) {
decrypt_stream_io(secret_key.as_bytes(), body.as_ref())
.or_else(|_| decrypt_data(secret_key.as_bytes(), body.as_ref()))
.map_err(|e| s3_error!(InvalidRequest, "failed to decrypt MinIO admin payload: {}", e))
} else {
Ok(body.to_vec())
}
}
pub(crate) fn encode_compatible_admin_payload(path: &str, secret_key: &str, data: Vec<u8>) -> S3Result<(Vec<u8>, &'static str)> {
if is_compat_admin_request(path) {
let encrypted = encrypt_stream_io(secret_key.as_bytes(), &data)
.map_err(|e| s3_error!(InternalError, "failed to encrypt MinIO admin payload: {}", e))?;
Ok((encrypted, "application/octet-stream"))
} else {
Ok((data, "application/json"))
}
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_crypto::encrypt_data;
use s3s::Body;
#[test]
fn detects_compat_admin_paths_only_for_external_prefix() {
assert!(is_compat_admin_request("/minio/admin/v3/list-users"));
assert!(!is_compat_admin_request("/rustfs/admin/v3/list-users"));
}
#[test]
fn encodes_plain_payload_for_rustfs_admin_paths() {
let payload = b"{\"ok\":true}".to_vec();
let (encoded, content_type) =
encode_compatible_admin_payload("/rustfs/admin/v3/list-users", "secret", payload.clone()).expect("encode payload");
assert_eq!(encoded, payload);
assert_eq!(content_type, "application/json");
}
#[test]
fn encodes_compat_payload_with_compatible_encryption() {
let payload = b"{\"ok\":true}".to_vec();
let (encoded, content_type) =
encode_compatible_admin_payload("/minio/admin/v3/list-users", "secret", payload.clone()).expect("encode payload");
assert_ne!(encoded, payload);
assert_eq!(content_type, "application/octet-stream");
assert_eq!(decrypt_stream_io(b"secret", &encoded).expect("decrypt payload"), payload);
}
#[tokio::test]
async fn reads_legacy_compat_payload_as_fallback() {
let payload = b"{\"ok\":true}".to_vec();
let encrypted = encrypt_data(b"secret", &payload).expect("encrypt payload");
let decoded = read_compatible_admin_body(Body::from(encrypted), 1024, "/minio/admin/v3/list-users", "secret")
.await
.expect("decode payload");
assert_eq!(decoded, payload);
}
}
+74 -123
View File
@@ -19,8 +19,8 @@ use crate::auth::get_condition_values;
use crate::error::ApiError;
use crate::server::RemoteAddr;
use crate::storage::access::{ReqInfo, authorize_request, req_info_ref};
use crate::storage::ecfs::{RUSTFS_OWNER, default_owner};
use crate::storage::helper::OperationHelper;
use crate::storage::s3_api::bucket::{build_list_buckets_output, build_list_objects_v2_output};
use crate::storage::s3_api::{acl, encryption, replication, tagging};
use crate::storage::*;
use futures::StreamExt;
@@ -51,7 +51,7 @@ use rustfs_targets::{
EventName,
arn::{ARN, TargetIDError},
};
use rustfs_utils::http::RUSTFS_FORCE_DELETE;
use rustfs_utils::http::{SUFFIX_FORCE_DELETE, get_header};
use rustfs_utils::string::parse_bool;
use s3s::dto::*;
use s3s::region::Region;
@@ -59,7 +59,6 @@ use s3s::xml;
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
use std::{collections::HashSet, fmt::Display, sync::Arc};
use tracing::{debug, error, info, instrument, warn};
use urlencoding::encode;
fn serialize_config<T: xml::Serialize>(value: &T) -> S3Result<Vec<u8>> {
serialize(value).map_err(to_internal_error)
@@ -69,6 +68,17 @@ fn to_internal_error(err: impl Display) -> S3Error {
S3Error::with_message(S3ErrorCode::InternalError, format!("{err}"))
}
fn create_bucket_exists_response(is_owner: bool) -> S3Result<S3Response<CreateBucketOutput>> {
if is_owner {
return Ok(S3Response::new(CreateBucketOutput::default()));
}
Err(s3_error!(
BucketAlreadyExists,
"The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again."
))
}
fn resolve_notification_region(global_region: Option<Region>, request_region: Option<Region>) -> String {
global_region
.or(request_region)
@@ -147,8 +157,8 @@ impl DefaultBucketUsecase {
}
let helper = OperationHelper::new(&req, EventName::BucketCreated, S3Operation::CreateBucket);
let requester_id = match req_info_ref(&req) {
Ok(r) => r.cred.as_ref().map(|c| c.access_key.clone()),
let requester_is_owner = match req_info_ref(&req) {
Ok(r) => r.is_owner,
Err(_) => {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Missing request info".to_string()));
}
@@ -179,18 +189,7 @@ impl DefaultBucketUsecase {
Err(StorageError::BucketExists(_)) => {
// Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK;
// non-owner gets 409 BucketAlreadyExists.
let is_owner = requester_id.as_deref().is_some_and(|req_id| req_id == default_owner().id);
if is_owner {
let output = CreateBucketOutput::default();
let result = Ok(S3Response::new(output));
let _ = helper.complete(&result);
return result;
}
let result = Err(s3_error!(
BucketAlreadyExists,
"The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again."
));
let result = create_bucket_exists_response(requester_is_owner);
let _ = helper.complete(&result);
return result;
}
@@ -247,19 +246,11 @@ impl DefaultBucketUsecase {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
// get value from header, support mc style
let force_str = req
.headers
.get(RUSTFS_FORCE_DELETE)
.map(|v| v.to_str().unwrap_or_default())
.unwrap_or(
req.headers
.get("x-minio-force-delete")
.map(|v| v.to_str().unwrap_or_default())
.unwrap_or_default(),
);
let force_str = get_header(&req.headers, SUFFIX_FORCE_DELETE)
.map(|v| v.into_owned())
.unwrap_or_default();
let force = parse_bool(force_str).unwrap_or_default();
let force = parse_bool(&force_str).unwrap_or_default();
if force {
authorize_request(&mut req, Action::S3Action(S3Action::ForceDeleteBucketAction)).await?;
@@ -404,20 +395,7 @@ impl DefaultBucketUsecase {
store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?
};
let buckets: Vec<Bucket> = bucket_infos
.iter()
.map(|v| Bucket {
creation_date: v.created.map(Timestamp::from),
name: Some(v.name.clone()),
..Default::default()
})
.collect();
Ok(S3Response::new(ListBucketsOutput {
buckets: Some(buckets),
owner: Some(RUSTFS_OWNER.to_owned()),
..Default::default()
}))
Ok(S3Response::new(build_list_buckets_output(&bucket_infos)))
}
pub async fn execute_delete_bucket_encryption(
@@ -1413,10 +1391,9 @@ impl DefaultBucketUsecase {
let store = get_validated_store(&bucket).await?;
let incl_deleted = req
.headers
.get(rustfs_utils::http::headers::RUSTFS_INCLUDE_DELETED)
.is_some_and(|v| v.to_str().unwrap_or_default() == "true");
let incl_deleted = rustfs_utils::http::get_header(&req.headers, rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
.map(|v| v.as_ref() == "true")
.unwrap_or_default();
let object_infos = store
.list_objects_v2(
@@ -1432,84 +1409,18 @@ impl DefaultBucketUsecase {
.await
.map_err(ApiError::from)?;
// warn!("object_infos objects {:?}", object_infos.objects);
// Apply URL encoding if encoding_type is "url"
// Note: S3 URL encoding should encode special characters but preserve path separators (/)
let should_encode = encoding_type.as_ref().map(|e| e.as_str() == "url").unwrap_or(false);
// Helper function to encode S3 keys/prefixes (preserving /)
// S3 URL encoding encodes special characters but keeps '/' unencoded
let encode_s3_name = |name: &str| -> String {
name.split('/')
.map(|part| encode(part).to_string())
.collect::<Vec<_>>()
.join("/")
};
let objects: Vec<Object> = object_infos
.objects
.iter()
.filter(|v| !v.name.is_empty())
.map(|v| {
let key = if should_encode {
encode_s3_name(&v.name)
} else {
v.name.to_owned()
};
let mut obj = Object {
key: Some(key),
last_modified: v.mod_time.map(Timestamp::from),
size: Some(v.get_actual_size().unwrap_or_default()),
e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)),
storage_class: v.storage_class.clone().map(ObjectStorageClass::from),
..Default::default()
};
if fetch_owner.is_some_and(|v| v) {
obj.owner = Some(Owner {
display_name: Some("rustfs".to_owned()),
id: Some("v0.1".to_owned()),
});
}
obj
})
.collect();
let common_prefixes: Vec<CommonPrefix> = object_infos
.prefixes
.into_iter()
.map(|v| {
let prefix = if should_encode { encode_s3_name(&v) } else { v };
CommonPrefix { prefix: Some(prefix) }
})
.collect();
// KeyCount should include both objects and common prefixes per S3 API spec
let key_count = (objects.len() + common_prefixes.len()) as i32;
// Encode next_continuation_token to base64
let next_continuation_token = object_infos
.next_continuation_token
.map(|token| base64_simd::STANDARD.encode_to_string(token.as_bytes()));
let output = ListObjectsV2Output {
is_truncated: Some(object_infos.is_truncated),
continuation_token: response_continuation_token,
next_continuation_token,
start_after: response_start_after,
key_count: Some(key_count),
max_keys: Some(max_keys),
contents: Some(objects),
let output = build_list_objects_v2_output(
object_infos,
fetch_owner.unwrap_or_default(),
max_keys,
bucket,
prefix,
delimiter,
encoding_type: encoding_type.clone(),
name: Some(bucket),
prefix: Some(prefix),
common_prefixes: Some(common_prefixes),
..Default::default()
};
encoding_type,
response_continuation_token,
response_start_after,
);
// let output = ListObjectsV2Output { ..Default::default() };
Ok(S3Response::new(output))
}
@@ -1679,6 +1590,12 @@ mod tests {
}
}
fn build_request_with_req_info<T>(input: T, method: Method, req_info: ReqInfo) -> S3Request<T> {
let mut req = build_request(input, method);
req.extensions.insert(req_info);
req
}
#[test]
fn resolve_notification_region_prefers_global_region() {
let binding = resolve_notification_region(Some("us-east-1".parse().unwrap()), Some("ap-southeast-1".parse().unwrap()));
@@ -1697,6 +1614,36 @@ mod tests {
assert_eq!(binding, RUSTFS_REGION);
}
#[test]
fn create_bucket_exists_response_returns_ok_for_owner() {
let response = create_bucket_exists_response(true).expect("owner recreate should succeed");
assert_eq!(response.output.location, None);
}
#[test]
fn create_bucket_exists_response_returns_bucket_already_exists_for_non_owner() {
let err = create_bucket_exists_response(false).expect_err("non-owner recreate should fail");
assert_eq!(err.code(), &S3ErrorCode::BucketAlreadyExists);
}
#[test]
fn build_request_with_req_info_preserves_owner_state() {
let input = CreateBucketInput::builder()
.bucket("test-bucket".to_string())
.build()
.unwrap();
let req = build_request_with_req_info(
input,
Method::PUT,
ReqInfo {
is_owner: true,
..Default::default()
},
);
assert!(req_info_ref(&req).expect("req info should be present").is_owner);
}
#[tokio::test]
async fn execute_create_bucket_returns_internal_error_when_store_uninitialized() {
let input = CreateBucketInput::builder()
@@ -1886,6 +1833,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -1900,6 +1848,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("rule-1".to_string()),
noncurrent_version_expiration: None,
@@ -1914,6 +1863,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -1939,6 +1889,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
+195 -75
View File
@@ -17,17 +17,17 @@
use crate::app::context::{AppContext, get_global_app_context};
use crate::error::ApiError;
use crate::storage::concurrency::get_concurrency_manager;
use crate::storage::ecfs::RUSTFS_OWNER;
use crate::storage::entity;
use crate::storage::helper::OperationHelper;
use crate::storage::options::{
copy_src_opts, extract_metadata, get_complete_multipart_upload_opts, get_content_sha256_with_query, parse_copy_source_range,
put_opts,
};
use crate::storage::s3_api::multipart::build_list_parts_output;
use crate::storage::*;
use bytes::Bytes;
use futures::StreamExt;
use rustfs_config::RUSTFS_REGION;
use http::{HeaderMap, Uri};
use rustfs_ecstore::bucket::quota::checker::QuotaChecker;
use rustfs_ecstore::bucket::{
metadata_sys,
@@ -47,18 +47,18 @@ use rustfs_s3_common::S3Operation;
use rustfs_targets::EventName;
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::http::{
AMZ_CHECKSUM_TYPE,
headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER},
AMZ_CHECKSUM_TYPE, get_source_scheme,
headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING},
};
use s3s::dto::*;
use s3s::region::Region;
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::str::FromStr;
use std::sync::Arc;
use tokio::sync::RwLock;
use tokio_util::io::StreamReader;
use tracing::{info, instrument, warn};
use urlencoding::encode;
/// Returns InvalidRange error if CopySourceRange end exceeds the source object size.
/// Used by execute_upload_part_copy to reject out-of-bounds ranges per S3 spec.
@@ -72,6 +72,74 @@ fn validate_copy_source_range_not_exceeds(range_spec: &HTTPRangeSpec, object_siz
Ok(())
}
fn validate_complete_multipart_parts(parts: &[CompletePart]) -> S3Result<()> {
if parts.windows(2).any(|window| window[0].part_num >= window[1].part_num) {
return Err(s3_error!(InvalidPartOrder, "Part numbers must be strictly increasing"));
}
Ok(())
}
fn normalize_complete_multipart_parts(parts: Vec<CompletePart>) -> S3Result<Vec<CompletePart>> {
// For duplicate part numbers, keep the last occurrence from the request.
// This matches retry/resend semantics where later uploads override earlier ones.
let mut seen = HashSet::with_capacity(parts.len());
let mut deduped_reversed = Vec::with_capacity(parts.len());
for part in parts.into_iter().rev() {
if seen.insert(part.part_num) {
deduped_reversed.push(part);
}
}
deduped_reversed.reverse();
validate_complete_multipart_parts(&deduped_reversed)?;
Ok(deduped_reversed)
}
fn encode_s3_path(path: &str) -> String {
path.split('/')
.map(|part| encode(part).to_string())
.collect::<Vec<_>>()
.join("/")
}
fn extract_request_scheme(headers: &HeaderMap, uri: &Uri) -> String {
get_source_scheme(headers)
.and_then(|value| {
value
.split(',')
.next()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
})
.or_else(|| uri.scheme_str().map(str::to_owned))
.unwrap_or_else(|| "http".to_string())
.to_ascii_lowercase()
}
fn extract_request_host(headers: &HeaderMap, uri: &Uri) -> Option<String> {
headers
.get(http::header::HOST)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.or_else(|| uri.authority().map(|authority| authority.as_str().to_string()))
}
fn build_complete_multipart_location(headers: &HeaderMap, uri: &Uri, bucket: &str, key: &str) -> String {
let object_path = format!("/{}/{}", encode(bucket), encode_s3_path(key));
match extract_request_host(headers, uri) {
Some(host) => {
let scheme = extract_request_scheme(headers, uri);
format!("{scheme}://{host}{object_path}")
}
None => object_path,
}
}
#[derive(Clone, Default)]
pub struct DefaultMultipartUsecase {
context: Option<Arc<AppContext>>,
@@ -93,10 +161,6 @@ impl DefaultMultipartUsecase {
self.context.as_ref().and_then(|context| context.bucket_metadata().handle())
}
fn global_region(&self) -> Option<Region> {
self.context.as_ref().and_then(|context| context.region().get())
}
#[instrument(level = "debug", skip(self))]
pub async fn execute_abort_multipart_upload(
&self,
@@ -213,23 +277,7 @@ impl DefaultMultipartUsecase {
.map(CompletePart::from)
.collect::<Vec<_>>();
// is part number sorted?
if !uploaded_parts_vec.is_sorted_by_key(|p| p.part_num) {
return Err(s3_error!(InvalidPart, "Part numbers must be sorted"));
}
// Handle duplicate part numbers: according to S3 specification, when the same part number
// is uploaded multiple times, the last uploaded part (in the list order) should be used.
// This can happen in concurrent upload scenarios where a part is re-uploaded before completion.
// We deduplicate by keeping the last occurrence of each part number using a HashMap.
let mut part_map: HashMap<usize, CompletePart> = HashMap::new();
for part in uploaded_parts_vec {
part_map.insert(part.part_num, part);
}
// Reconstruct the parts list in sorted order, keeping only the last occurrence of each part number
let mut uploaded_parts: Vec<CompletePart> = part_map.into_values().collect();
uploaded_parts.sort_by_key(|p| p.part_num);
let uploaded_parts = normalize_complete_multipart_parts(uploaded_parts_vec)?;
// TODO: check object lock
@@ -354,15 +402,12 @@ impl DefaultMultipartUsecase {
}
}
let region = self
.global_region()
.map(|region| region.to_string())
.unwrap_or_else(|| RUSTFS_REGION.to_string());
let location = build_complete_multipart_location(&req.headers, &req.uri, &bucket, &key);
let output = CompleteMultipartUploadOutput {
bucket: Some(bucket.clone()),
key: Some(key.clone()),
e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)),
location: Some(region.clone()),
location: Some(location.clone()),
server_side_encryption: server_side_encryption.clone(),
ssekms_key_id: ssekms_key_id.clone(),
checksum_crc32: checksum_crc32.clone(),
@@ -378,7 +423,7 @@ impl DefaultMultipartUsecase {
bucket: Some(bucket.clone()),
key: Some(key.clone()),
e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)),
location: Some(region),
location: Some(location),
server_side_encryption,
ssekms_key_id,
checksum_crc32,
@@ -482,8 +527,9 @@ impl DefaultMultipartUsecase {
};
if is_compressible(&req.headers, &key) {
metadata.insert(
format!("{RESERVED_METADATA_PREFIX_LOWER}compression"),
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
CompressionAlgorithm::default().to_string(),
);
}
@@ -603,9 +649,7 @@ impl DefaultMultipartUsecase {
StreamReader::new(body_stream.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))),
);
let is_compressible = fi
.user_defined
.contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}compression").as_str());
let is_compressible = rustfs_utils::http::contains_key_str(&fi.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
let mut reader: Box<dyn Reader> = Box::new(WarpReader::new(body));
@@ -857,39 +901,7 @@ impl DefaultMultipartUsecase {
.await
.map_err(ApiError::from)?;
let output = ListPartsOutput {
bucket: Some(res.bucket),
key: Some(res.object),
upload_id: Some(res.upload_id),
parts: Some(
res.parts
.into_iter()
.map(|p| Part {
e_tag: p.etag.map(|etag| to_s3s_etag(&etag)),
last_modified: p.last_mod.map(Timestamp::from),
part_number: Some(p.part_num as i32),
size: Some(p.size as i64),
..Default::default()
})
.collect(),
),
owner: Some(RUSTFS_OWNER.to_owned()),
initiator: Some(Initiator {
id: RUSTFS_OWNER.id.clone(),
display_name: RUSTFS_OWNER.display_name.clone(),
}),
is_truncated: Some(res.is_truncated),
next_part_number_marker: res.next_part_number_marker.try_into().ok(),
max_parts: res.max_parts.try_into().ok(),
part_number_marker: res.part_number_marker.try_into().ok(),
storage_class: if res.storage_class.is_empty() {
None
} else {
Some(res.storage_class.into())
},
..Default::default()
};
Ok(S3Response::new(output))
Ok(S3Response::new(build_list_parts_output(res)))
}
#[instrument(level = "debug", skip(self, req))]
@@ -1014,9 +1026,7 @@ impl DefaultMultipartUsecase {
.map_err(ApiError::from)?;
let src_stream = src_reader.stream;
let is_compressible = mp_info
.user_defined
.contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}compression").as_str());
let is_compressible = rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
let mut reader: Box<dyn Reader> = Box::new(WarpReader::new(src_stream));
@@ -1129,7 +1139,7 @@ impl DefaultMultipartUsecase {
#[cfg(test)]
mod tests {
use super::*;
use http::{Extensions, HeaderMap, Method, Uri};
use http::{Extensions, HeaderMap, Method, Uri, header::HeaderValue};
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
@@ -1149,6 +1159,41 @@ mod tests {
DefaultMultipartUsecase::without_context()
}
#[test]
fn test_build_complete_multipart_location_uses_forwarded_proto_and_encodes_key() {
let mut headers = HeaderMap::new();
headers.insert(http::header::HOST, HeaderValue::from_static("storage.example.com:9000"));
headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
let location = build_complete_multipart_location(
&headers,
&Uri::from_static("/bucket/object?uploadId=1"),
"bucket",
"dir/file name.txt",
);
assert_eq!(location, "https://storage.example.com:9000/bucket/dir/file%20name.txt");
}
#[test]
fn test_build_complete_multipart_location_falls_back_to_uri_authority_and_scheme() {
let location = build_complete_multipart_location(
&HeaderMap::new(),
&"https://gateway.example.com:9443/complete".parse::<Uri>().unwrap(),
"bucket",
"object.txt",
);
assert_eq!(location, "https://gateway.example.com:9443/bucket/object.txt");
}
#[test]
fn test_build_complete_multipart_location_returns_path_without_host() {
let location = build_complete_multipart_location(&HeaderMap::new(), &Uri::from_static("/"), "bucket", "nested/object");
assert_eq!(location, "/bucket/nested/object");
}
#[tokio::test]
async fn execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized() {
let input = AbortMultipartUploadInput::builder()
@@ -1191,6 +1236,81 @@ mod tests {
assert_eq!(err.code(), &S3ErrorCode::InvalidPart);
}
#[tokio::test]
async fn execute_complete_multipart_upload_allows_duplicate_part_numbers_by_using_last_occurrence() {
let multipart_upload = CompletedMultipartUpload {
parts: Some(vec![
CompletedPart {
part_number: Some(1),
..Default::default()
},
CompletedPart {
part_number: Some(1),
..Default::default()
},
]),
};
let input = CompleteMultipartUploadInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.upload_id("upload-id".to_string())
.multipart_upload(Some(multipart_upload))
.build()
.unwrap();
let req = build_request(input, Method::POST);
let err = make_usecase().execute_complete_multipart_upload(req).await.unwrap_err();
assert_ne!(err.code(), &S3ErrorCode::InvalidPartOrder);
}
#[tokio::test]
async fn execute_complete_multipart_upload_rejects_out_of_order_parts() {
let multipart_upload = CompletedMultipartUpload {
parts: Some(vec![
CompletedPart {
part_number: Some(2),
..Default::default()
},
CompletedPart {
part_number: Some(1),
..Default::default()
},
]),
};
let input = CompleteMultipartUploadInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.upload_id("upload-id".to_string())
.multipart_upload(Some(multipart_upload))
.build()
.unwrap();
let req = build_request(input, Method::POST);
let err = make_usecase().execute_complete_multipart_upload(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidPartOrder);
}
#[test]
fn normalize_complete_multipart_parts_keeps_last_duplicate_part() {
let input = vec![
CompletePart {
part_num: 1,
etag: Some("old".to_string()),
..Default::default()
},
CompletePart {
part_num: 1,
etag: Some("new".to_string()),
..Default::default()
},
];
let normalized = normalize_complete_multipart_parts(input).expect("normalization should succeed");
assert_eq!(normalized.len(), 1);
assert_eq!(normalized[0].part_num, 1);
assert_eq!(normalized[0].etag.as_deref(), Some("new"));
}
#[tokio::test]
async fn execute_list_multipart_uploads_returns_internal_error_when_store_uninitialized() {
let input = ListMultipartUploadsInput::builder()
+20 -41
View File
@@ -82,13 +82,14 @@ use rustfs_s3select_api::{
use rustfs_s3select_query::get_global_db;
use rustfs_targets::EventName;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, RESERVED_METADATA_PREFIX,
AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION,
SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP,
headers::{
AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE,
AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT,
RESERVED_METADATA_PREFIX_LOWER,
},
insert_str, remove_str,
};
use rustfs_utils::path::{is_dir_object, path_join_buf};
use rustfs_utils::{
@@ -426,9 +427,8 @@ impl DefaultObjectUsecase {
if is_compressible(&req.headers, &key) && size > MIN_COMPRESSIBLE_SIZE as i64 {
let algorithm = CompressionAlgorithm::default();
metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), algorithm.to_string());
metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size",), size.to_string());
insert_str(&mut metadata, SUFFIX_COMPRESSION, algorithm.to_string());
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
let mut hrd = HashReader::new(reader, size as i64, size as i64, md5hex, sha256hex, false).map_err(ApiError::from)?;
@@ -437,10 +437,8 @@ impl DefaultObjectUsecase {
}
opts.want_checksum = hrd.checksum();
opts.user_defined
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), algorithm.to_string());
opts.user_defined
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size",), size.to_string());
insert_str(&mut opts.user_defined, SUFFIX_COMPRESSION, algorithm.to_string());
insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string());
reader = Box::new(CompressReader::new(hrd, algorithm));
size = HashReader::SIZE_PRESERVE_LAYER;
@@ -497,10 +495,12 @@ impl DefaultObjectUsecase {
let dsc = must_replicate(&bucket, &key, repoptions).await;
if dsc.replicate_any() {
let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp");
opts.user_defined.insert(k, jiff::Zoned::now().to_string());
let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status");
opts.user_defined.insert(k, dsc.pending_status().unwrap_or_default());
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(
&mut opts.user_defined,
SUFFIX_REPLICATION_STATUS,
dsc.pending_status().unwrap_or_default(),
);
}
let obj_info = store
@@ -2104,11 +2104,8 @@ impl DefaultObjectUsecase {
let mut compress_metadata = HashMap::new();
if is_compressible(&req.headers, &key) && actual_size > MIN_COMPRESSIBLE_SIZE as i64 {
compress_metadata.insert(
format!("{RESERVED_METADATA_PREFIX_LOWER}compression"),
CompressionAlgorithm::default().to_string(),
);
compress_metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size",), actual_size.to_string());
insert_str(&mut compress_metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string());
insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
let hrd = EtagReader::new(reader, None);
@@ -2117,24 +2114,9 @@ impl DefaultObjectUsecase {
reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default()));
length = HashReader::SIZE_PRESERVE_LAYER;
} else {
src_info
.user_defined
.remove(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression"));
src_info
.user_defined
.remove(&format!("{RESERVED_METADATA_PREFIX}compression"));
src_info
.user_defined
.remove(&format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"));
src_info
.user_defined
.remove(&format!("{RESERVED_METADATA_PREFIX}actual-size"));
src_info
.user_defined
.remove(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression-size"));
src_info
.user_defined
.remove(&format!("{RESERVED_METADATA_PREFIX}compression-size"));
remove_str(&mut src_info.user_defined, SUFFIX_COMPRESSION);
remove_str(&mut src_info.user_defined, SUFFIX_ACTUAL_SIZE);
remove_str(&mut src_info.user_defined, SUFFIX_COMPRESSION_SIZE);
}
// Handle MetadataDirective REPLACE: replace user metadata while preserving system metadata.
@@ -3480,11 +3462,8 @@ impl DefaultObjectUsecase {
let actual_size = size;
if is_compressible(&HeaderMap::new(), &fpath) && size > MIN_COMPRESSIBLE_SIZE as i64 {
metadata.insert(
format!("{RESERVED_METADATA_PREFIX_LOWER}compression"),
CompressionAlgorithm::default().to_string(),
);
metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"), size.to_string());
insert_str(&mut metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string());
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
let hrd = HashReader::new(reader, size, actual_size, None, None, false).map_err(ApiError::from)?;
+72 -2
View File
@@ -15,8 +15,7 @@
#[cfg(test)]
#[allow(unsafe_op_in_unsafe_fn)]
mod tests {
use crate::config::Opt;
use clap::Parser;
use crate::config::{Config, Opt};
use rustfs_ecstore::disks_layout::DisksLayout;
use serial_test::serial;
use std::env;
@@ -56,6 +55,19 @@ mod tests {
verify_fn(&layout);
}
#[test]
#[serial]
fn test_server_subcommand_and_legacy_equivalence() {
// rustfs server <volume> and rustfs <volume> (legacy) must produce identical results
let legacy_args = vec!["rustfs", "/data/vol1"];
let server_args = vec!["rustfs", "server", "/data/vol1"];
let opt_legacy = Opt::parse_from(legacy_args);
let opt_server = Opt::parse_from(server_args);
assert_eq!(opt_legacy.volumes, opt_server.volumes);
assert_eq!(opt_legacy.address, opt_server.address);
assert_eq!(opt_legacy.console_address, opt_server.console_address);
}
#[test]
#[serial]
fn test_default_console_configuration() {
@@ -109,6 +121,64 @@ mod tests {
assert_eq!(console_port, 9001);
}
#[test]
#[serial]
fn test_external_prefixed_envs_are_accepted_by_parser() {
temp_env::with_vars(
[
("MINIO_VOLUMES", Some("/compat/vol1")),
("MINIO_ADDRESS", Some(":9100")),
("RUSTFS_VOLUMES", None),
("RUSTFS_ADDRESS", None),
],
|| {
let opt = Opt::parse_from(["rustfs"]);
assert_eq!(opt.volumes, vec!["/compat/vol1"]);
assert_eq!(opt.address, ":9100");
assert_eq!(std::env::var("RUSTFS_VOLUMES").as_deref(), Ok("/compat/vol1"));
assert_eq!(std::env::var("RUSTFS_ADDRESS").as_deref(), Ok(":9100"));
},
);
}
#[test]
#[serial]
fn test_root_envs_are_used_for_bootstrap_credentials() {
temp_env::with_vars(
[
("RUSTFS_VOLUMES", Some("/compat/vol1")),
("RUSTFS_ROOT_USER", Some("root-user")),
("RUSTFS_ROOT_PASSWORD", Some("root-password")),
("RUSTFS_ACCESS_KEY", None),
("RUSTFS_SECRET_KEY", None),
],
|| {
let config = Config::from_opt(Opt::parse_from(["rustfs"])).expect("config should parse");
assert_eq!(config.access_key, "root-user");
assert_eq!(config.secret_key, "root-password");
},
);
}
#[test]
#[serial]
fn test_access_key_env_takes_precedence_over_root_aliases() {
temp_env::with_vars(
[
("RUSTFS_VOLUMES", Some("/compat/vol1")),
("RUSTFS_ACCESS_KEY", Some("canonical-access")),
("RUSTFS_SECRET_KEY", Some("canonical-secret")),
("RUSTFS_ROOT_USER", Some("root-user")),
("RUSTFS_ROOT_PASSWORD", Some("root-password")),
],
|| {
let config = Config::from_opt(Opt::parse_from(["rustfs"])).expect("config should parse");
assert_eq!(config.access_key, "canonical-access");
assert_eq!(config.secret_key, "canonical-secret");
},
);
}
#[test]
#[serial]
fn test_volumes_and_disk_layout_parsing() {
+146 -9
View File
@@ -12,10 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use clap::Parser;
use clap::builder::NonEmptyStringValueParser;
use clap::{Args, Parser, Subcommand};
use const_str::concat;
use rustfs_config::RUSTFS_REGION;
use rustfs_utils::{apply_external_env_compat, get_env_opt_str};
use std::path::PathBuf;
use std::string::ToString;
@@ -50,9 +51,47 @@ const LONG_VERSION: &str = concat!(
concat!("git status :\n", build::GIT_STATUS_FILE),
);
/// Known subcommands. When the first arg matches one of these, it is treated as a subcommand.
const KNOWN_SUBCOMMANDS: &[&str] = &["server"];
/// Preprocess argv for legacy compatibility: `rustfs <volume>` and `rustfs --address ...` are
/// treated as `rustfs server <volume>` and `rustfs server --address ...` respectively.
/// Also: `rustfs` with no args becomes `rustfs server` (volumes from env).
fn preprocess_args_for_legacy(args: Vec<String>) -> Vec<String> {
if args.len() < 2 {
// rustfs -> rustfs server (volumes from RUSTFS_VOLUMES env)
return vec![args[0].clone(), "server".to_string()];
}
let first = &args[1];
// If first arg looks like a subcommand, do nothing
if KNOWN_SUBCOMMANDS.contains(&first.as_str()) {
return args;
}
// If first arg is a global flag (--help, --version), do nothing
if first == "--help" || first == "-h" || first == "--version" || first == "-V" {
return args;
}
// Legacy: rustfs <volume> or rustfs --address ... -> rustfs server <volume|--address ...>
let mut out = vec![args[0].clone(), "server".to_string()];
out.extend(args[1..].iter().cloned());
out
}
#[derive(Parser, Clone)]
#[command(version = SHORT_VERSION, long_version = LONG_VERSION)]
pub struct Opt {
#[command(name = "rustfs", version = SHORT_VERSION, long_version = LONG_VERSION)]
struct Cli {
#[command(subcommand)]
command: Option<Commands>,
}
#[derive(Subcommand, Clone)]
enum Commands {
/// Start the object storage server (default when no subcommand is given)
Server(ServerOpts),
}
#[derive(Args, Clone)]
struct ServerOpts {
/// DIR points to a directory on a filesystem.
#[arg(
required = true,
@@ -164,6 +203,96 @@ pub struct Opt {
pub buffer_profile: String,
}
/// Parsed server options. Public for tests and backward compatibility.
/// Use `Opt::parse_from` or `Config::parse()` to obtain.
#[derive(Clone)]
pub struct Opt {
pub volumes: Vec<String>,
pub address: String,
pub server_domains: Vec<String>,
pub access_key: Option<String>,
pub access_key_file: Option<PathBuf>,
pub secret_key: Option<String>,
pub secret_key_file: Option<PathBuf>,
pub console_enable: bool,
pub console_address: String,
pub obs_endpoint: String,
pub tls_path: Option<String>,
pub license: Option<String>,
pub region: Option<String>,
pub kms_enable: bool,
pub kms_backend: String,
pub kms_key_dir: Option<String>,
pub kms_vault_address: Option<String>,
pub kms_vault_token: Option<String>,
pub kms_default_key_id: Option<String>,
pub buffer_profile_disable: bool,
pub buffer_profile: String,
}
impl Opt {
fn from_server_opts(o: ServerOpts) -> Self {
Self {
volumes: o.volumes,
address: o.address,
server_domains: o.server_domains,
access_key: o.access_key,
access_key_file: o.access_key_file,
secret_key: o.secret_key,
secret_key_file: o.secret_key_file,
console_enable: o.console_enable,
console_address: o.console_address,
obs_endpoint: o.obs_endpoint,
tls_path: o.tls_path,
license: o.license,
region: o.region,
kms_enable: o.kms_enable,
kms_backend: o.kms_backend,
kms_key_dir: o.kms_key_dir,
kms_vault_address: o.kms_vault_address,
kms_vault_token: o.kms_vault_token,
kms_default_key_id: o.kms_default_key_id,
buffer_profile_disable: o.buffer_profile_disable,
buffer_profile: o.buffer_profile,
}
}
/// Parse from preprocessed args. Supports both `rustfs <volume>` and `rustfs server <volume>`.
pub fn parse_from<I, T>(args: I) -> Self
where
I: IntoIterator<Item = T>,
T: Into<std::ffi::OsString> + Clone,
{
let _ = apply_external_env_compat();
let args: Vec<String> = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect();
let args = preprocess_args_for_legacy(args);
let cli = Cli::parse_from(args);
let Commands::Server(opts) = cli.command.expect("server is the default subcommand");
Self::from_server_opts(opts)
}
/// Try parse from args, returns error on invalid input.
#[allow(dead_code)] // used in config_test
pub fn try_parse_from<I, T>(args: I) -> Result<Self, clap::Error>
where
I: IntoIterator<Item = T>,
T: Into<std::ffi::OsString> + Clone,
{
let _ = apply_external_env_compat();
let args: Vec<String> = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect();
let args = preprocess_args_for_legacy(args);
let cli = Cli::try_parse_from(args)?;
let Commands::Server(opts) = cli.command.expect("server is the default subcommand");
Ok(Self::from_server_opts(opts))
}
/// Parse from env::args(). Used by Config::parse().
fn parse() -> Self {
let args: Vec<String> = std::env::args().collect();
Self::parse_from(args)
}
}
#[derive(Clone)]
pub struct Config {
/// DIR points to a directory on a filesystem.
@@ -227,11 +356,7 @@ pub struct Config {
}
impl Config {
/// parse the command line arguments and environment arguments from [`Opt`] and convert them
/// into a ready to use [`Config`]
///
/// This includes some intermediate checks for mutually exclusive options
pub fn parse() -> std::io::Result<Self> {
fn from_opt(opt: Opt) -> std::io::Result<Self> {
let Opt {
volumes,
address,
@@ -254,7 +379,7 @@ impl Config {
kms_default_key_id,
buffer_profile_disable,
buffer_profile,
} = Opt::parse();
} = opt;
let access_key = access_key
.map(Ok)
@@ -262,6 +387,7 @@ impl Config {
let path = access_key_file.as_ref()?;
Some(std::fs::read_to_string(path))
})
.or_else(|| get_env_opt_str("RUSTFS_ROOT_USER").map(Ok))
.transpose()?
.unwrap_or_else(|| {
// neither argument was specified ... using default
@@ -276,6 +402,7 @@ impl Config {
let path = secret_key_file.as_ref()?;
Some(std::fs::read_to_string(path))
})
.or_else(|| get_env_opt_str("RUSTFS_ROOT_PASSWORD").map(Ok))
.transpose()?
.unwrap_or_else(|| {
// neither argument was specified ... using default
@@ -309,6 +436,16 @@ impl Config {
buffer_profile,
})
}
/// Parse the command line arguments and environment arguments from [`Opt`] and convert them
/// into a ready to use [`Config`].
///
/// Supports both `rustfs <volume>` (legacy) and `rustfs server <volume>`.
///
/// This includes some intermediate checks for mutually exclusive options.
pub fn parse() -> std::io::Result<Self> {
Self::from_opt(Opt::parse())
}
}
impl std::fmt::Debug for Config {
+73 -1
View File
@@ -50,6 +50,7 @@ use rustfs_credentials::init_global_action_credentials;
use rustfs_ecstore::store::init_lock_clients;
use rustfs_ecstore::{
bucket::metadata_sys::init_bucket_metadata_sys,
bucket::migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
bucket::replication::{get_global_replication_pool, init_background_replication},
config as ecconfig,
endpoints::EndpointServerPools,
@@ -69,7 +70,9 @@ use rustfs_iam::{init_iam_sys, init_oidc_sys};
use rustfs_metrics::init_metrics_system;
use rustfs_obs::{init_obs, set_global_guard};
use rustfs_scanner::init_data_scanner;
use rustfs_utils::{get_env_bool_with_aliases, net::parse_and_resolve_address};
use rustfs_utils::{
ExternalEnvCompatReport, apply_external_env_compat, get_env_bool_with_aliases, net::parse_and_resolve_address,
};
use rustls::crypto::aws_lc_rs::default_provider;
use std::io::{Error, Result};
use std::sync::Arc;
@@ -98,6 +101,10 @@ static GLOBAL: profiling::allocator::TracingAllocator<mimalloc::MiMalloc> =
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
fn main() {
if let Err(err) = bootstrap_external_prefix_compat() {
eprintln!("[WARN] Failed to bootstrap external-prefix compatibility: {err}");
}
let runtime = server::tokio_runtime_builder()
.build()
.expect("Failed to build Tokio runtime");
@@ -108,6 +115,41 @@ fn main() {
std::process::exit(1);
}
}
fn bootstrap_external_prefix_compat() -> Result<()> {
let env_compat_report = apply_external_env_compat();
if env_compat_report.conflict_count() > 0 {
// RUSTFS_* is the canonical namespace in this codebase, so on key conflicts we keep RUSTFS_*
// to preserve explicit user/operator overrides and avoid changing existing runtime behavior.
eprintln!(
"[WARN] Found {} source/RUSTFS_ conflict(s), keeping RUSTFS_ values: {}",
env_compat_report.conflict_count(),
env_compat_report.conflict_keys.join(", ")
);
}
if env_compat_report.mapped_count() == 0 {
return Ok(());
}
eprintln!(
"[INFO] Applying external-prefix compatibility in-process for {} variable(s): {}",
env_compat_report.mapped_count(),
format_external_prefix_mappings(&env_compat_report)
);
Ok(())
}
fn format_external_prefix_mappings(report: &ExternalEnvCompatReport) -> String {
report
.mapped_pairs
.iter()
.map(|(source_key, rustfs_key)| format!("{source_key}->{rustfs_key}"))
.collect::<Vec<_>>()
.join(", ")
}
async fn async_main() -> Result<()> {
// Parse the obtained parameters
let config = config::Config::parse()?;
@@ -298,6 +340,7 @@ async fn run(config: config::Config) -> Result<()> {
})?;
ecconfig::init();
ecconfig::try_migrate_server_config(store.clone()).await;
// // Initialize global configuration system
let mut retry_count = 0;
@@ -398,10 +441,13 @@ async fn run(config: config::Config) -> Result<()> {
// Collect bucket names into a vector
let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
try_migrate_bucket_metadata(store.clone()).await;
if let Some(pool) = get_global_replication_pool() {
pool.init_resync(ctx.clone(), buckets.clone()).await?;
}
try_migrate_iam_config(store.clone()).await;
init_bucket_metadata_sys(store.clone(), buckets.clone()).await;
// 3. Initialize IAM System (Blocking load)
@@ -634,3 +680,29 @@ async fn handle_shutdown(
state_manager.update(ServiceState::Stopped);
info!(target: "rustfs::main::handle_shutdown", "Server stopped successfully.");
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn format_external_prefix_mappings_lists_mapped_pairs() {
let report = ExternalEnvCompatReport {
mapped_pairs: vec![
("MINIO_ROOT_USER".to_string(), "RUSTFS_ROOT_USER".to_string()),
(
"MINIO_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(),
"RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(),
),
],
conflict_keys: Vec::new(),
};
let formatted = format_external_prefix_mappings(&report);
assert_eq!(
formatted,
"MINIO_ROOT_USER->RUSTFS_ROOT_USER, MINIO_NOTIFY_WEBHOOK_ENABLE_PRIMARY->RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY"
);
}
}
+2 -1
View File
@@ -21,7 +21,7 @@ use crate::server::{
ReadinessGateLayer, RemoteAddr, ServiceState, ServiceStateManager,
compress::{CompressionConfig, CompressionPredicate},
hybrid::hybrid,
layer::{ConditionalCorsLayer, ObjectAttributesEtagFixLayer, RedirectLayer},
layer::{AdminChunkedContentLengthCompatLayer, ConditionalCorsLayer, ObjectAttributesEtagFixLayer, RedirectLayer},
};
use crate::storage;
use crate::storage::tonic_service::make_server;
@@ -621,6 +621,7 @@ fn process_connection(
None
})
.layer(SetRequestIdLayer::x_request_id(MakeRequestUuid))
.layer(AdminChunkedContentLengthCompatLayer)
.layer(CatchPanicLayer::new())
// CRITICAL: Insert ReadinessGateLayer before business logic
// This stops requests from hitting IAMAuth or Storage if they are not ready.
+109 -2
View File
@@ -15,13 +15,14 @@
use crate::admin::console::is_console_path;
use crate::server::cors;
use crate::server::hybrid::HybridBody;
use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, RPC_PREFIX, RUSTFS_ADMIN_PREFIX};
use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX, MINIO_ADMIN_V3_PREFIX, RPC_PREFIX, RUSTFS_ADMIN_PREFIX};
use crate::storage::apply_cors_headers;
use bytes::Bytes;
use http::{HeaderMap, HeaderValue, Method, Request as HttpRequest, Response, StatusCode};
use http_body::Body;
use http_body_util::BodyExt;
use hyper::body::Incoming;
use rustfs_utils::get_env_opt_str;
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
@@ -98,6 +99,64 @@ where
}
}
#[derive(Clone)]
pub struct AdminChunkedContentLengthCompatLayer;
impl<S> Layer<S> for AdminChunkedContentLengthCompatLayer {
type Service = AdminChunkedContentLengthCompatService<S>;
fn layer(&self, inner: S) -> Self::Service {
AdminChunkedContentLengthCompatService { inner }
}
}
#[derive(Clone)]
pub struct AdminChunkedContentLengthCompatService<S> {
inner: S,
}
impl<S, ResBody> Service<HttpRequest<Incoming>> for AdminChunkedContentLengthCompatService<S>
where
S: Service<HttpRequest<Incoming>, Response = Response<ResBody>> + Clone + Send + 'static,
S::Future: Send + 'static,
S::Error: Into<Box<dyn std::error::Error + Send + Sync>> + Send + 'static,
ResBody: Send + 'static,
{
type Response = Response<ResBody>;
type Error = Box<dyn std::error::Error + Send + Sync>;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx).map_err(Into::into)
}
fn call(&mut self, mut req: HttpRequest<Incoming>) -> Self::Future {
if should_force_zero_content_length_for_admin_empty_body(&req) {
req.headers_mut()
.insert(http::header::CONTENT_LENGTH, HeaderValue::from_static("0"));
}
let mut inner = self.inner.clone();
Box::pin(async move { inner.call(req).await.map_err(Into::into) })
}
}
fn should_force_zero_content_length_for_admin_empty_body<B>(req: &HttpRequest<B>) -> bool {
req.method() == Method::PUT
&& is_empty_body_admin_put_path(req.uri().path())
&& !req.headers().contains_key(http::header::CONTENT_LENGTH)
}
fn is_empty_body_admin_put_path(path: &str) -> bool {
matches!(
path,
"/minio/admin/v3/set-user-status"
| "/minio/admin/v3/set-group-status"
| "/rustfs/admin/v3/set-user-status"
| "/rustfs/admin/v3/set-group-status"
)
}
#[derive(Clone)]
pub struct ObjectAttributesEtagFixLayer;
@@ -220,7 +279,9 @@ fn is_object_attributes_request(req: &HttpRequest<Incoming>) -> bool {
let path = req.uri().path();
if path.starts_with(ADMIN_PREFIX)
|| path.starts_with(MINIO_ADMIN_PREFIX)
|| path.starts_with(RUSTFS_ADMIN_PREFIX)
|| path.starts_with(MINIO_ADMIN_V3_PREFIX)
|| path.starts_with(CONSOLE_PREFIX)
|| path.starts_with(RPC_PREFIX)
{
@@ -253,7 +314,7 @@ pub struct ConditionalCorsLayer {
impl ConditionalCorsLayer {
pub fn new() -> Self {
let cors_origins = std::env::var("RUSTFS_CORS_ALLOWED_ORIGINS").ok().filter(|s| !s.is_empty());
let cors_origins = get_env_opt_str("RUSTFS_CORS_ALLOWED_ORIGINS").filter(|s| !s.is_empty());
Self { cors_origins }
}
@@ -263,6 +324,7 @@ impl ConditionalCorsLayer {
fn is_s3_path(path: &str) -> bool {
// Exclude Admin, Console, RPC, and configured special paths
!path.starts_with(ADMIN_PREFIX)
&& !path.starts_with(MINIO_ADMIN_PREFIX)
&& !path.starts_with(RPC_PREFIX)
&& !is_console_path(path)
&& !Self::EXCLUDED_EXACT_PATHS.contains(&path)
@@ -501,8 +563,44 @@ where
#[cfg(test)]
mod tests {
use super::*;
use http::Request;
use http_body_util::BodyExt;
use http_body_util::Full;
use temp_env::with_var;
#[test]
fn admin_chunked_put_without_content_length_is_normalized() {
let request = Request::builder()
.method(Method::PUT)
.uri("/minio/admin/v3/set-user-status?accessKey=test&status=enabled")
.body(())
.expect("request");
assert!(should_force_zero_content_length_for_admin_empty_body(&request));
}
#[test]
fn admin_request_with_explicit_content_length_is_left_unchanged() {
let request = Request::builder()
.method(Method::PUT)
.uri("/minio/admin/v3/set-group-status?group=test&status=enabled")
.header(http::header::CONTENT_LENGTH, "0")
.body(())
.expect("request");
assert!(!should_force_zero_content_length_for_admin_empty_body(&request));
}
#[test]
fn non_admin_chunked_put_is_not_normalized() {
let request = Request::builder()
.method(Method::PUT)
.uri("/bucket/object")
.body(())
.expect("request");
assert!(!should_force_zero_content_length_for_admin_empty_body(&request));
}
#[test]
fn test_strip_quotes_from_first_etag_removes_quotes() {
@@ -553,6 +651,7 @@ mod tests {
assert!(ConditionalCorsLayer::is_s3_path("/my-bucket/key"));
assert!(ConditionalCorsLayer::is_s3_path("/"));
assert!(!ConditionalCorsLayer::is_s3_path("/rustfs/admin/v3/info"));
assert!(!ConditionalCorsLayer::is_s3_path("/minio/admin/v3/info"));
assert!(!ConditionalCorsLayer::is_s3_path("/health"));
assert!(!ConditionalCorsLayer::is_s3_path("/health/ready"));
}
@@ -594,6 +693,14 @@ mod tests {
);
}
#[test]
fn test_conditional_cors_layer_reads_env() {
with_var("RUSTFS_CORS_ALLOWED_ORIGINS", Some("https://allowed.com"), || {
let cors = ConditionalCorsLayer::new();
assert_eq!(cors.cors_origins.as_deref(), Some("https://allowed.com"));
});
}
#[tokio::test]
async fn test_resolve_s3_options_cors_headers_no_headers_without_match() {
let mut req_headers = HeaderMap::new();
+7
View File
@@ -36,10 +36,17 @@ pub(crate) const HEALTH_READY_PATH: &str = "/health/ready";
/// such as configuration, monitoring, and management.
pub(crate) const ADMIN_PREFIX: &str = "/rustfs/admin";
/// MinIO-compatible administrative prefix accepted by RustFS.
/// This alias allows stock MinIO admin tooling to reach RustFS handlers.
pub(crate) const MINIO_ADMIN_PREFIX: &str = "/minio/admin";
/// Environment variable name for overriding the default
/// administrative prefix path.
pub(crate) const RUSTFS_ADMIN_PREFIX: &str = "/rustfs/admin/v3";
/// MinIO-compatible admin API prefix accepted by RustFS.
pub(crate) const MINIO_ADMIN_V3_PREFIX: &str = "/minio/admin/v3";
/// Predefined console prefix for RustFS server routes.
/// This prefix is used for endpoints that handle console-related tasks
/// such as user interface and management.
+2
View File
@@ -102,9 +102,11 @@ where
// 2) Prefix matching: the entire set of route prefixes (including their subpaths)
let is_prefix_probe = path.starts_with(crate::server::RUSTFS_ADMIN_PREFIX)
|| path.starts_with(crate::server::MINIO_ADMIN_V3_PREFIX)
|| path.starts_with(crate::server::CONSOLE_PREFIX)
|| path.starts_with(crate::server::RPC_PREFIX)
|| path.starts_with(crate::server::ADMIN_PREFIX)
|| path.starts_with(crate::server::MINIO_ADMIN_PREFIX)
|| path.starts_with(crate::server::TONIC_PREFIX);
let is_probe = is_exact_probe || is_prefix_probe;
+1 -20
View File
@@ -23,30 +23,11 @@ use rustfs_ecstore::{
};
use rustfs_s3_common::{S3Operation, record_s3_op};
use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error};
use std::{fmt::Debug, sync::LazyLock};
use std::fmt::Debug;
use tokio::io::{AsyncRead, AsyncSeek};
use tracing::{debug, error, instrument, warn};
use uuid::Uuid;
const DEFAULT_OWNER_ID: &str = "rustfsadmin";
const DEFAULT_OWNER_DISPLAY_NAME: &str = "RustFS Tester";
pub(crate) static RUSTFS_OWNER: LazyLock<Owner> = LazyLock::new(|| Owner {
display_name: Some(DEFAULT_OWNER_DISPLAY_NAME.to_owned()),
id: Some(DEFAULT_OWNER_ID.to_owned()),
});
#[derive(Clone, Debug, Default)]
pub(crate) struct StoredOwner {
pub(crate) id: String,
}
pub(crate) fn default_owner() -> StoredOwner {
StoredOwner {
id: DEFAULT_OWNER_ID.to_string(),
}
}
#[derive(Debug, Clone)]
pub struct FS {
// pub store: ECStore,
+7 -5
View File
@@ -31,7 +31,7 @@ use rustfs_targets::EventName;
use rustfs_targets::arn::{TargetID, TargetIDError};
use rustfs_utils::http::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
RESERVED_METADATA_PREFIX_LOWER,
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, insert_str,
};
use s3s::dto::{
Delimiter, LambdaFunctionConfiguration, NotificationConfigurationFilter, ObjectLockConfiguration, ObjectLockEnabled,
@@ -284,8 +284,9 @@ pub(crate) fn parse_object_lock_retention(retention: Option<ObjectLockRetention>
// This is intentional behavior. Empty string represents "retention cleared" which is different from "retention never set". Consistent with minio
eval_metadata.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), mode);
eval_metadata.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), retain_until_date);
eval_metadata.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-retention-timestamp"),
insert_str(
&mut eval_metadata,
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
format!("{}.{:09}Z", now.format(&Rfc3339).unwrap(), now.nanosecond()),
);
}
@@ -310,8 +311,9 @@ pub(crate) fn parse_object_lock_legal_hold(legal_hold: Option<ObjectLockLegalHol
let now = OffsetDateTime::now_utc();
// This is intentional behavior. Empty string represents "status cleared" which is different from "status never set".
eval_metadata.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), status);
eval_metadata.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-legalhold-timestamp"),
insert_str(
&mut eval_metadata,
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
format!("{}.{:09}Z", now.format(&Rfc3339).unwrap(), now.nanosecond()),
);
}
+16 -11
View File
@@ -17,7 +17,7 @@ mod tests {
use crate::config::workload_profiles::WorkloadProfile;
use crate::server::cors;
use crate::storage::ecfs::FS;
use crate::storage::ecfs::RUSTFS_OWNER;
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use crate::storage::{
apply_cors_headers, check_preconditions, get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal,
matches_origin_pattern, parse_etag, parse_object_lock_legal_hold, parse_object_lock_retention,
@@ -29,7 +29,10 @@ mod tests {
use rustfs_ecstore::bucket::{metadata::BucketMetadata, metadata_sys};
use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
use rustfs_ecstore::store_api::ObjectInfo;
use rustfs_utils::http::{AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, RESERVED_METADATA_PREFIX_LOWER};
use rustfs_utils::http::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
contains_key_str,
};
use rustfs_zip::CompressionFormat;
use s3s::dto::{
CORSConfiguration, CORSRule, Delimiter, LambdaFunctionConfiguration, ObjectLockLegalHold, ObjectLockLegalHoldStatus,
@@ -68,11 +71,15 @@ mod tests {
}
#[test]
fn test_rustfs_owner_constant() {
// Test that RUSTFS_OWNER constant is properly defined
assert!(!RUSTFS_OWNER.display_name.as_ref().unwrap().is_empty());
assert!(!RUSTFS_OWNER.id.as_ref().unwrap().is_empty());
assert_eq!(RUSTFS_OWNER.display_name.as_ref().unwrap(), "RustFS Tester");
fn test_rustfs_owner_helpers_are_stable() {
let owner = rustfs_owner();
let initiator = rustfs_initiator();
assert!(!owner.display_name.as_ref().unwrap().is_empty());
assert!(!owner.id.as_ref().unwrap().is_empty());
assert_eq!(owner.display_name.as_deref(), Some("rustfs"));
assert_eq!(initiator.display_name, owner.display_name);
assert_eq!(initiator.id, owner.id);
}
// Note: Most S3 API methods require complex setup with global state, storage backend,
@@ -440,9 +447,7 @@ mod tests {
compliance_metadata.get("x-amz-object-lock-retain-until-date").unwrap(),
"2030-01-01T00:00:00Z"
);
assert!(
compliance_metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-retention-timestamp"))
);
assert!(contains_key_str(&compliance_metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP));
// [3] Normal case: Retention with valid GOVERNANCE mode (future date)
let valid_governance_retention = ObjectLockRetention {
@@ -502,7 +507,7 @@ mod tests {
};
let on_metadata = parse_object_lock_legal_hold(Some(valid_on_legal_hold)).unwrap();
assert_eq!(on_metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).unwrap(), "ON");
assert!(on_metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-legalhold-timestamp")));
assert!(contains_key_str(&on_metadata, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP));
// [3] Normal case: Legal hold with valid OFF status
let valid_off_legal_hold = ObjectLockLegalHold {
+64 -70
View File
@@ -18,7 +18,11 @@ use rustfs_ecstore::error::Result;
use rustfs_ecstore::error::StorageError;
use rustfs_utils::http::AMZ_META_UNENCRYPTED_CONTENT_LENGTH;
use rustfs_utils::http::AMZ_META_UNENCRYPTED_CONTENT_MD5;
use rustfs_utils::http::RUSTFS_FORCE_DELETE;
use rustfs_utils::http::{
SUFFIX_FORCE_DELETE, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_SOURCE_DELETEMARKER,
SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, get_header, insert_header_map,
is_encryption_metadata_key, is_internal_key,
};
use s3s::header::X_AMZ_OBJECT_LOCK_MODE;
use s3s::header::X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE;
@@ -28,14 +32,6 @@ use rustfs_ecstore::store_api::{HTTPPreconditions, HTTPRangeSpec, ObjectOptions}
use rustfs_policy::service_type::ServiceType;
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use rustfs_utils::http::AMZ_CONTENT_SHA256;
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_DELETE_MARKER;
use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_REQUEST;
use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM;
use rustfs_utils::http::RUSTFS_BUCKET_SOURCE_MTIME;
use rustfs_utils::http::RUSTFS_BUCKET_SOURCE_VERSION_ID;
use rustfs_utils::http::RUSTFS_ENCRYPTION_LOWER;
use rustfs_utils::http::RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE;
use rustfs_utils::path::is_dir_object;
use s3s::{S3Result, s3_error};
use std::collections::HashMap;
@@ -48,7 +44,8 @@ use crate::auth::get_query_param;
use crate::auth::get_request_auth_type_with_query;
use crate::auth::is_request_presigned_signature_v4_with_query;
const REPLICATION_REQUEST_TRUE: HeaderValue = HeaderValue::from_static("true");
#[cfg(test)]
use rustfs_utils::http::insert_header;
/// Creates options for deleting an object in a bucket.
pub async fn del_opts(
@@ -62,9 +59,7 @@ pub async fn del_opts(
let version_suspended = BucketVersioningSys::suspended(bucket).await;
let vid = if vid.is_none() {
headers
.get(RUSTFS_BUCKET_SOURCE_VERSION_ID)
.and_then(|v| v.to_str().ok().map(|s| s.to_owned()))
get_header(headers, SUFFIX_SOURCE_VERSION_ID).map(|s| s.into_owned())
} else {
vid
};
@@ -94,9 +89,8 @@ pub async fn del_opts(
StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string())
})?;
opts.delete_prefix = headers
.get(RUSTFS_FORCE_DELETE)
.map(|v| v.to_str().unwrap_or_default() == "true")
opts.delete_prefix = get_header(headers, SUFFIX_FORCE_DELETE)
.map(|v| v.as_ref() == "true")
.unwrap_or_default();
opts.version_id = {
@@ -109,9 +103,8 @@ pub async fn del_opts(
opts.version_suspended = version_suspended;
opts.versioned = versioned;
opts.delete_marker = headers
.get(RUSTFS_BUCKET_REPLICATION_DELETE_MARKER)
.map(|v| v.to_str().unwrap() == "true")
opts.delete_marker = get_header(headers, SUFFIX_SOURCE_DELETEMARKER)
.map(|v| v.as_ref() == "true")
.unwrap_or_default();
fill_conditional_writes_opts_from_header(headers, &mut opts)?;
@@ -214,9 +207,7 @@ pub async fn put_opts(
let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
let vid = if vid.is_none() {
headers
.get(RUSTFS_BUCKET_SOURCE_VERSION_ID)
.and_then(|v| v.to_str().ok().map(|s| s.to_owned()))
get_header(headers, SUFFIX_SOURCE_VERSION_ID).map(|s| s.into_owned())
} else {
vid
};
@@ -252,23 +243,21 @@ pub fn get_complete_multipart_upload_opts(headers: &HeaderMap<HeaderValue>) -> s
let mut user_defined = HashMap::new();
let mut replication_request = false;
if headers.get(RUSTFS_BUCKET_REPLICATION_REQUEST) == Some(&REPLICATION_REQUEST_TRUE) {
if get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") {
replication_request = true;
if let Some(actual_size_str) = headers
.get(RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE)
.and_then(|h| h.to_str().ok())
{
user_defined.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}Actual-Object-Size"), actual_size_str.to_string());
if let Some(actual_size_str) = get_header(headers, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE) {
rustfs_utils::http::insert_str(
&mut user_defined,
rustfs_utils::http::SUFFIX_ACTUAL_OBJECT_SIZE_CAP,
actual_size_str.into_owned(),
);
} else {
tracing::warn!("Failed to get or parse {} header", RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE);
tracing::warn!("Failed to get or parse replication actual object size header (x-rustfs-* or x-minio-*)");
}
}
if let Some(v) = headers.get(RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM) {
user_defined.insert(
RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM.to_string(),
v.to_str().unwrap_or_default().to_owned(),
);
if let Some(v) = get_header(headers, SUFFIX_REPLICATION_SSEC_CRC) {
insert_header_map(&mut user_defined, SUFFIX_REPLICATION_SSEC_CRC, v.into_owned());
}
let mut opts = ObjectOptions {
@@ -299,26 +288,14 @@ pub fn copy_src_opts(_bucket: &str, _object: &str, headers: &HeaderMap<HeaderVal
pub fn put_opts_from_headers(headers: &HeaderMap<HeaderValue>, metadata: HashMap<String, String>) -> Result<ObjectOptions> {
let mut opts = get_default_opts(headers, metadata, false)?;
if headers.get(RUSTFS_BUCKET_REPLICATION_REQUEST) == Some(&REPLICATION_REQUEST_TRUE) {
if get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") {
opts.replication_request = true;
if let Some(v) = headers.get(RUSTFS_BUCKET_SOURCE_MTIME) {
match v.to_str() {
Ok(s) => {
let trimmed_s = s.trim();
match time::OffsetDateTime::parse(trimmed_s, &time::format_description::well_known::Rfc3339) {
Ok(mtime) => opts.mod_time = Some(mtime),
Err(e) => {
tracing::warn!(
"Invalid X-RustFS-Source-Mtime value '{}' (replication request=true): {}",
trimmed_s,
e
);
opts.mod_time = None;
}
}
}
if let Some(v) = get_header(headers, SUFFIX_SOURCE_MTIME) {
let trimmed_s = v.trim();
match time::OffsetDateTime::parse(trimmed_s, &time::format_description::well_known::Rfc3339) {
Ok(mtime) => opts.mod_time = Some(mtime),
Err(e) => {
tracing::warn!("X-RustFS-Source-Mtime header is not valid UTF-8 (replication request=true): {}", e);
tracing::warn!("Invalid source-mtime value '{}' (replication request=true): {}", trimmed_s, e);
opts.mod_time = None;
}
}
@@ -379,12 +356,17 @@ pub fn extract_metadata_from_mime_with_object_name(
skip_content_type: bool,
object_name: Option<&str>,
) {
const USER_METADATA_PREFIXES: &[&str] = &["x-amz-meta-", "x-rustfs-meta-", "x-minio-meta-"];
for (k, v) in headers.iter() {
if k.as_str() == "content-type" && skip_content_type {
continue;
}
if let Some(key) = k.as_str().strip_prefix("x-amz-meta-") {
if let Some(key) = USER_METADATA_PREFIXES
.iter()
.find_map(|prefix| k.as_str().strip_prefix(prefix))
{
if key.is_empty() {
continue;
}
@@ -393,11 +375,6 @@ pub fn extract_metadata_from_mime_with_object_name(
continue;
}
if let Some(key) = k.as_str().strip_prefix("x-rustfs-meta-") {
metadata.insert(key.to_owned(), String::from_utf8_lossy(v.as_bytes()).to_string());
continue;
}
for hd in SUPPORTED_HEADERS.iter() {
if k.as_str() == *hd {
let raw = String::from_utf8_lossy(v.as_bytes()).to_string();
@@ -446,13 +423,13 @@ pub(crate) fn filter_object_metadata(metadata: &HashMap<String, String>) -> Opti
let mut filtered_metadata = HashMap::new();
for (k, v) in metadata {
let lower_key = k.to_ascii_lowercase();
// Skip internal/reserved metadata
if lower_key.starts_with(RESERVED_METADATA_PREFIX_LOWER) {
// Skip internal/reserved metadata (x-rustfs-internal-* or x-minio-internal-*)
if is_internal_key(&lower_key) {
continue;
}
// Skip internal encryption metadata
if lower_key.starts_with(RUSTFS_ENCRYPTION_LOWER) {
// Skip internal encryption metadata (x-rustfs-encryption-* or x-minio-encryption-*)
if is_encryption_metadata_key(&lower_key) {
continue;
}
@@ -804,28 +781,28 @@ mod tests {
let mut headers = create_test_headers();
let metadata = create_test_metadata();
// Test without RUSTFS_FORCE_DELETE header - should default to false
// Test without force-delete header - should default to false
let result = del_opts("test-bucket", "test-object", None, &headers, metadata.clone()).await;
assert!(result.is_ok());
let opts = result.unwrap();
assert!(!opts.delete_prefix);
// Test with RUSTFS_FORCE_DELETE header set to "true"
headers.insert(RUSTFS_FORCE_DELETE, HeaderValue::from_static("true"));
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
let result = del_opts("test-bucket", "test-object", None, &headers, metadata.clone()).await;
assert!(result.is_ok());
let opts = result.unwrap();
assert!(opts.delete_prefix);
// Test with RUSTFS_FORCE_DELETE header set to "false"
headers.insert(RUSTFS_FORCE_DELETE, HeaderValue::from_static("false"));
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "false");
let result = del_opts("test-bucket", "test-object", None, &headers, metadata.clone()).await;
assert!(result.is_ok());
let opts = result.unwrap();
assert!(!opts.delete_prefix);
// Test with RUSTFS_FORCE_DELETE header set to other value
headers.insert(RUSTFS_FORCE_DELETE, HeaderValue::from_static("maybe"));
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "maybe");
let result = del_opts("test-bucket", "test-object", None, &headers, metadata).await;
assert!(result.is_ok());
let opts = result.unwrap();
@@ -994,9 +971,9 @@ mod tests {
#[test]
fn test_put_opts_from_headers_with_replication_request() {
let mut headers = HeaderMap::new();
headers.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE.clone());
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
let valid_mtime = "2024-05-20T10:30:00+08:00";
headers.insert(RUSTFS_BUCKET_SOURCE_MTIME, HeaderValue::from_static(valid_mtime));
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, valid_mtime);
let metadata = HashMap::new();
@@ -1011,8 +988,8 @@ mod tests {
assert_eq!(opts.mod_time, Some(expected_mtime));
let mut headers_invalid_mtime = HeaderMap::new();
headers_invalid_mtime.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE.clone());
headers_invalid_mtime.insert(RUSTFS_BUCKET_SOURCE_MTIME, HeaderValue::from_static("invalid-time"));
insert_header(&mut headers_invalid_mtime, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
insert_header(&mut headers_invalid_mtime, SUFFIX_SOURCE_MTIME, "invalid-time");
let result_invalid = put_opts_from_headers(&headers_invalid_mtime, HashMap::new());
assert!(result_invalid.is_ok());
let opts_invalid = result_invalid.unwrap();
@@ -1090,6 +1067,21 @@ mod tests {
assert_eq!(metadata.get("category"), Some(&"documents".to_string()));
}
#[test]
fn test_extract_metadata_from_mime_minio_meta() {
let mut headers = HeaderMap::new();
headers.insert("x-minio-meta-origin", HeaderValue::from_static("gateway"));
headers.insert("x-minio-meta-source-id", HeaderValue::from_static("abc123"));
headers.insert("x-minio-meta-", HeaderValue::from_static("empty-key"));
let mut metadata = HashMap::new();
extract_metadata_from_mime(&headers, &mut metadata);
assert_eq!(metadata.get("origin"), Some(&"gateway".to_string()));
assert_eq!(metadata.get("source-id"), Some(&"abc123".to_string()));
assert!(!metadata.contains_key(""));
}
#[test]
fn test_extract_metadata_from_mime_supported_headers() {
let mut headers = HeaderMap::new();
@@ -1249,6 +1241,7 @@ mod tests {
headers.insert("content-type", HeaderValue::from_static("application/xml"));
headers.insert("x-amz-meta-version", HeaderValue::from_static("1.0"));
headers.insert("x-rustfs-meta-source", HeaderValue::from_static("upload"));
headers.insert("x-minio-meta-origin", HeaderValue::from_static("replication"));
headers.insert("cache-control", HeaderValue::from_static("public"));
headers.insert("authorization", HeaderValue::from_static("Bearer xyz")); // Should be ignored
@@ -1257,6 +1250,7 @@ mod tests {
assert_eq!(metadata.get("content-type"), Some(&"application/xml".to_string()));
assert_eq!(metadata.get("version"), Some(&"1.0".to_string()));
assert_eq!(metadata.get("source"), Some(&"upload".to_string()));
assert_eq!(metadata.get("origin"), Some(&"replication".to_string()));
assert_eq!(metadata.get("cache-control"), Some(&"public".to_string()));
assert!(!metadata.contains_key("authorization"));
}
+8 -5
View File
@@ -17,7 +17,7 @@ use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
use rustfs_ecstore::store_api::{BucketInfo, ListObjectVersionsInfo, ListObjectsV2Info};
use s3s::dto::{
Bucket, CommonPrefix, DeleteMarkerEntry, EncodingType, ListBucketsOutput, ListObjectVersionsOutput, ListObjectsOutput,
ListObjectsV2Output, Object, ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Owner, Timestamp,
ListObjectsV2Output, Object, ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Timestamp,
};
use s3s::{S3Error, S3ErrorCode};
use tracing::debug;
@@ -265,10 +265,7 @@ pub(crate) fn build_list_objects_v2_output(
};
if fetch_owner {
obj.owner = Some(Owner {
display_name: Some("rustfs".to_owned()),
id: Some("v0.1".to_owned()),
});
obj.owner = Some(rustfs_owner());
}
obj
@@ -482,10 +479,16 @@ mod tests {
let contents = output.contents.as_ref().expect("contents should exist");
let common_prefixes = output.common_prefixes.as_ref().expect("common prefixes should exist");
let expected_owner = rustfs_owner();
assert_eq!(contents[0].key.as_deref(), Some("dir%20a/file%2Bb%25.txt"));
assert_eq!(common_prefixes[0].prefix.as_deref(), Some("prefix%20a/sub%2B"));
assert!(contents[0].owner.is_some());
assert_eq!(
contents[0].owner.as_ref().and_then(|owner| owner.display_name.clone()),
expected_owner.display_name
);
assert_eq!(contents[0].owner.as_ref().and_then(|owner| owner.id.clone()), expected_owner.id);
assert_eq!(output.encoding_type.as_ref().map(EncodingType::as_str), Some(EncodingType::URL));
}
+3 -2
View File
@@ -184,6 +184,7 @@ mod tests {
build_list_multipart_uploads_output, build_list_parts_output, parse_list_multipart_uploads_params,
parse_list_parts_params,
};
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
use rustfs_ecstore::set_disk::MAX_PARTS_COUNT;
use rustfs_ecstore::store_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo};
@@ -226,8 +227,8 @@ mod tests {
assert_eq!(parts[0].part_number, Some(1));
assert_eq!(parts[0].size, Some(11));
assert_eq!(parts[0].e_tag, Some(to_s3s_etag("etag-1")));
assert!(output.owner.is_some());
assert!(output.initiator.is_some());
assert_eq!(output.owner, Some(rustfs_owner()));
assert_eq!(output.initiator, Some(rustfs_initiator()));
}
#[test]
+3 -2
View File
@@ -88,6 +88,7 @@ use rustfs_kms::{
types::{EncryptionMetadata, ObjectEncryptionContext},
};
use rustfs_rio::{DecryptReader, EncryptReader, HardLimitReader, Reader, WarpReader};
use rustfs_utils::get_env_opt_str;
use s3s::S3ErrorCode;
use s3s::dto::ServerSideEncryption;
use std::collections::HashMap;
@@ -1397,8 +1398,8 @@ impl TestSseDekProvider {
/// Uses RUSTFS_SSE_S3_MASTER_KEY (base64 32-byte) if set; otherwise a built-in default.
/// Allows PUT/GET to work without KMS (backward compatible).
pub fn new_for_local_sse() -> Self {
let master_key = match std::env::var("RUSTFS_SSE_S3_MASTER_KEY") {
Ok(v) if !v.trim().is_empty() => match BASE64_STANDARD.decode(v.trim()) {
let master_key = match get_env_opt_str("RUSTFS_SSE_S3_MASTER_KEY") {
Some(v) if !v.trim().is_empty() => match BASE64_STANDARD.decode(v.trim()) {
Ok(decoded) if decoded.len() == 32 => {
let mut arr = [0u8; 32];
arr.copy_from_slice(&decoded[..32]);