mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-06 13:27:43 +00:00
feat: improve legacy metadata and admin compatibility (#2202)
This commit is contained in:
@@ -49,6 +49,7 @@ rustfs-appauth = { workspace = true }
|
||||
rustfs-audit = { workspace = true }
|
||||
rustfs-common = { workspace = true }
|
||||
rustfs-config = { workspace = true, features = ["constants", "notify"] }
|
||||
rustfs-crypto = { workspace = true }
|
||||
rustfs-credentials = { workspace = true }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-filemeta.workspace = true
|
||||
@@ -168,6 +169,7 @@ aws-sdk-s3 = { workspace = true }
|
||||
aws-config = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
temp-env = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
http.workspace = true
|
||||
|
||||
@@ -142,7 +142,7 @@ impl Config {
|
||||
Config {
|
||||
port,
|
||||
api: Api {
|
||||
base_url: format!("{http_prefix}{local_ip}:{port}/{RUSTFS_ADMIN_PREFIX}"),
|
||||
base_url: build_console_api_base_url(&format!("{http_prefix}{local_ip}:{port}")),
|
||||
},
|
||||
s3: S3 {
|
||||
endpoint: format!("{http_prefix}{local_ip}:{port}"),
|
||||
@@ -192,6 +192,10 @@ impl Config {
|
||||
}
|
||||
}
|
||||
|
||||
fn build_console_api_base_url(base_url: &str) -> String {
|
||||
format!("{base_url}{RUSTFS_ADMIN_PREFIX}")
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Clone)]
|
||||
struct Api {
|
||||
#[serde(rename = "baseURL")]
|
||||
@@ -350,7 +354,7 @@ async fn config_handler(uri: Uri, headers: HeaderMap) -> impl IntoResponse {
|
||||
};
|
||||
|
||||
let url = format!("{}://{}:{}", scheme, host_for_url, cfg.port);
|
||||
cfg.api.base_url = format!("{url}{RUSTFS_ADMIN_PREFIX}");
|
||||
cfg.api.base_url = build_console_api_base_url(&url);
|
||||
cfg.s3.endpoint = url;
|
||||
|
||||
Response::builder()
|
||||
@@ -646,3 +650,38 @@ pub(crate) fn make_console_server() -> Router {
|
||||
// Build console router with enhanced middleware stack using tower-http features
|
||||
setup_console_middleware_stack(cors_layer, rate_limit_enable, rate_limit_rpm, auth_timeout)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::net::{IpAddr, Ipv4Addr};
|
||||
|
||||
#[test]
|
||||
fn console_api_base_url_keeps_rustfs_admin_prefix() {
|
||||
let cfg = Config::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001, "test", "2026-03-16T00:00:00Z");
|
||||
|
||||
assert!(
|
||||
cfg.api.base_url.ends_with("/rustfs/admin/v3"),
|
||||
"console baseURL must keep using the RustFS admin prefix"
|
||||
);
|
||||
assert!(
|
||||
!cfg.api.base_url.ends_with("/minio/admin/v3"),
|
||||
"console baseURL must not switch to the MinIO admin prefix by default"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn console_api_base_url_builder_preserves_existing_console_contract() {
|
||||
assert_eq!(
|
||||
build_console_api_base_url("http://127.0.0.1:9001"),
|
||||
"http://127.0.0.1:9001/rustfs/admin/v3"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_admin_paths_are_not_console_paths() {
|
||||
assert!(is_console_path("/rustfs/console/"));
|
||||
assert!(!is_console_path("/minio/admin/v3/info"));
|
||||
assert!(!is_console_path("/rustfs/admin/v3/info"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::config::Opt;
|
||||
use clap::Parser;
|
||||
use serial_test::serial;
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::{
|
||||
admin::{
|
||||
auth::validate_admin_request,
|
||||
router::{AdminOperation, Operation, S3Router},
|
||||
utils::has_space_be,
|
||||
utils::{encode_compatible_admin_payload, has_space_be, read_compatible_admin_body},
|
||||
},
|
||||
auth::{check_key_valid, get_session_token},
|
||||
server::{ADMIN_PREFIX, RemoteAddr},
|
||||
@@ -28,6 +28,7 @@ use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
|
||||
use rustfs_credentials::get_global_action_cred;
|
||||
use rustfs_iam::error::is_err_no_such_user;
|
||||
use rustfs_iam::store::MappedPolicy;
|
||||
use rustfs_madmin::{GroupPolicyEntities, PolicyEntities, PolicyEntitiesResult, UserPolicyEntities};
|
||||
use rustfs_policy::policy::{
|
||||
Policy,
|
||||
action::{Action, AdminAction},
|
||||
@@ -37,10 +38,12 @@ use s3s::{
|
||||
header::{CONTENT_LENGTH, CONTENT_TYPE},
|
||||
s3_error,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_urlencoded::from_bytes;
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::warn;
|
||||
use url::form_urlencoded;
|
||||
|
||||
pub fn register_iam_policy_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
|
||||
r.insert(
|
||||
@@ -72,6 +75,26 @@ pub fn register_iam_policy_route(r: &mut S3Router<AdminOperation>) -> std::io::R
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/set-user-or-group-policy").as_str(),
|
||||
AdminOperation(&SetPolicyForUserOrGroup {}),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::PUT,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/set-policy").as_str(),
|
||||
AdminOperation(&SetPolicyForUserOrGroup {}),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/idp/builtin/policy/attach").as_str(),
|
||||
AdminOperation(&AttachPolicyBuiltin {}),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/idp/builtin/policy/detach").as_str(),
|
||||
AdminOperation(&DetachPolicyBuiltin {}),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::GET,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/idp/builtin/policy-entities").as_str(),
|
||||
AdminOperation(&ListPolicyEntitiesBuiltin {}),
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -323,11 +346,11 @@ impl Operation for RemoveCannedPolicy {
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
pub struct SetPolicyForUserOrGroupQuery {
|
||||
#[serde(rename = "policyName")]
|
||||
#[serde(rename = "policyName", alias = "policy")]
|
||||
pub policy_name: String,
|
||||
#[serde(rename = "userOrGroup")]
|
||||
#[serde(rename = "userOrGroup", alias = "user-or-group")]
|
||||
pub user_or_group: String,
|
||||
#[serde(rename = "isGroup")]
|
||||
#[serde(rename = "isGroup", alias = "is-group")]
|
||||
pub is_group: bool,
|
||||
}
|
||||
|
||||
@@ -419,3 +442,552 @@ impl Operation for SetPolicyForUserOrGroup {
|
||||
Ok(S3Response::with_headers((StatusCode::OK, Body::empty()), header))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct PolicyAssociationReq {
|
||||
#[serde(default)]
|
||||
policies: Vec<String>,
|
||||
#[serde(default)]
|
||||
user: String,
|
||||
#[serde(default)]
|
||||
group: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct PolicyAssociationResp {
|
||||
#[serde(rename = "policiesAttached", skip_serializing_if = "Vec::is_empty")]
|
||||
policies_attached: Vec<String>,
|
||||
#[serde(rename = "policiesDetached", skip_serializing_if = "Vec::is_empty")]
|
||||
policies_detached: Vec<String>,
|
||||
#[serde(rename = "updatedAt", with = "time::serde::rfc3339")]
|
||||
updated_at: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct PolicyEntitiesQuery {
|
||||
users: Vec<String>,
|
||||
groups: Vec<String>,
|
||||
policies: Vec<String>,
|
||||
}
|
||||
|
||||
fn split_policy_names(policy_names: &str) -> Vec<String> {
|
||||
policy_names
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|name| !name.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn attach_policy_names(existing: &[String], requested: &[String]) -> (Vec<String>, Vec<String>) {
|
||||
let mut updated = existing.to_vec();
|
||||
let mut attached = Vec::new();
|
||||
|
||||
for policy in requested {
|
||||
if updated.iter().any(|current| current == policy) {
|
||||
continue;
|
||||
}
|
||||
updated.push(policy.clone());
|
||||
attached.push(policy.clone());
|
||||
}
|
||||
|
||||
(updated, attached)
|
||||
}
|
||||
|
||||
fn detach_policy_names(existing: &[String], requested: &[String]) -> (Vec<String>, Vec<String>) {
|
||||
let mut detached = Vec::new();
|
||||
let updated = existing
|
||||
.iter()
|
||||
.filter(|policy| {
|
||||
let should_detach = requested.iter().any(|requested_policy| requested_policy == *policy);
|
||||
if should_detach {
|
||||
detached.push((*policy).clone());
|
||||
}
|
||||
!should_detach
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
(updated, detached)
|
||||
}
|
||||
|
||||
fn validate_policy_association_req(req: &PolicyAssociationReq) -> S3Result<()> {
|
||||
if req.policies.is_empty() {
|
||||
return Err(s3_error!(InvalidArgument, "no policy names were given"));
|
||||
}
|
||||
|
||||
if req.policies.iter().any(|policy| policy.is_empty()) {
|
||||
return Err(s3_error!(InvalidArgument, "an empty policy name was given"));
|
||||
}
|
||||
|
||||
let has_user = !req.user.is_empty();
|
||||
let has_group = !req.group.is_empty();
|
||||
|
||||
match (has_user, has_group) {
|
||||
(false, false) => Err(s3_error!(InvalidArgument, "no user or group association was given")),
|
||||
(true, true) => Err(s3_error!(InvalidArgument, "either a group or a user association must be given, not both")),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_policy_entities_query(query: Option<&str>) -> PolicyEntitiesQuery {
|
||||
let mut parsed = PolicyEntitiesQuery::default();
|
||||
let Some(query) = query else {
|
||||
return parsed;
|
||||
};
|
||||
|
||||
for (key, value) in form_urlencoded::parse(query.as_bytes()) {
|
||||
match key.as_ref() {
|
||||
"user" => parsed.users.push(value.into_owned()),
|
||||
"group" => parsed.groups.push(value.into_owned()),
|
||||
"policy" => parsed.policies.push(value.into_owned()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
parsed
|
||||
}
|
||||
|
||||
fn split_policy_list(policy_names: Option<&String>) -> Vec<String> {
|
||||
policy_names.map_or_else(Vec::new, |names| split_policy_names(names))
|
||||
}
|
||||
|
||||
fn direct_user_policy_names(user_info: &rustfs_madmin::UserInfo) -> Vec<String> {
|
||||
split_policy_list(user_info.policy_name.as_ref())
|
||||
}
|
||||
|
||||
fn sorted_group_policy_entities(mut entities: Vec<GroupPolicyEntities>) -> Vec<GroupPolicyEntities> {
|
||||
entities.sort_by(|left, right| left.group.cmp(&right.group));
|
||||
entities
|
||||
}
|
||||
|
||||
fn build_policy_mappings(
|
||||
user_mappings: &[UserPolicyEntities],
|
||||
group_mappings: &[GroupPolicyEntities],
|
||||
requested_policies: &[String],
|
||||
) -> Vec<PolicyEntities> {
|
||||
let mut policy_map: HashMap<String, PolicyEntities> = HashMap::new();
|
||||
|
||||
for user_mapping in user_mappings {
|
||||
for policy in &user_mapping.policies {
|
||||
let entry = policy_map.entry(policy.clone()).or_insert_with(|| PolicyEntities {
|
||||
policy: policy.clone(),
|
||||
..Default::default()
|
||||
});
|
||||
entry.users.push(user_mapping.user.clone());
|
||||
}
|
||||
}
|
||||
|
||||
for group_mapping in group_mappings {
|
||||
for policy in &group_mapping.policies {
|
||||
let entry = policy_map.entry(policy.clone()).or_insert_with(|| PolicyEntities {
|
||||
policy: policy.clone(),
|
||||
..Default::default()
|
||||
});
|
||||
entry.groups.push(group_mapping.group.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let mut results: Vec<PolicyEntities> = policy_map
|
||||
.into_iter()
|
||||
.filter_map(|(_, mut mapping)| {
|
||||
if !requested_policies.is_empty() && !requested_policies.iter().any(|policy| policy == &mapping.policy) {
|
||||
return None;
|
||||
}
|
||||
mapping.users.sort();
|
||||
mapping.users.dedup();
|
||||
mapping.groups.sort();
|
||||
mapping.groups.dedup();
|
||||
Some(mapping)
|
||||
})
|
||||
.collect();
|
||||
results.sort_by(|left, right| left.policy.cmp(&right.policy));
|
||||
results
|
||||
}
|
||||
|
||||
async fn collect_group_policy_mappings(
|
||||
iam_store: &std::sync::Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>>,
|
||||
requested_groups: &[String],
|
||||
) -> S3Result<HashMap<String, GroupPolicyEntities>> {
|
||||
let mut mappings = HashMap::new();
|
||||
let groups = if requested_groups.is_empty() {
|
||||
iam_store
|
||||
.list_groups_load()
|
||||
.await
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?
|
||||
} else {
|
||||
requested_groups.to_vec()
|
||||
};
|
||||
|
||||
for group in groups {
|
||||
let group_desc = iam_store.get_group_description(&group).await.map_err(|e| {
|
||||
warn!("get group description failed, e: {:?}", e);
|
||||
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
|
||||
})?;
|
||||
let policies = split_policy_names(&group_desc.policy);
|
||||
if policies.is_empty() {
|
||||
continue;
|
||||
}
|
||||
mappings.insert(group.clone(), GroupPolicyEntities { group, policies });
|
||||
}
|
||||
|
||||
Ok(mappings)
|
||||
}
|
||||
|
||||
async fn handle_builtin_policy_entities(req: S3Request<Body>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let Some(input_cred) = req.credentials else {
|
||||
return Err(s3_error!(InvalidRequest, "get cred failed"));
|
||||
};
|
||||
|
||||
let (cred, owner) =
|
||||
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
|
||||
|
||||
validate_admin_request(
|
||||
&req.headers,
|
||||
&cred,
|
||||
owner,
|
||||
false,
|
||||
vec![
|
||||
Action::AdminAction(AdminAction::ListGroupsAdminAction),
|
||||
Action::AdminAction(AdminAction::ListUsersAdminAction),
|
||||
Action::AdminAction(AdminAction::ListUserPoliciesAdminAction),
|
||||
],
|
||||
req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let query = parse_policy_entities_query(req.uri.query());
|
||||
|
||||
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
|
||||
|
||||
let all_group_policy_mappings = collect_group_policy_mappings(&iam_store, &[]).await?;
|
||||
let users = iam_store.list_users().await.map_err(|e| {
|
||||
warn!("list users failed, e: {:?}", e);
|
||||
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
|
||||
})?;
|
||||
|
||||
let all_user_policy_mappings = users
|
||||
.iter()
|
||||
.filter_map(|(user, user_info)| {
|
||||
let policies = split_policy_list(user_info.policy_name.as_ref());
|
||||
if policies.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(UserPolicyEntities {
|
||||
user: user.clone(),
|
||||
policies,
|
||||
member_of_mappings: Vec::new(),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let user_mappings = if query.users.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
let mut mappings = Vec::new();
|
||||
for user in &query.users {
|
||||
let Some(user_info) = users.get(user) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let mut member_of_mappings = user_info
|
||||
.member_of
|
||||
.as_ref()
|
||||
.map(|groups| {
|
||||
groups
|
||||
.iter()
|
||||
.filter_map(|group| all_group_policy_mappings.get(group).cloned())
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
member_of_mappings = sorted_group_policy_entities(member_of_mappings);
|
||||
|
||||
let policies = split_policy_list(user_info.policy_name.as_ref());
|
||||
if policies.is_empty() && member_of_mappings.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
mappings.push(UserPolicyEntities {
|
||||
user: user.clone(),
|
||||
policies,
|
||||
member_of_mappings,
|
||||
});
|
||||
}
|
||||
mappings.sort_by(|left, right| left.user.cmp(&right.user));
|
||||
mappings
|
||||
};
|
||||
|
||||
let mut group_mappings = if query.groups.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
query
|
||||
.groups
|
||||
.iter()
|
||||
.filter_map(|group| all_group_policy_mappings.get(group).cloned())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
group_mappings = sorted_group_policy_entities(group_mappings);
|
||||
|
||||
let policy_mappings = if query.users.is_empty() && query.groups.is_empty() && query.policies.is_empty() {
|
||||
build_policy_mappings(
|
||||
&all_user_policy_mappings,
|
||||
&all_group_policy_mappings.values().cloned().collect::<Vec<_>>(),
|
||||
&[],
|
||||
)
|
||||
} else if !query.policies.is_empty() {
|
||||
build_policy_mappings(
|
||||
&all_user_policy_mappings,
|
||||
&all_group_policy_mappings.values().cloned().collect::<Vec<_>>(),
|
||||
&query.policies,
|
||||
)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
let resp = PolicyEntitiesResult {
|
||||
timestamp: OffsetDateTime::now_utc(),
|
||||
user_mappings,
|
||||
group_mappings,
|
||||
policy_mappings,
|
||||
};
|
||||
|
||||
let req_path = req.uri.path().to_string();
|
||||
let body =
|
||||
serde_json::to_vec(&resp).map_err(|e| s3_error!(InternalError, "marshal policy entities body failed, e: {:?}", e))?;
|
||||
let (body, content_type) = encode_compatible_admin_payload(&req_path, &cred.secret_key, body)?;
|
||||
|
||||
let mut header = HeaderMap::new();
|
||||
header.insert(CONTENT_TYPE, content_type.parse().unwrap());
|
||||
header.insert(CONTENT_LENGTH, body.len().to_string().parse().unwrap());
|
||||
Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), header))
|
||||
}
|
||||
|
||||
async fn handle_builtin_policy_association(req: S3Request<Body>, is_attach: bool) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let Some(input_cred) = req.credentials else {
|
||||
return Err(s3_error!(InvalidRequest, "get cred failed"));
|
||||
};
|
||||
|
||||
let (cred, owner) =
|
||||
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
|
||||
|
||||
validate_admin_request(
|
||||
&req.headers,
|
||||
&cred,
|
||||
owner,
|
||||
false,
|
||||
vec![Action::AdminAction(AdminAction::AttachPolicyAdminAction)],
|
||||
req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let req_path = req.uri.path().to_string();
|
||||
let body = read_compatible_admin_body(req.input, MAX_ADMIN_REQUEST_BODY_SIZE, &req_path, &cred.secret_key).await?;
|
||||
let assoc_req: PolicyAssociationReq = serde_json::from_slice(&body)
|
||||
.map_err(|e| s3_error!(InvalidRequest, "unmarshal policy association body failed, e: {:?}", e))?;
|
||||
validate_policy_association_req(&assoc_req)?;
|
||||
|
||||
let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) };
|
||||
|
||||
let (target_name, is_group, existing_policies) = if !assoc_req.user.is_empty() {
|
||||
match iam_store.is_temp_user(&assoc_req.user).await {
|
||||
Ok((true, _)) => return Err(s3_error!(InvalidArgument, "temp user can't set policy")),
|
||||
Ok((false, _)) => {}
|
||||
Err(err) => {
|
||||
if !is_err_no_such_user(&err) {
|
||||
warn!("is temp user failed, e: {:?}", err);
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, err.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let Some(sys_cred) = get_global_action_cred() else {
|
||||
return Err(s3_error!(InternalError, "get global action cred failed"));
|
||||
};
|
||||
|
||||
if assoc_req.user == sys_cred.access_key {
|
||||
return Err(s3_error!(InvalidArgument, "can't set policy for system user"));
|
||||
}
|
||||
|
||||
if iam_store.get_user(&assoc_req.user).await.is_none() {
|
||||
return Err(s3_error!(InvalidArgument, "user not exist"));
|
||||
}
|
||||
|
||||
let user_info = iam_store.get_user_info(&assoc_req.user).await.map_err(|e| {
|
||||
warn!("get user info failed, e: {:?}", e);
|
||||
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
|
||||
})?;
|
||||
|
||||
(assoc_req.user, false, direct_user_policy_names(&user_info))
|
||||
} else {
|
||||
let group_desc = iam_store.get_group_description(&assoc_req.group).await.map_err(|e| {
|
||||
warn!("get group description failed, e: {:?}", e);
|
||||
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
|
||||
})?;
|
||||
|
||||
(assoc_req.group, true, split_policy_names(&group_desc.policy))
|
||||
};
|
||||
|
||||
let (updated_policies, changed_policies) = if is_attach {
|
||||
attach_policy_names(&existing_policies, &assoc_req.policies)
|
||||
} else {
|
||||
detach_policy_names(&existing_policies, &assoc_req.policies)
|
||||
};
|
||||
|
||||
let updated_at = iam_store
|
||||
.policy_db_set(&target_name, rustfs_iam::store::UserType::Reg, is_group, &updated_policies.join(","))
|
||||
.await
|
||||
.map_err(|e| {
|
||||
warn!("policy db set failed, e: {:?}", e);
|
||||
S3Error::with_message(S3ErrorCode::InternalError, e.to_string())
|
||||
})?;
|
||||
|
||||
let policies_attached = if is_attach { changed_policies.clone() } else { Vec::new() };
|
||||
let policies_detached = if is_attach { Vec::new() } else { changed_policies };
|
||||
|
||||
let resp = PolicyAssociationResp {
|
||||
policies_attached,
|
||||
policies_detached,
|
||||
updated_at,
|
||||
};
|
||||
|
||||
let body = serde_json::to_vec(&resp).map_err(|e| s3_error!(InternalError, "marshal body failed, e: {:?}", e))?;
|
||||
let (body, content_type) = encode_compatible_admin_payload(&req_path, &cred.secret_key, body)?;
|
||||
|
||||
let mut header = HeaderMap::new();
|
||||
header.insert(CONTENT_TYPE, content_type.parse().unwrap());
|
||||
header.insert(CONTENT_LENGTH, body.len().to_string().parse().unwrap());
|
||||
Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), header))
|
||||
}
|
||||
|
||||
pub struct AttachPolicyBuiltin {}
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for AttachPolicyBuiltin {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
handle_builtin_policy_association(req, true).await
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DetachPolicyBuiltin {}
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for DetachPolicyBuiltin {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
handle_builtin_policy_association(req, false).await
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ListPolicyEntitiesBuiltin {}
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for ListPolicyEntitiesBuiltin {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
handle_builtin_policy_entities(req).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
GroupPolicyEntities, PolicyAssociationReq, SetPolicyForUserOrGroupQuery, UserPolicyEntities, attach_policy_names,
|
||||
build_policy_mappings, detach_policy_names, direct_user_policy_names, parse_policy_entities_query,
|
||||
validate_policy_association_req,
|
||||
};
|
||||
use rustfs_madmin::UserInfo;
|
||||
|
||||
#[test]
|
||||
fn set_policy_query_supports_external_parameter_names() {
|
||||
let query: SetPolicyForUserOrGroupQuery =
|
||||
serde_urlencoded::from_str("policy=readwrite&user-or-group=test-user&is-group=true").expect("query should parse");
|
||||
|
||||
assert_eq!(query.policy_name, "readwrite");
|
||||
assert_eq!(query.user_or_group, "test-user");
|
||||
assert!(query.is_group);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_policy_query_supports_rustfs_parameter_names() {
|
||||
let query: SetPolicyForUserOrGroupQuery =
|
||||
serde_urlencoded::from_str("policyName=readwrite&userOrGroup=test-user&isGroup=false").expect("query should parse");
|
||||
|
||||
assert_eq!(query.policy_name, "readwrite");
|
||||
assert_eq!(query.user_or_group, "test-user");
|
||||
assert!(!query.is_group);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_association_req_requires_exactly_one_target() {
|
||||
let err = validate_policy_association_req(&PolicyAssociationReq {
|
||||
policies: vec!["readonly".to_string()],
|
||||
user: "user-a".to_string(),
|
||||
group: "group-a".to_string(),
|
||||
})
|
||||
.expect_err("request should be invalid");
|
||||
|
||||
assert_eq!(*err.code(), s3s::S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attach_policy_names_appends_only_missing_values() {
|
||||
let existing = vec!["readonly".to_string()];
|
||||
let requested = vec!["readonly".to_string(), "writeonly".to_string()];
|
||||
|
||||
let (updated, attached) = attach_policy_names(&existing, &requested);
|
||||
|
||||
assert_eq!(updated, vec!["readonly".to_string(), "writeonly".to_string()]);
|
||||
assert_eq!(attached, vec!["writeonly".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detach_policy_names_removes_only_requested_values() {
|
||||
let existing = vec!["readonly".to_string(), "writeonly".to_string()];
|
||||
let requested = vec!["writeonly".to_string(), "missing".to_string()];
|
||||
|
||||
let (updated, detached) = detach_policy_names(&existing, &requested);
|
||||
|
||||
assert_eq!(updated, vec!["readonly".to_string()]);
|
||||
assert_eq!(detached, vec!["writeonly".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_entities_query_supports_repeated_external_parameters() {
|
||||
let query = parse_policy_entities_query(Some("user=alice&user=bob&group=ops&policy=readonly&policy=writeonly"));
|
||||
|
||||
assert_eq!(query.users, vec!["alice".to_string(), "bob".to_string()]);
|
||||
assert_eq!(query.groups, vec!["ops".to_string()]);
|
||||
assert_eq!(query.policies, vec!["readonly".to_string(), "writeonly".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_policy_mappings_indexes_users_and_groups() {
|
||||
let user_mappings = vec![UserPolicyEntities {
|
||||
user: "alice".to_string(),
|
||||
policies: vec!["readonly".to_string()],
|
||||
member_of_mappings: Vec::new(),
|
||||
}];
|
||||
let group_mappings = vec![GroupPolicyEntities {
|
||||
group: "ops".to_string(),
|
||||
policies: vec!["readonly".to_string(), "writeonly".to_string()],
|
||||
}];
|
||||
|
||||
let mappings = build_policy_mappings(&user_mappings, &group_mappings, &[]);
|
||||
|
||||
assert_eq!(mappings.len(), 2);
|
||||
assert_eq!(mappings[0].policy, "readonly");
|
||||
assert_eq!(mappings[0].users, vec!["alice".to_string()]);
|
||||
assert_eq!(mappings[0].groups, vec!["ops".to_string()]);
|
||||
assert_eq!(mappings[1].policy, "writeonly");
|
||||
assert_eq!(mappings[1].groups, vec!["ops".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn direct_user_policy_names_only_reads_direct_mapping() {
|
||||
let user_info = UserInfo {
|
||||
policy_name: Some("readonly,writeonly".to_string()),
|
||||
member_of: Some(vec!["disabled-group".to_string()]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
direct_user_policy_names(&user_info),
|
||||
vec!["readonly".to_string(), "writeonly".to_string()]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,7 +220,12 @@ impl Operation for StartDecommission {
|
||||
));
|
||||
}
|
||||
|
||||
// TODO: check IsRebalanceStarted
|
||||
if store.is_rebalance_started().await {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::OperationAborted,
|
||||
"Decommission cannot be started, rebalance is already in progress".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let query = {
|
||||
if let Some(query) = req.uri.query() {
|
||||
|
||||
@@ -31,6 +31,7 @@ use serde_json;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, info, warn};
|
||||
use url::form_urlencoded;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct SetBucketQuotaRequest {
|
||||
@@ -39,10 +40,86 @@ pub struct SetBucketQuotaRequest {
|
||||
pub quota_type: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CompatibleBucketQuotaRequest {
|
||||
#[serde(default)]
|
||||
quota: Option<u64>,
|
||||
#[serde(default)]
|
||||
size: Option<u64>,
|
||||
#[serde(default, alias = "quotatype")]
|
||||
quota_type: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct CompatibleBucketQuotaResponse {
|
||||
quota: u64,
|
||||
size: u64,
|
||||
rate: u64,
|
||||
requests: u64,
|
||||
#[serde(skip_serializing_if = "String::is_empty")]
|
||||
quotatype: String,
|
||||
}
|
||||
|
||||
fn default_quota_type() -> String {
|
||||
rustfs_config::QUOTA_TYPE_HARD.to_string()
|
||||
}
|
||||
|
||||
fn is_compat_set_bucket_quota_path(path: &str) -> bool {
|
||||
path.ends_with("/v3/set-bucket-quota")
|
||||
}
|
||||
|
||||
fn is_compat_get_bucket_quota_path(path: &str) -> bool {
|
||||
path.ends_with("/v3/get-bucket-quota")
|
||||
}
|
||||
|
||||
fn bucket_from_params_or_query(params: &Params<'_, '_>, uri: &hyper::Uri) -> String {
|
||||
if let Some(bucket) = params.get("bucket") {
|
||||
return bucket.to_string();
|
||||
}
|
||||
|
||||
if let Some(query) = uri.query() {
|
||||
for (key, value) in form_urlencoded::parse(query.as_bytes()) {
|
||||
if key == "bucket" {
|
||||
return value.into_owned();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn parse_set_bucket_quota_request(body: &[u8]) -> Result<SetBucketQuotaRequest, s3s::S3Error> {
|
||||
if body.is_empty() {
|
||||
return Ok(SetBucketQuotaRequest {
|
||||
quota: None,
|
||||
quota_type: default_quota_type(),
|
||||
});
|
||||
}
|
||||
|
||||
let request: CompatibleBucketQuotaRequest =
|
||||
serde_json::from_slice(body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))?;
|
||||
|
||||
Ok(SetBucketQuotaRequest {
|
||||
quota: request
|
||||
.size
|
||||
.filter(|quota| *quota > 0)
|
||||
.or(request.quota.filter(|quota| *quota > 0)),
|
||||
quota_type: request.quota_type.unwrap_or_else(default_quota_type),
|
||||
})
|
||||
}
|
||||
|
||||
fn compat_bucket_quota_response(quota: &BucketQuota) -> CompatibleBucketQuotaResponse {
|
||||
let size = quota.quota.unwrap_or(0);
|
||||
|
||||
CompatibleBucketQuotaResponse {
|
||||
quota: size,
|
||||
size,
|
||||
rate: 0,
|
||||
requests: 0,
|
||||
quotatype: if size > 0 { "hard".to_string() } else { String::new() },
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct BucketQuotaResponse {
|
||||
pub bucket: String,
|
||||
@@ -105,6 +182,18 @@ async fn current_usage_from_context(bucket: &str) -> u64 {
|
||||
}
|
||||
|
||||
pub fn register_quota_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
|
||||
r.insert(
|
||||
Method::PUT,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/set-bucket-quota").as_str(),
|
||||
AdminOperation(&SetBucketQuotaHandler {}),
|
||||
)?;
|
||||
|
||||
r.insert(
|
||||
Method::GET,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/get-bucket-quota").as_str(),
|
||||
AdminOperation(&GetBucketQuotaHandler {}),
|
||||
)?;
|
||||
|
||||
r.insert(
|
||||
Method::PUT,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/quota/{bucket}").as_str(),
|
||||
@@ -161,7 +250,7 @@ impl Operation for SetBucketQuotaHandler {
|
||||
)
|
||||
.await?;
|
||||
|
||||
let bucket = params.get("bucket").unwrap_or("").to_string();
|
||||
let bucket = bucket_from_params_or_query(¶ms, &req.uri);
|
||||
if bucket.is_empty() {
|
||||
return Err(s3_error!(InvalidRequest, "bucket name is required"));
|
||||
}
|
||||
@@ -172,14 +261,7 @@ impl Operation for SetBucketQuotaHandler {
|
||||
.await
|
||||
.map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
|
||||
|
||||
let request: SetBucketQuotaRequest = if body.is_empty() {
|
||||
SetBucketQuotaRequest {
|
||||
quota: None,
|
||||
quota_type: default_quota_type(),
|
||||
}
|
||||
} else {
|
||||
serde_json::from_slice(&body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))?
|
||||
};
|
||||
let request = parse_set_bucket_quota_request(&body)?;
|
||||
|
||||
if request.quota_type.to_uppercase() != rustfs_config::QUOTA_TYPE_HARD {
|
||||
return Err(s3_error!(InvalidArgument, "{}", rustfs_config::QUOTA_INVALID_TYPE_ERROR_MSG));
|
||||
@@ -199,15 +281,18 @@ impl Operation for SetBucketQuotaHandler {
|
||||
// Get real-time usage from data usage system
|
||||
let current_usage = current_usage_from_context(&bucket).await;
|
||||
|
||||
let response = BucketQuotaResponse {
|
||||
bucket,
|
||||
quota: quota.quota,
|
||||
size: current_usage,
|
||||
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
|
||||
};
|
||||
let json = if is_compat_set_bucket_quota_path(req.uri.path()) {
|
||||
String::new()
|
||||
} else {
|
||||
let response = BucketQuotaResponse {
|
||||
bucket,
|
||||
quota: quota.quota,
|
||||
size: current_usage,
|
||||
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
|
||||
};
|
||||
|
||||
let json =
|
||||
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?;
|
||||
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?
|
||||
};
|
||||
|
||||
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
|
||||
}
|
||||
@@ -226,7 +311,7 @@ impl Operation for GetBucketQuotaHandler {
|
||||
let (cred, owner) =
|
||||
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?;
|
||||
|
||||
let bucket = params.get("bucket").unwrap_or("").to_string();
|
||||
let bucket = bucket_from_params_or_query(¶ms, &req.uri);
|
||||
if bucket.is_empty() {
|
||||
return Err(s3_error!(InvalidRequest, "bucket name is required"));
|
||||
}
|
||||
@@ -254,15 +339,19 @@ impl Operation for GetBucketQuotaHandler {
|
||||
_ => s3_error!(InternalError, "Failed to get quota: {}", e),
|
||||
})?;
|
||||
|
||||
let response = BucketQuotaResponse {
|
||||
bucket,
|
||||
quota: quota.quota,
|
||||
size: current_usage.unwrap_or(0),
|
||||
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
|
||||
};
|
||||
let json = if is_compat_get_bucket_quota_path(req.uri.path()) {
|
||||
serde_json::to_string(&compat_bucket_quota_response("a))
|
||||
.map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?
|
||||
} else {
|
||||
let response = BucketQuotaResponse {
|
||||
bucket,
|
||||
quota: quota.quota,
|
||||
size: current_usage.unwrap_or(0),
|
||||
quota_type: rustfs_config::QUOTA_TYPE_HARD.to_string(),
|
||||
};
|
||||
|
||||
let json =
|
||||
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?;
|
||||
serde_json::to_string(&response).map_err(|e| s3_error!(InternalError, "Failed to serialize response: {}", e))?
|
||||
};
|
||||
|
||||
Ok(S3Response::new((StatusCode::OK, Body::from(json))))
|
||||
}
|
||||
@@ -486,6 +575,34 @@ mod tests {
|
||||
assert_eq!(default_quota_type(), "HARD");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_set_bucket_quota_request_accepts_compat_shape() {
|
||||
let request = parse_set_bucket_quota_request(br#"{"quota":1073741824,"size":1073741824,"quotatype":"hard"}"#)
|
||||
.expect("parse quota request");
|
||||
|
||||
assert_eq!(request.quota, Some(1073741824));
|
||||
assert_eq!(request.quota_type, "hard");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_set_bucket_quota_request_prefers_non_zero_quota_over_zero_size() {
|
||||
let request =
|
||||
parse_set_bucket_quota_request(br#"{"quota":1073741824,"size":0,"quotatype":"hard"}"#).expect("parse quota request");
|
||||
|
||||
assert_eq!(request.quota, Some(1073741824));
|
||||
assert_eq!(request.quota_type, "hard");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compat_bucket_quota_response_uses_external_field_names() {
|
||||
let quota = BucketQuota::new(Some(1024));
|
||||
let json = serde_json::to_string(&compat_bucket_quota_response("a)).expect("serialize");
|
||||
|
||||
assert!(json.contains("\"quota\":1024"));
|
||||
assert!(json.contains("\"size\":1024"));
|
||||
assert!(json.contains("\"quotatype\":\"hard\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_quota_operation_parsing() {
|
||||
let parse_operation = |operation: &str| match operation.to_uppercase().as_str() {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17,7 +17,7 @@ use crate::{
|
||||
admin::{
|
||||
auth::validate_admin_request,
|
||||
router::{AdminOperation, Operation, S3Router},
|
||||
utils::has_space_be,
|
||||
utils::{encode_compatible_admin_payload, has_space_be, read_compatible_admin_body},
|
||||
},
|
||||
auth::{check_key_valid, constant_time_eq, get_session_token},
|
||||
server::RemoteAddr,
|
||||
@@ -28,7 +28,7 @@ use rustfs_config::{MAX_ADMIN_REQUEST_BODY_SIZE, MAX_IAM_IMPORT_SIZE};
|
||||
use rustfs_credentials::{Credentials, get_global_action_cred};
|
||||
use rustfs_iam::{
|
||||
store::{GroupInfo, MappedPolicy, UserType},
|
||||
sys::NewServiceAccountOpts,
|
||||
sys::{NewServiceAccountOpts, UpdateServiceAccountOpts},
|
||||
};
|
||||
use rustfs_madmin::{
|
||||
AccountStatus, AddOrUpdateUserReq, IAMEntities, IAMErrEntities, IAMErrEntity, IAMErrPolicyEntity,
|
||||
@@ -50,7 +50,7 @@ use zip::{ZipArchive, ZipWriter, result::ZipError, write::SimpleFileOptions};
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
pub struct AddUserQuery {
|
||||
#[serde(rename = "accessKey")]
|
||||
#[serde(rename = "accessKey", alias = "access-key")]
|
||||
pub access_key: Option<String>,
|
||||
pub status: Option<String>,
|
||||
}
|
||||
@@ -73,6 +73,30 @@ fn should_check_deny_only(target_access_key: &str, requester: &Credentials) -> b
|
||||
&& !requester.is_service_account()
|
||||
}
|
||||
|
||||
fn should_reject_group_import_name(group_name: &str, group_lookup: &rustfs_iam::error::Error) -> bool {
|
||||
has_space_be(group_name) || !matches!(group_lookup, rustfs_iam::error::Error::NoSuchGroup(_))
|
||||
}
|
||||
|
||||
fn should_restore_group_as_disabled(status: &str) -> bool {
|
||||
status.eq_ignore_ascii_case(rustfs_iam::sys::STATUS_DISABLED)
|
||||
}
|
||||
|
||||
fn imported_service_account_status(status: &str) -> Option<String> {
|
||||
if status.eq_ignore_ascii_case(rustfs_policy::auth::ACCOUNT_OFF)
|
||||
|| status.eq_ignore_ascii_case(rustfs_madmin::AccountStatus::Disabled.as_ref())
|
||||
{
|
||||
return Some(rustfs_policy::auth::ACCOUNT_OFF.to_string());
|
||||
}
|
||||
|
||||
if status.eq_ignore_ascii_case(rustfs_policy::auth::ACCOUNT_ON)
|
||||
|| status.eq_ignore_ascii_case(rustfs_madmin::AccountStatus::Enabled.as_ref())
|
||||
{
|
||||
return Some(rustfs_policy::auth::ACCOUNT_ON.to_string());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub struct AddUser {}
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for AddUser {
|
||||
@@ -100,14 +124,7 @@ impl Operation for AddUser {
|
||||
return Err(s3_error!(InvalidArgument, "access key is empty"));
|
||||
}
|
||||
|
||||
let mut input = req.input;
|
||||
let body = match input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
warn!("get body failed, e: {:?}", e);
|
||||
return Err(s3_error!(InvalidRequest, "get body failed"));
|
||||
}
|
||||
};
|
||||
let body = read_compatible_admin_body(req.input, MAX_ADMIN_REQUEST_BODY_SIZE, req.uri.path(), &cred.secret_key).await?;
|
||||
|
||||
// let body_bytes = decrypt_data(input_cred.secret_key.expose().as_bytes(), &body)
|
||||
// .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, format!("decrypt_data err {}", e)))?;
|
||||
@@ -282,9 +299,10 @@ impl Operation for ListUsers {
|
||||
|
||||
let data = serde_json::to_vec(&users)
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("marshal users err {e}")))?;
|
||||
let (data, content_type) = encode_compatible_admin_payload(req.uri.path(), &cred.secret_key, data)?;
|
||||
|
||||
let mut header = HeaderMap::new();
|
||||
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
|
||||
header.insert(CONTENT_TYPE, content_type.parse().unwrap());
|
||||
|
||||
Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
|
||||
}
|
||||
@@ -435,6 +453,7 @@ const ALL_SVC_ACCTS_FILE: &str = "svcaccts.json";
|
||||
const USER_POLICY_MAPPINGS_FILE: &str = "user_mappings.json";
|
||||
const GROUP_POLICY_MAPPINGS_FILE: &str = "group_mappings.json";
|
||||
const STS_USER_POLICY_MAPPINGS_FILE: &str = "stsuser_mappings.json";
|
||||
const GROUP_POLICY_MAPPING_USER_TYPE: UserType = UserType::Reg;
|
||||
|
||||
const IAM_ASSETS_DIR: &str = "iam-assets";
|
||||
|
||||
@@ -619,7 +638,7 @@ impl Operation for ExportIam {
|
||||
GROUP_POLICY_MAPPINGS_FILE => {
|
||||
let mut group_policy_mappings = HashMap::new();
|
||||
iam_store
|
||||
.load_mapped_policies(UserType::Reg, true, &mut group_policy_mappings)
|
||||
.load_mapped_policies(GROUP_POLICY_MAPPING_USER_TYPE, true, &mut group_policy_mappings)
|
||||
.await
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?;
|
||||
|
||||
@@ -805,7 +824,7 @@ impl Operation for ImportIam {
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?;
|
||||
for (group_name, group_info) in groups {
|
||||
if let Err(e) = iam_store.get_group_description(&group_name).await
|
||||
&& (matches!(e, rustfs_iam::error::Error::NoSuchGroup(_)) || has_space_be(&group_name))
|
||||
&& should_reject_group_import_name(&group_name, &e)
|
||||
{
|
||||
return Err(s3_error!(InvalidArgument, "group not found or has space be"));
|
||||
}
|
||||
@@ -816,6 +835,14 @@ impl Operation for ImportIam {
|
||||
error: e.to_string(),
|
||||
});
|
||||
} else {
|
||||
if should_restore_group_as_disabled(&group_info.status) {
|
||||
iam_store.set_group_status(&group_name, false).await.map_err(|e| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("set group status failed, name: {group_name}, err: {e}"),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
added.groups.push(group_name.clone());
|
||||
}
|
||||
}
|
||||
@@ -893,6 +920,29 @@ impl Operation for ImportIam {
|
||||
error: e.to_string(),
|
||||
});
|
||||
} else {
|
||||
if let Some(status) = imported_service_account_status(&req.status)
|
||||
&& status == rustfs_policy::auth::ACCOUNT_OFF
|
||||
{
|
||||
iam_store
|
||||
.update_service_account(
|
||||
&ak,
|
||||
UpdateServiceAccountOpts {
|
||||
session_policy: None,
|
||||
secret_key: None,
|
||||
name: None,
|
||||
description: None,
|
||||
expiration: None,
|
||||
status: Some(status),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("update service account status failed, name: {ak}, err: {e}"),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
added.service_accounts.push(ak.clone());
|
||||
}
|
||||
}
|
||||
@@ -973,7 +1023,7 @@ impl Operation for ImportIam {
|
||||
}
|
||||
|
||||
if let Err(e) = iam_store
|
||||
.policy_db_set(&group_name, UserType::None, true, &policies.policies)
|
||||
.policy_db_set(&group_name, GROUP_POLICY_MAPPING_USER_TYPE, true, &policies.policies)
|
||||
.await
|
||||
{
|
||||
failed.group_policies.push(IAMErrPolicyEntity {
|
||||
@@ -1064,8 +1114,12 @@ impl Operation for ImportIam {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::should_check_deny_only;
|
||||
use super::{
|
||||
GROUP_POLICY_MAPPING_USER_TYPE, imported_service_account_status, should_check_deny_only, should_reject_group_import_name,
|
||||
should_restore_group_as_disabled,
|
||||
};
|
||||
use rustfs_credentials::{Credentials, IAM_POLICY_CLAIM_NAME_SA};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -1119,4 +1173,40 @@ mod tests {
|
||||
};
|
||||
assert!(!should_check_deny_only("alice", &cred));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_group_import_allows_missing_group_without_spaces() {
|
||||
assert!(!should_reject_group_import_name(
|
||||
"new-group",
|
||||
&IamError::NoSuchGroup("new-group".to_string())
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_group_import_rejects_group_names_with_spaces() {
|
||||
assert!(should_reject_group_import_name(
|
||||
" bad-group",
|
||||
&IamError::NoSuchGroup(" bad-group".to_string())
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_group_import_restores_disabled_status_only_when_needed() {
|
||||
assert!(should_restore_group_as_disabled("disabled"));
|
||||
assert!(!should_restore_group_as_disabled("enabled"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_imported_service_account_status_maps_on_and_off() {
|
||||
assert_eq!(imported_service_account_status("off").as_deref(), Some("off"));
|
||||
assert_eq!(imported_service_account_status("on").as_deref(), Some("on"));
|
||||
assert_eq!(imported_service_account_status("disabled").as_deref(), Some("off"));
|
||||
assert_eq!(imported_service_account_status("enabled").as_deref(), Some("on"));
|
||||
assert!(imported_service_account_status("unknown").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_group_policy_mappings_use_regular_user_type() {
|
||||
assert_eq!(GROUP_POLICY_MAPPING_USER_TYPE, rustfs_iam::store::UserType::Reg);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,13 +17,17 @@ use crate::admin::{
|
||||
router::{AdminOperation, S3Router},
|
||||
rpc,
|
||||
};
|
||||
use crate::server::{ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH};
|
||||
use crate::server::{ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH};
|
||||
use hyper::Method;
|
||||
|
||||
fn admin_path(path: &str) -> String {
|
||||
format!("{}{}", ADMIN_PREFIX, path)
|
||||
}
|
||||
|
||||
fn compat_admin_alias_path(path: &str) -> String {
|
||||
format!("{}{}", MINIO_ADMIN_PREFIX, path)
|
||||
}
|
||||
|
||||
fn assert_route(router: &S3Router<AdminOperation>, method: Method, path: &str) {
|
||||
assert!(
|
||||
router.contains_route(method.clone(), path),
|
||||
@@ -69,9 +73,17 @@ fn test_register_routes_cover_representative_admin_paths() {
|
||||
assert_route(&router, Method::DELETE, &admin_path("/v3/group/test-group"));
|
||||
assert_route(&router, Method::PUT, &admin_path("/v3/update-group-members"));
|
||||
assert_route(&router, Method::PUT, &admin_path("/v3/add-service-accounts"));
|
||||
assert_route(&router, Method::PUT, &admin_path("/v3/add-service-account"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/temporary-account-info"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/info-access-key"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/list-access-keys-bulk"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/export-iam"));
|
||||
assert_route(&router, Method::PUT, &admin_path("/v3/import-iam"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/list-canned-policies"));
|
||||
assert_route(&router, Method::PUT, &admin_path("/v3/set-policy"));
|
||||
assert_route(&router, Method::POST, &admin_path("/v3/idp/builtin/policy/attach"));
|
||||
assert_route(&router, Method::POST, &admin_path("/v3/idp/builtin/policy/detach"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/idp/builtin/policy-entities"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/target/list"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/accountinfo"));
|
||||
|
||||
@@ -89,6 +101,8 @@ fn test_register_routes_cover_representative_admin_paths() {
|
||||
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/tier"));
|
||||
assert_route(&router, Method::POST, &admin_path("/v3/tier/clear"));
|
||||
assert_route(&router, Method::PUT, &admin_path("/v3/set-bucket-quota"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/get-bucket-quota"));
|
||||
assert_route(&router, Method::PUT, &admin_path("/v3/quota/test-bucket"));
|
||||
assert_route(&router, Method::GET, &admin_path("/v3/quota-stats/test-bucket"));
|
||||
|
||||
@@ -107,6 +121,44 @@ fn test_register_routes_cover_representative_admin_paths() {
|
||||
assert_route(&router, Method::HEAD, "/rustfs/rpc/read_file_stream");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_admin_alias_paths_match_existing_admin_routes() {
|
||||
let mut router: S3Router<AdminOperation> = S3Router::new(false);
|
||||
|
||||
health::register_health_route(&mut router).expect("register health route");
|
||||
sts::register_admin_auth_route(&mut router).expect("register sts route");
|
||||
user::register_user_route(&mut router).expect("register user route");
|
||||
system::register_system_route(&mut router).expect("register system route");
|
||||
pools::register_pool_route(&mut router).expect("register pool route");
|
||||
rebalance::register_rebalance_route(&mut router).expect("register rebalance route");
|
||||
quota::register_quota_route(&mut router).expect("register quota route");
|
||||
|
||||
for (method, path) in [
|
||||
(Method::GET, compat_admin_alias_path("/v3/is-admin")),
|
||||
(Method::GET, compat_admin_alias_path("/v3/info")),
|
||||
(Method::GET, compat_admin_alias_path("/v3/storageinfo")),
|
||||
(Method::GET, compat_admin_alias_path("/v3/pools/list")),
|
||||
(Method::PUT, compat_admin_alias_path("/v3/add-service-account")),
|
||||
(Method::GET, compat_admin_alias_path("/v3/temporary-account-info")),
|
||||
(Method::GET, compat_admin_alias_path("/v3/info-access-key")),
|
||||
(Method::GET, compat_admin_alias_path("/v3/list-access-keys-bulk")),
|
||||
(Method::PUT, compat_admin_alias_path("/v3/set-policy")),
|
||||
(Method::PUT, compat_admin_alias_path("/v3/set-bucket-quota")),
|
||||
(Method::GET, compat_admin_alias_path("/v3/get-bucket-quota")),
|
||||
(Method::POST, compat_admin_alias_path("/v3/idp/builtin/policy/attach")),
|
||||
(Method::POST, compat_admin_alias_path("/v3/idp/builtin/policy/detach")),
|
||||
(Method::GET, compat_admin_alias_path("/v3/idp/builtin/policy-entities")),
|
||||
(Method::POST, compat_admin_alias_path("/v3/rebalance/start")),
|
||||
] {
|
||||
assert!(
|
||||
router.contains_compatible_route(method.clone(), &path),
|
||||
"expected MinIO admin alias path to match: {} {}",
|
||||
method.as_str(),
|
||||
path
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_phase5_admin_info_and_rpc_read_file_contract() {
|
||||
let system_src = include_str!("handlers/system.rs");
|
||||
|
||||
@@ -14,7 +14,9 @@
|
||||
|
||||
use crate::admin::console::{is_console_path, make_console_server};
|
||||
use crate::admin::handlers::oidc::is_oidc_path;
|
||||
use crate::server::{ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX};
|
||||
use crate::server::{
|
||||
ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX,
|
||||
};
|
||||
use hyper::HeaderMap;
|
||||
use hyper::Method;
|
||||
use hyper::StatusCode;
|
||||
@@ -43,6 +45,18 @@ fn is_public_health_path(path: &str) -> bool {
|
||||
path == HEALTH_PREFIX || path == HEALTH_READY_PATH
|
||||
}
|
||||
|
||||
fn is_admin_path(path: &str) -> bool {
|
||||
path.starts_with(ADMIN_PREFIX) || path.starts_with(MINIO_ADMIN_PREFIX)
|
||||
}
|
||||
|
||||
fn canonicalize_admin_path(path: &str) -> std::borrow::Cow<'_, str> {
|
||||
if let Some(suffix) = path.strip_prefix(MINIO_ADMIN_PREFIX) {
|
||||
return std::borrow::Cow::Owned(format!("{ADMIN_PREFIX}{suffix}"));
|
||||
}
|
||||
|
||||
std::borrow::Cow::Borrowed(path)
|
||||
}
|
||||
|
||||
impl<T: Operation> S3Router<T> {
|
||||
pub fn new(console_enabled: bool) -> Self {
|
||||
let router = Router::new();
|
||||
@@ -81,6 +95,12 @@ impl<T: Operation> S3Router<T> {
|
||||
let route = Self::make_route_str(method, path);
|
||||
self.router.at(&route).is_ok()
|
||||
}
|
||||
|
||||
pub(crate) fn contains_compatible_route(&self, method: Method, path: &str) -> bool {
|
||||
let canonical_path = canonicalize_admin_path(path);
|
||||
let route = Self::make_route_str(method, canonical_path.as_ref());
|
||||
self.router.at(&route).is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Operation> Default for S3Router<T> {
|
||||
@@ -120,7 +140,7 @@ where
|
||||
return true;
|
||||
}
|
||||
|
||||
path.starts_with(ADMIN_PREFIX) || path.starts_with(RPC_PREFIX) || is_console_path(path)
|
||||
is_admin_path(path) || path.starts_with(RPC_PREFIX) || is_console_path(path)
|
||||
}
|
||||
|
||||
// check_access before call
|
||||
@@ -207,7 +227,8 @@ where
|
||||
return Err(s3_error!(InternalError, "console is not enabled"));
|
||||
}
|
||||
|
||||
let uri = format!("{}|{}", &req.method, req.uri.path());
|
||||
let canonical_path = canonicalize_admin_path(req.uri.path());
|
||||
let uri = format!("{}|{}", &req.method, canonical_path.as_ref());
|
||||
|
||||
if let Ok(mat) = self.router.at(&uri) {
|
||||
let op: &T = mat.value;
|
||||
@@ -238,6 +259,24 @@ impl Operation for AdminOperation {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn canonicalize_admin_path_maps_compat_prefix_to_rustfs_prefix() {
|
||||
assert_eq!(canonicalize_admin_path("/minio/admin/v3/info").as_ref(), "/rustfs/admin/v3/info");
|
||||
assert_eq!(canonicalize_admin_path("/rustfs/admin/v3/info").as_ref(), "/rustfs/admin/v3/info");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_admin_path_accepts_rustfs_and_compat_prefixes() {
|
||||
assert!(is_admin_path("/rustfs/admin/v3/info"));
|
||||
assert!(is_admin_path("/minio/admin/v3/info"));
|
||||
assert!(!is_admin_path("/bucket/object"));
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Extra {
|
||||
|
||||
@@ -12,6 +12,90 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::server::MINIO_ADMIN_PREFIX;
|
||||
use rustfs_crypto::{decrypt_data, decrypt_stream_io, encrypt_stream_io};
|
||||
use s3s::{Body, S3Result, s3_error};
|
||||
|
||||
pub(crate) fn has_space_be(s: &str) -> bool {
|
||||
s.trim().len() != s.len()
|
||||
}
|
||||
|
||||
pub(crate) fn is_compat_admin_request(path: &str) -> bool {
|
||||
path.starts_with(MINIO_ADMIN_PREFIX)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_compatible_admin_body(
|
||||
mut input: Body,
|
||||
max_len: usize,
|
||||
path: &str,
|
||||
secret_key: &str,
|
||||
) -> S3Result<Vec<u8>> {
|
||||
let body = input
|
||||
.store_all_limited(max_len)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
|
||||
|
||||
if is_compat_admin_request(path) {
|
||||
decrypt_stream_io(secret_key.as_bytes(), body.as_ref())
|
||||
.or_else(|_| decrypt_data(secret_key.as_bytes(), body.as_ref()))
|
||||
.map_err(|e| s3_error!(InvalidRequest, "failed to decrypt MinIO admin payload: {}", e))
|
||||
} else {
|
||||
Ok(body.to_vec())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn encode_compatible_admin_payload(path: &str, secret_key: &str, data: Vec<u8>) -> S3Result<(Vec<u8>, &'static str)> {
|
||||
if is_compat_admin_request(path) {
|
||||
let encrypted = encrypt_stream_io(secret_key.as_bytes(), &data)
|
||||
.map_err(|e| s3_error!(InternalError, "failed to encrypt MinIO admin payload: {}", e))?;
|
||||
Ok((encrypted, "application/octet-stream"))
|
||||
} else {
|
||||
Ok((data, "application/json"))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_crypto::encrypt_data;
|
||||
use s3s::Body;
|
||||
|
||||
#[test]
|
||||
fn detects_compat_admin_paths_only_for_external_prefix() {
|
||||
assert!(is_compat_admin_request("/minio/admin/v3/list-users"));
|
||||
assert!(!is_compat_admin_request("/rustfs/admin/v3/list-users"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encodes_plain_payload_for_rustfs_admin_paths() {
|
||||
let payload = b"{\"ok\":true}".to_vec();
|
||||
let (encoded, content_type) =
|
||||
encode_compatible_admin_payload("/rustfs/admin/v3/list-users", "secret", payload.clone()).expect("encode payload");
|
||||
|
||||
assert_eq!(encoded, payload);
|
||||
assert_eq!(content_type, "application/json");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encodes_compat_payload_with_compatible_encryption() {
|
||||
let payload = b"{\"ok\":true}".to_vec();
|
||||
let (encoded, content_type) =
|
||||
encode_compatible_admin_payload("/minio/admin/v3/list-users", "secret", payload.clone()).expect("encode payload");
|
||||
|
||||
assert_ne!(encoded, payload);
|
||||
assert_eq!(content_type, "application/octet-stream");
|
||||
assert_eq!(decrypt_stream_io(b"secret", &encoded).expect("decrypt payload"), payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reads_legacy_compat_payload_as_fallback() {
|
||||
let payload = b"{\"ok\":true}".to_vec();
|
||||
let encrypted = encrypt_data(b"secret", &payload).expect("encrypt payload");
|
||||
|
||||
let decoded = read_compatible_admin_body(Body::from(encrypted), 1024, "/minio/admin/v3/list-users", "secret")
|
||||
.await
|
||||
.expect("decode payload");
|
||||
|
||||
assert_eq!(decoded, payload);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,8 +19,8 @@ use crate::auth::get_condition_values;
|
||||
use crate::error::ApiError;
|
||||
use crate::server::RemoteAddr;
|
||||
use crate::storage::access::{ReqInfo, authorize_request, req_info_ref};
|
||||
use crate::storage::ecfs::{RUSTFS_OWNER, default_owner};
|
||||
use crate::storage::helper::OperationHelper;
|
||||
use crate::storage::s3_api::bucket::{build_list_buckets_output, build_list_objects_v2_output};
|
||||
use crate::storage::s3_api::{acl, encryption, replication, tagging};
|
||||
use crate::storage::*;
|
||||
use futures::StreamExt;
|
||||
@@ -51,7 +51,7 @@ use rustfs_targets::{
|
||||
EventName,
|
||||
arn::{ARN, TargetIDError},
|
||||
};
|
||||
use rustfs_utils::http::RUSTFS_FORCE_DELETE;
|
||||
use rustfs_utils::http::{SUFFIX_FORCE_DELETE, get_header};
|
||||
use rustfs_utils::string::parse_bool;
|
||||
use s3s::dto::*;
|
||||
use s3s::region::Region;
|
||||
@@ -59,7 +59,6 @@ use s3s::xml;
|
||||
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use std::{collections::HashSet, fmt::Display, sync::Arc};
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
use urlencoding::encode;
|
||||
|
||||
fn serialize_config<T: xml::Serialize>(value: &T) -> S3Result<Vec<u8>> {
|
||||
serialize(value).map_err(to_internal_error)
|
||||
@@ -69,6 +68,17 @@ fn to_internal_error(err: impl Display) -> S3Error {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, format!("{err}"))
|
||||
}
|
||||
|
||||
fn create_bucket_exists_response(is_owner: bool) -> S3Result<S3Response<CreateBucketOutput>> {
|
||||
if is_owner {
|
||||
return Ok(S3Response::new(CreateBucketOutput::default()));
|
||||
}
|
||||
|
||||
Err(s3_error!(
|
||||
BucketAlreadyExists,
|
||||
"The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again."
|
||||
))
|
||||
}
|
||||
|
||||
fn resolve_notification_region(global_region: Option<Region>, request_region: Option<Region>) -> String {
|
||||
global_region
|
||||
.or(request_region)
|
||||
@@ -147,8 +157,8 @@ impl DefaultBucketUsecase {
|
||||
}
|
||||
|
||||
let helper = OperationHelper::new(&req, EventName::BucketCreated, S3Operation::CreateBucket);
|
||||
let requester_id = match req_info_ref(&req) {
|
||||
Ok(r) => r.cred.as_ref().map(|c| c.access_key.clone()),
|
||||
let requester_is_owner = match req_info_ref(&req) {
|
||||
Ok(r) => r.is_owner,
|
||||
Err(_) => {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Missing request info".to_string()));
|
||||
}
|
||||
@@ -179,18 +189,7 @@ impl DefaultBucketUsecase {
|
||||
Err(StorageError::BucketExists(_)) => {
|
||||
// Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK;
|
||||
// non-owner gets 409 BucketAlreadyExists.
|
||||
let is_owner = requester_id.as_deref().is_some_and(|req_id| req_id == default_owner().id);
|
||||
|
||||
if is_owner {
|
||||
let output = CreateBucketOutput::default();
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
return result;
|
||||
}
|
||||
let result = Err(s3_error!(
|
||||
BucketAlreadyExists,
|
||||
"The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again."
|
||||
));
|
||||
let result = create_bucket_exists_response(requester_is_owner);
|
||||
let _ = helper.complete(&result);
|
||||
return result;
|
||||
}
|
||||
@@ -247,19 +246,11 @@ impl DefaultBucketUsecase {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
// get value from header, support mc style
|
||||
let force_str = req
|
||||
.headers
|
||||
.get(RUSTFS_FORCE_DELETE)
|
||||
.map(|v| v.to_str().unwrap_or_default())
|
||||
.unwrap_or(
|
||||
req.headers
|
||||
.get("x-minio-force-delete")
|
||||
.map(|v| v.to_str().unwrap_or_default())
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
let force_str = get_header(&req.headers, SUFFIX_FORCE_DELETE)
|
||||
.map(|v| v.into_owned())
|
||||
.unwrap_or_default();
|
||||
|
||||
let force = parse_bool(force_str).unwrap_or_default();
|
||||
let force = parse_bool(&force_str).unwrap_or_default();
|
||||
|
||||
if force {
|
||||
authorize_request(&mut req, Action::S3Action(S3Action::ForceDeleteBucketAction)).await?;
|
||||
@@ -404,20 +395,7 @@ impl DefaultBucketUsecase {
|
||||
store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?
|
||||
};
|
||||
|
||||
let buckets: Vec<Bucket> = bucket_infos
|
||||
.iter()
|
||||
.map(|v| Bucket {
|
||||
creation_date: v.created.map(Timestamp::from),
|
||||
name: Some(v.name.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(S3Response::new(ListBucketsOutput {
|
||||
buckets: Some(buckets),
|
||||
owner: Some(RUSTFS_OWNER.to_owned()),
|
||||
..Default::default()
|
||||
}))
|
||||
Ok(S3Response::new(build_list_buckets_output(&bucket_infos)))
|
||||
}
|
||||
|
||||
pub async fn execute_delete_bucket_encryption(
|
||||
@@ -1413,10 +1391,9 @@ impl DefaultBucketUsecase {
|
||||
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
|
||||
let incl_deleted = req
|
||||
.headers
|
||||
.get(rustfs_utils::http::headers::RUSTFS_INCLUDE_DELETED)
|
||||
.is_some_and(|v| v.to_str().unwrap_or_default() == "true");
|
||||
let incl_deleted = rustfs_utils::http::get_header(&req.headers, rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
|
||||
.map(|v| v.as_ref() == "true")
|
||||
.unwrap_or_default();
|
||||
|
||||
let object_infos = store
|
||||
.list_objects_v2(
|
||||
@@ -1432,84 +1409,18 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
// warn!("object_infos objects {:?}", object_infos.objects);
|
||||
|
||||
// Apply URL encoding if encoding_type is "url"
|
||||
// Note: S3 URL encoding should encode special characters but preserve path separators (/)
|
||||
let should_encode = encoding_type.as_ref().map(|e| e.as_str() == "url").unwrap_or(false);
|
||||
|
||||
// Helper function to encode S3 keys/prefixes (preserving /)
|
||||
// S3 URL encoding encodes special characters but keeps '/' unencoded
|
||||
let encode_s3_name = |name: &str| -> String {
|
||||
name.split('/')
|
||||
.map(|part| encode(part).to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join("/")
|
||||
};
|
||||
|
||||
let objects: Vec<Object> = object_infos
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|v| !v.name.is_empty())
|
||||
.map(|v| {
|
||||
let key = if should_encode {
|
||||
encode_s3_name(&v.name)
|
||||
} else {
|
||||
v.name.to_owned()
|
||||
};
|
||||
let mut obj = Object {
|
||||
key: Some(key),
|
||||
last_modified: v.mod_time.map(Timestamp::from),
|
||||
size: Some(v.get_actual_size().unwrap_or_default()),
|
||||
e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)),
|
||||
storage_class: v.storage_class.clone().map(ObjectStorageClass::from),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if fetch_owner.is_some_and(|v| v) {
|
||||
obj.owner = Some(Owner {
|
||||
display_name: Some("rustfs".to_owned()),
|
||||
id: Some("v0.1".to_owned()),
|
||||
});
|
||||
}
|
||||
obj
|
||||
})
|
||||
.collect();
|
||||
|
||||
let common_prefixes: Vec<CommonPrefix> = object_infos
|
||||
.prefixes
|
||||
.into_iter()
|
||||
.map(|v| {
|
||||
let prefix = if should_encode { encode_s3_name(&v) } else { v };
|
||||
CommonPrefix { prefix: Some(prefix) }
|
||||
})
|
||||
.collect();
|
||||
|
||||
// KeyCount should include both objects and common prefixes per S3 API spec
|
||||
let key_count = (objects.len() + common_prefixes.len()) as i32;
|
||||
|
||||
// Encode next_continuation_token to base64
|
||||
let next_continuation_token = object_infos
|
||||
.next_continuation_token
|
||||
.map(|token| base64_simd::STANDARD.encode_to_string(token.as_bytes()));
|
||||
|
||||
let output = ListObjectsV2Output {
|
||||
is_truncated: Some(object_infos.is_truncated),
|
||||
continuation_token: response_continuation_token,
|
||||
next_continuation_token,
|
||||
start_after: response_start_after,
|
||||
key_count: Some(key_count),
|
||||
max_keys: Some(max_keys),
|
||||
contents: Some(objects),
|
||||
let output = build_list_objects_v2_output(
|
||||
object_infos,
|
||||
fetch_owner.unwrap_or_default(),
|
||||
max_keys,
|
||||
bucket,
|
||||
prefix,
|
||||
delimiter,
|
||||
encoding_type: encoding_type.clone(),
|
||||
name: Some(bucket),
|
||||
prefix: Some(prefix),
|
||||
common_prefixes: Some(common_prefixes),
|
||||
..Default::default()
|
||||
};
|
||||
encoding_type,
|
||||
response_continuation_token,
|
||||
response_start_after,
|
||||
);
|
||||
|
||||
// let output = ListObjectsV2Output { ..Default::default() };
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
@@ -1679,6 +1590,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn build_request_with_req_info<T>(input: T, method: Method, req_info: ReqInfo) -> S3Request<T> {
|
||||
let mut req = build_request(input, method);
|
||||
req.extensions.insert(req_info);
|
||||
req
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_notification_region_prefers_global_region() {
|
||||
let binding = resolve_notification_region(Some("us-east-1".parse().unwrap()), Some("ap-southeast-1".parse().unwrap()));
|
||||
@@ -1697,6 +1614,36 @@ mod tests {
|
||||
assert_eq!(binding, RUSTFS_REGION);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_bucket_exists_response_returns_ok_for_owner() {
|
||||
let response = create_bucket_exists_response(true).expect("owner recreate should succeed");
|
||||
assert_eq!(response.output.location, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_bucket_exists_response_returns_bucket_already_exists_for_non_owner() {
|
||||
let err = create_bucket_exists_response(false).expect_err("non-owner recreate should fail");
|
||||
assert_eq!(err.code(), &S3ErrorCode::BucketAlreadyExists);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_request_with_req_info_preserves_owner_state() {
|
||||
let input = CreateBucketInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request_with_req_info(
|
||||
input,
|
||||
Method::PUT,
|
||||
ReqInfo {
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert!(req_info_ref(&req).expect("req info should be present").is_owner);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_create_bucket_returns_internal_error_when_store_uninitialized() {
|
||||
let input = CreateBucketInput::builder()
|
||||
@@ -1886,6 +1833,7 @@ mod tests {
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: None,
|
||||
noncurrent_version_expiration: None,
|
||||
@@ -1900,6 +1848,7 @@ mod tests {
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some("rule-1".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
@@ -1914,6 +1863,7 @@ mod tests {
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: None,
|
||||
noncurrent_version_expiration: None,
|
||||
@@ -1939,6 +1889,7 @@ mod tests {
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: None,
|
||||
noncurrent_version_expiration: None,
|
||||
|
||||
@@ -17,17 +17,17 @@
|
||||
use crate::app::context::{AppContext, get_global_app_context};
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::concurrency::get_concurrency_manager;
|
||||
use crate::storage::ecfs::RUSTFS_OWNER;
|
||||
use crate::storage::entity;
|
||||
use crate::storage::helper::OperationHelper;
|
||||
use crate::storage::options::{
|
||||
copy_src_opts, extract_metadata, get_complete_multipart_upload_opts, get_content_sha256_with_query, parse_copy_source_range,
|
||||
put_opts,
|
||||
};
|
||||
use crate::storage::s3_api::multipart::build_list_parts_output;
|
||||
use crate::storage::*;
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use rustfs_config::RUSTFS_REGION;
|
||||
use http::{HeaderMap, Uri};
|
||||
use rustfs_ecstore::bucket::quota::checker::QuotaChecker;
|
||||
use rustfs_ecstore::bucket::{
|
||||
metadata_sys,
|
||||
@@ -47,18 +47,18 @@ use rustfs_s3_common::S3Operation;
|
||||
use rustfs_targets::EventName;
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_CHECKSUM_TYPE,
|
||||
headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER},
|
||||
AMZ_CHECKSUM_TYPE, get_source_scheme,
|
||||
headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING},
|
||||
};
|
||||
use s3s::dto::*;
|
||||
use s3s::region::Region;
|
||||
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
use tokio_util::io::StreamReader;
|
||||
use tracing::{info, instrument, warn};
|
||||
use urlencoding::encode;
|
||||
|
||||
/// Returns InvalidRange error if CopySourceRange end exceeds the source object size.
|
||||
/// Used by execute_upload_part_copy to reject out-of-bounds ranges per S3 spec.
|
||||
@@ -72,6 +72,74 @@ fn validate_copy_source_range_not_exceeds(range_spec: &HTTPRangeSpec, object_siz
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_complete_multipart_parts(parts: &[CompletePart]) -> S3Result<()> {
|
||||
if parts.windows(2).any(|window| window[0].part_num >= window[1].part_num) {
|
||||
return Err(s3_error!(InvalidPartOrder, "Part numbers must be strictly increasing"));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn normalize_complete_multipart_parts(parts: Vec<CompletePart>) -> S3Result<Vec<CompletePart>> {
|
||||
// For duplicate part numbers, keep the last occurrence from the request.
|
||||
// This matches retry/resend semantics where later uploads override earlier ones.
|
||||
let mut seen = HashSet::with_capacity(parts.len());
|
||||
let mut deduped_reversed = Vec::with_capacity(parts.len());
|
||||
for part in parts.into_iter().rev() {
|
||||
if seen.insert(part.part_num) {
|
||||
deduped_reversed.push(part);
|
||||
}
|
||||
}
|
||||
deduped_reversed.reverse();
|
||||
|
||||
validate_complete_multipart_parts(&deduped_reversed)?;
|
||||
Ok(deduped_reversed)
|
||||
}
|
||||
|
||||
fn encode_s3_path(path: &str) -> String {
|
||||
path.split('/')
|
||||
.map(|part| encode(part).to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join("/")
|
||||
}
|
||||
|
||||
fn extract_request_scheme(headers: &HeaderMap, uri: &Uri) -> String {
|
||||
get_source_scheme(headers)
|
||||
.and_then(|value| {
|
||||
value
|
||||
.split(',')
|
||||
.next()
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
})
|
||||
.or_else(|| uri.scheme_str().map(str::to_owned))
|
||||
.unwrap_or_else(|| "http".to_string())
|
||||
.to_ascii_lowercase()
|
||||
}
|
||||
|
||||
fn extract_request_host(headers: &HeaderMap, uri: &Uri) -> Option<String> {
|
||||
headers
|
||||
.get(http::header::HOST)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
.or_else(|| uri.authority().map(|authority| authority.as_str().to_string()))
|
||||
}
|
||||
|
||||
fn build_complete_multipart_location(headers: &HeaderMap, uri: &Uri, bucket: &str, key: &str) -> String {
|
||||
let object_path = format!("/{}/{}", encode(bucket), encode_s3_path(key));
|
||||
|
||||
match extract_request_host(headers, uri) {
|
||||
Some(host) => {
|
||||
let scheme = extract_request_scheme(headers, uri);
|
||||
format!("{scheme}://{host}{object_path}")
|
||||
}
|
||||
None => object_path,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct DefaultMultipartUsecase {
|
||||
context: Option<Arc<AppContext>>,
|
||||
@@ -93,10 +161,6 @@ impl DefaultMultipartUsecase {
|
||||
self.context.as_ref().and_then(|context| context.bucket_metadata().handle())
|
||||
}
|
||||
|
||||
fn global_region(&self) -> Option<Region> {
|
||||
self.context.as_ref().and_then(|context| context.region().get())
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
pub async fn execute_abort_multipart_upload(
|
||||
&self,
|
||||
@@ -213,23 +277,7 @@ impl DefaultMultipartUsecase {
|
||||
.map(CompletePart::from)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// is part number sorted?
|
||||
if !uploaded_parts_vec.is_sorted_by_key(|p| p.part_num) {
|
||||
return Err(s3_error!(InvalidPart, "Part numbers must be sorted"));
|
||||
}
|
||||
|
||||
// Handle duplicate part numbers: according to S3 specification, when the same part number
|
||||
// is uploaded multiple times, the last uploaded part (in the list order) should be used.
|
||||
// This can happen in concurrent upload scenarios where a part is re-uploaded before completion.
|
||||
// We deduplicate by keeping the last occurrence of each part number using a HashMap.
|
||||
let mut part_map: HashMap<usize, CompletePart> = HashMap::new();
|
||||
for part in uploaded_parts_vec {
|
||||
part_map.insert(part.part_num, part);
|
||||
}
|
||||
|
||||
// Reconstruct the parts list in sorted order, keeping only the last occurrence of each part number
|
||||
let mut uploaded_parts: Vec<CompletePart> = part_map.into_values().collect();
|
||||
uploaded_parts.sort_by_key(|p| p.part_num);
|
||||
let uploaded_parts = normalize_complete_multipart_parts(uploaded_parts_vec)?;
|
||||
|
||||
// TODO: check object lock
|
||||
|
||||
@@ -354,15 +402,12 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
let region = self
|
||||
.global_region()
|
||||
.map(|region| region.to_string())
|
||||
.unwrap_or_else(|| RUSTFS_REGION.to_string());
|
||||
let location = build_complete_multipart_location(&req.headers, &req.uri, &bucket, &key);
|
||||
let output = CompleteMultipartUploadOutput {
|
||||
bucket: Some(bucket.clone()),
|
||||
key: Some(key.clone()),
|
||||
e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)),
|
||||
location: Some(region.clone()),
|
||||
location: Some(location.clone()),
|
||||
server_side_encryption: server_side_encryption.clone(),
|
||||
ssekms_key_id: ssekms_key_id.clone(),
|
||||
checksum_crc32: checksum_crc32.clone(),
|
||||
@@ -378,7 +423,7 @@ impl DefaultMultipartUsecase {
|
||||
bucket: Some(bucket.clone()),
|
||||
key: Some(key.clone()),
|
||||
e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)),
|
||||
location: Some(region),
|
||||
location: Some(location),
|
||||
server_side_encryption,
|
||||
ssekms_key_id,
|
||||
checksum_crc32,
|
||||
@@ -482,8 +527,9 @@ impl DefaultMultipartUsecase {
|
||||
};
|
||||
|
||||
if is_compressible(&req.headers, &key) {
|
||||
metadata.insert(
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}compression"),
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
CompressionAlgorithm::default().to_string(),
|
||||
);
|
||||
}
|
||||
@@ -603,9 +649,7 @@ impl DefaultMultipartUsecase {
|
||||
StreamReader::new(body_stream.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))),
|
||||
);
|
||||
|
||||
let is_compressible = fi
|
||||
.user_defined
|
||||
.contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}compression").as_str());
|
||||
let is_compressible = rustfs_utils::http::contains_key_str(&fi.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
|
||||
|
||||
let mut reader: Box<dyn Reader> = Box::new(WarpReader::new(body));
|
||||
|
||||
@@ -857,39 +901,7 @@ impl DefaultMultipartUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let output = ListPartsOutput {
|
||||
bucket: Some(res.bucket),
|
||||
key: Some(res.object),
|
||||
upload_id: Some(res.upload_id),
|
||||
parts: Some(
|
||||
res.parts
|
||||
.into_iter()
|
||||
.map(|p| Part {
|
||||
e_tag: p.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
last_modified: p.last_mod.map(Timestamp::from),
|
||||
part_number: Some(p.part_num as i32),
|
||||
size: Some(p.size as i64),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
owner: Some(RUSTFS_OWNER.to_owned()),
|
||||
initiator: Some(Initiator {
|
||||
id: RUSTFS_OWNER.id.clone(),
|
||||
display_name: RUSTFS_OWNER.display_name.clone(),
|
||||
}),
|
||||
is_truncated: Some(res.is_truncated),
|
||||
next_part_number_marker: res.next_part_number_marker.try_into().ok(),
|
||||
max_parts: res.max_parts.try_into().ok(),
|
||||
part_number_marker: res.part_number_marker.try_into().ok(),
|
||||
storage_class: if res.storage_class.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(res.storage_class.into())
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
Ok(S3Response::new(output))
|
||||
Ok(S3Response::new(build_list_parts_output(res)))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
@@ -1014,9 +1026,7 @@ impl DefaultMultipartUsecase {
|
||||
.map_err(ApiError::from)?;
|
||||
let src_stream = src_reader.stream;
|
||||
|
||||
let is_compressible = mp_info
|
||||
.user_defined
|
||||
.contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}compression").as_str());
|
||||
let is_compressible = rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
|
||||
|
||||
let mut reader: Box<dyn Reader> = Box::new(WarpReader::new(src_stream));
|
||||
|
||||
@@ -1129,7 +1139,7 @@ impl DefaultMultipartUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::{Extensions, HeaderMap, Method, Uri};
|
||||
use http::{Extensions, HeaderMap, Method, Uri, header::HeaderValue};
|
||||
|
||||
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
@@ -1149,6 +1159,41 @@ mod tests {
|
||||
DefaultMultipartUsecase::without_context()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_complete_multipart_location_uses_forwarded_proto_and_encodes_key() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(http::header::HOST, HeaderValue::from_static("storage.example.com:9000"));
|
||||
headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
|
||||
|
||||
let location = build_complete_multipart_location(
|
||||
&headers,
|
||||
&Uri::from_static("/bucket/object?uploadId=1"),
|
||||
"bucket",
|
||||
"dir/file name.txt",
|
||||
);
|
||||
|
||||
assert_eq!(location, "https://storage.example.com:9000/bucket/dir/file%20name.txt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_complete_multipart_location_falls_back_to_uri_authority_and_scheme() {
|
||||
let location = build_complete_multipart_location(
|
||||
&HeaderMap::new(),
|
||||
&"https://gateway.example.com:9443/complete".parse::<Uri>().unwrap(),
|
||||
"bucket",
|
||||
"object.txt",
|
||||
);
|
||||
|
||||
assert_eq!(location, "https://gateway.example.com:9443/bucket/object.txt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_complete_multipart_location_returns_path_without_host() {
|
||||
let location = build_complete_multipart_location(&HeaderMap::new(), &Uri::from_static("/"), "bucket", "nested/object");
|
||||
|
||||
assert_eq!(location, "/bucket/nested/object");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized() {
|
||||
let input = AbortMultipartUploadInput::builder()
|
||||
@@ -1191,6 +1236,81 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidPart);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_complete_multipart_upload_allows_duplicate_part_numbers_by_using_last_occurrence() {
|
||||
let multipart_upload = CompletedMultipartUpload {
|
||||
parts: Some(vec![
|
||||
CompletedPart {
|
||||
part_number: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
CompletedPart {
|
||||
part_number: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
]),
|
||||
};
|
||||
let input = CompleteMultipartUploadInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.multipart_upload(Some(multipart_upload))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = make_usecase().execute_complete_multipart_upload(req).await.unwrap_err();
|
||||
assert_ne!(err.code(), &S3ErrorCode::InvalidPartOrder);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_complete_multipart_upload_rejects_out_of_order_parts() {
|
||||
let multipart_upload = CompletedMultipartUpload {
|
||||
parts: Some(vec![
|
||||
CompletedPart {
|
||||
part_number: Some(2),
|
||||
..Default::default()
|
||||
},
|
||||
CompletedPart {
|
||||
part_number: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
]),
|
||||
};
|
||||
let input = CompleteMultipartUploadInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.multipart_upload(Some(multipart_upload))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = make_usecase().execute_complete_multipart_upload(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidPartOrder);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_complete_multipart_parts_keeps_last_duplicate_part() {
|
||||
let input = vec![
|
||||
CompletePart {
|
||||
part_num: 1,
|
||||
etag: Some("old".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
CompletePart {
|
||||
part_num: 1,
|
||||
etag: Some("new".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
|
||||
let normalized = normalize_complete_multipart_parts(input).expect("normalization should succeed");
|
||||
assert_eq!(normalized.len(), 1);
|
||||
assert_eq!(normalized[0].part_num, 1);
|
||||
assert_eq!(normalized[0].etag.as_deref(), Some("new"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_multipart_uploads_returns_internal_error_when_store_uninitialized() {
|
||||
let input = ListMultipartUploadsInput::builder()
|
||||
|
||||
@@ -82,13 +82,14 @@ use rustfs_s3select_api::{
|
||||
use rustfs_s3select_query::get_global_db;
|
||||
use rustfs_targets::EventName;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, RESERVED_METADATA_PREFIX,
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION,
|
||||
SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP,
|
||||
headers::{
|
||||
AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE,
|
||||
AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
|
||||
AMZ_OBJECT_TAGGING, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT,
|
||||
RESERVED_METADATA_PREFIX_LOWER,
|
||||
},
|
||||
insert_str, remove_str,
|
||||
};
|
||||
use rustfs_utils::path::{is_dir_object, path_join_buf};
|
||||
use rustfs_utils::{
|
||||
@@ -426,9 +427,8 @@ impl DefaultObjectUsecase {
|
||||
|
||||
if is_compressible(&req.headers, &key) && size > MIN_COMPRESSIBLE_SIZE as i64 {
|
||||
let algorithm = CompressionAlgorithm::default();
|
||||
metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), algorithm.to_string());
|
||||
|
||||
metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size",), size.to_string());
|
||||
insert_str(&mut metadata, SUFFIX_COMPRESSION, algorithm.to_string());
|
||||
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
|
||||
|
||||
let mut hrd = HashReader::new(reader, size as i64, size as i64, md5hex, sha256hex, false).map_err(ApiError::from)?;
|
||||
|
||||
@@ -437,10 +437,8 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
|
||||
opts.want_checksum = hrd.checksum();
|
||||
opts.user_defined
|
||||
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), algorithm.to_string());
|
||||
opts.user_defined
|
||||
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size",), size.to_string());
|
||||
insert_str(&mut opts.user_defined, SUFFIX_COMPRESSION, algorithm.to_string());
|
||||
insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string());
|
||||
|
||||
reader = Box::new(CompressReader::new(hrd, algorithm));
|
||||
size = HashReader::SIZE_PRESERVE_LAYER;
|
||||
@@ -497,10 +495,12 @@ impl DefaultObjectUsecase {
|
||||
let dsc = must_replicate(&bucket, &key, repoptions).await;
|
||||
|
||||
if dsc.replicate_any() {
|
||||
let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp");
|
||||
opts.user_defined.insert(k, jiff::Zoned::now().to_string());
|
||||
let k = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status");
|
||||
opts.user_defined.insert(k, dsc.pending_status().unwrap_or_default());
|
||||
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
|
||||
insert_str(
|
||||
&mut opts.user_defined,
|
||||
SUFFIX_REPLICATION_STATUS,
|
||||
dsc.pending_status().unwrap_or_default(),
|
||||
);
|
||||
}
|
||||
|
||||
let obj_info = store
|
||||
@@ -2104,11 +2104,8 @@ impl DefaultObjectUsecase {
|
||||
let mut compress_metadata = HashMap::new();
|
||||
|
||||
if is_compressible(&req.headers, &key) && actual_size > MIN_COMPRESSIBLE_SIZE as i64 {
|
||||
compress_metadata.insert(
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}compression"),
|
||||
CompressionAlgorithm::default().to_string(),
|
||||
);
|
||||
compress_metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size",), actual_size.to_string());
|
||||
insert_str(&mut compress_metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string());
|
||||
insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
|
||||
|
||||
let hrd = EtagReader::new(reader, None);
|
||||
|
||||
@@ -2117,24 +2114,9 @@ impl DefaultObjectUsecase {
|
||||
reader = Box::new(CompressReader::new(hrd, CompressionAlgorithm::default()));
|
||||
length = HashReader::SIZE_PRESERVE_LAYER;
|
||||
} else {
|
||||
src_info
|
||||
.user_defined
|
||||
.remove(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression"));
|
||||
src_info
|
||||
.user_defined
|
||||
.remove(&format!("{RESERVED_METADATA_PREFIX}compression"));
|
||||
src_info
|
||||
.user_defined
|
||||
.remove(&format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"));
|
||||
src_info
|
||||
.user_defined
|
||||
.remove(&format!("{RESERVED_METADATA_PREFIX}actual-size"));
|
||||
src_info
|
||||
.user_defined
|
||||
.remove(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression-size"));
|
||||
src_info
|
||||
.user_defined
|
||||
.remove(&format!("{RESERVED_METADATA_PREFIX}compression-size"));
|
||||
remove_str(&mut src_info.user_defined, SUFFIX_COMPRESSION);
|
||||
remove_str(&mut src_info.user_defined, SUFFIX_ACTUAL_SIZE);
|
||||
remove_str(&mut src_info.user_defined, SUFFIX_COMPRESSION_SIZE);
|
||||
}
|
||||
|
||||
// Handle MetadataDirective REPLACE: replace user metadata while preserving system metadata.
|
||||
@@ -3480,11 +3462,8 @@ impl DefaultObjectUsecase {
|
||||
let actual_size = size;
|
||||
|
||||
if is_compressible(&HeaderMap::new(), &fpath) && size > MIN_COMPRESSIBLE_SIZE as i64 {
|
||||
metadata.insert(
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}compression"),
|
||||
CompressionAlgorithm::default().to_string(),
|
||||
);
|
||||
metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"), size.to_string());
|
||||
insert_str(&mut metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string());
|
||||
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
|
||||
|
||||
let hrd = HashReader::new(reader, size, actual_size, None, None, false).map_err(ApiError::from)?;
|
||||
|
||||
|
||||
@@ -15,8 +15,7 @@
|
||||
#[cfg(test)]
|
||||
#[allow(unsafe_op_in_unsafe_fn)]
|
||||
mod tests {
|
||||
use crate::config::Opt;
|
||||
use clap::Parser;
|
||||
use crate::config::{Config, Opt};
|
||||
use rustfs_ecstore::disks_layout::DisksLayout;
|
||||
use serial_test::serial;
|
||||
use std::env;
|
||||
@@ -56,6 +55,19 @@ mod tests {
|
||||
verify_fn(&layout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_server_subcommand_and_legacy_equivalence() {
|
||||
// rustfs server <volume> and rustfs <volume> (legacy) must produce identical results
|
||||
let legacy_args = vec!["rustfs", "/data/vol1"];
|
||||
let server_args = vec!["rustfs", "server", "/data/vol1"];
|
||||
let opt_legacy = Opt::parse_from(legacy_args);
|
||||
let opt_server = Opt::parse_from(server_args);
|
||||
assert_eq!(opt_legacy.volumes, opt_server.volumes);
|
||||
assert_eq!(opt_legacy.address, opt_server.address);
|
||||
assert_eq!(opt_legacy.console_address, opt_server.console_address);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_default_console_configuration() {
|
||||
@@ -109,6 +121,64 @@ mod tests {
|
||||
assert_eq!(console_port, 9001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_external_prefixed_envs_are_accepted_by_parser() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
("MINIO_VOLUMES", Some("/compat/vol1")),
|
||||
("MINIO_ADDRESS", Some(":9100")),
|
||||
("RUSTFS_VOLUMES", None),
|
||||
("RUSTFS_ADDRESS", None),
|
||||
],
|
||||
|| {
|
||||
let opt = Opt::parse_from(["rustfs"]);
|
||||
assert_eq!(opt.volumes, vec!["/compat/vol1"]);
|
||||
assert_eq!(opt.address, ":9100");
|
||||
assert_eq!(std::env::var("RUSTFS_VOLUMES").as_deref(), Ok("/compat/vol1"));
|
||||
assert_eq!(std::env::var("RUSTFS_ADDRESS").as_deref(), Ok(":9100"));
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_root_envs_are_used_for_bootstrap_credentials() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_VOLUMES", Some("/compat/vol1")),
|
||||
("RUSTFS_ROOT_USER", Some("root-user")),
|
||||
("RUSTFS_ROOT_PASSWORD", Some("root-password")),
|
||||
("RUSTFS_ACCESS_KEY", None),
|
||||
("RUSTFS_SECRET_KEY", None),
|
||||
],
|
||||
|| {
|
||||
let config = Config::from_opt(Opt::parse_from(["rustfs"])).expect("config should parse");
|
||||
assert_eq!(config.access_key, "root-user");
|
||||
assert_eq!(config.secret_key, "root-password");
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_access_key_env_takes_precedence_over_root_aliases() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_VOLUMES", Some("/compat/vol1")),
|
||||
("RUSTFS_ACCESS_KEY", Some("canonical-access")),
|
||||
("RUSTFS_SECRET_KEY", Some("canonical-secret")),
|
||||
("RUSTFS_ROOT_USER", Some("root-user")),
|
||||
("RUSTFS_ROOT_PASSWORD", Some("root-password")),
|
||||
],
|
||||
|| {
|
||||
let config = Config::from_opt(Opt::parse_from(["rustfs"])).expect("config should parse");
|
||||
assert_eq!(config.access_key, "canonical-access");
|
||||
assert_eq!(config.secret_key, "canonical-secret");
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_volumes_and_disk_layout_parsing() {
|
||||
|
||||
+146
-9
@@ -12,10 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use clap::Parser;
|
||||
use clap::builder::NonEmptyStringValueParser;
|
||||
use clap::{Args, Parser, Subcommand};
|
||||
use const_str::concat;
|
||||
use rustfs_config::RUSTFS_REGION;
|
||||
use rustfs_utils::{apply_external_env_compat, get_env_opt_str};
|
||||
use std::path::PathBuf;
|
||||
use std::string::ToString;
|
||||
|
||||
@@ -50,9 +51,47 @@ const LONG_VERSION: &str = concat!(
|
||||
concat!("git status :\n", build::GIT_STATUS_FILE),
|
||||
);
|
||||
|
||||
/// Known subcommands. When the first arg matches one of these, it is treated as a subcommand.
|
||||
const KNOWN_SUBCOMMANDS: &[&str] = &["server"];
|
||||
|
||||
/// Preprocess argv for legacy compatibility: `rustfs <volume>` and `rustfs --address ...` are
|
||||
/// treated as `rustfs server <volume>` and `rustfs server --address ...` respectively.
|
||||
/// Also: `rustfs` with no args becomes `rustfs server` (volumes from env).
|
||||
fn preprocess_args_for_legacy(args: Vec<String>) -> Vec<String> {
|
||||
if args.len() < 2 {
|
||||
// rustfs -> rustfs server (volumes from RUSTFS_VOLUMES env)
|
||||
return vec![args[0].clone(), "server".to_string()];
|
||||
}
|
||||
let first = &args[1];
|
||||
// If first arg looks like a subcommand, do nothing
|
||||
if KNOWN_SUBCOMMANDS.contains(&first.as_str()) {
|
||||
return args;
|
||||
}
|
||||
// If first arg is a global flag (--help, --version), do nothing
|
||||
if first == "--help" || first == "-h" || first == "--version" || first == "-V" {
|
||||
return args;
|
||||
}
|
||||
// Legacy: rustfs <volume> or rustfs --address ... -> rustfs server <volume|--address ...>
|
||||
let mut out = vec![args[0].clone(), "server".to_string()];
|
||||
out.extend(args[1..].iter().cloned());
|
||||
out
|
||||
}
|
||||
|
||||
#[derive(Parser, Clone)]
|
||||
#[command(version = SHORT_VERSION, long_version = LONG_VERSION)]
|
||||
pub struct Opt {
|
||||
#[command(name = "rustfs", version = SHORT_VERSION, long_version = LONG_VERSION)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Option<Commands>,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Clone)]
|
||||
enum Commands {
|
||||
/// Start the object storage server (default when no subcommand is given)
|
||||
Server(ServerOpts),
|
||||
}
|
||||
|
||||
#[derive(Args, Clone)]
|
||||
struct ServerOpts {
|
||||
/// DIR points to a directory on a filesystem.
|
||||
#[arg(
|
||||
required = true,
|
||||
@@ -164,6 +203,96 @@ pub struct Opt {
|
||||
pub buffer_profile: String,
|
||||
}
|
||||
|
||||
/// Parsed server options. Public for tests and backward compatibility.
|
||||
/// Use `Opt::parse_from` or `Config::parse()` to obtain.
|
||||
#[derive(Clone)]
|
||||
pub struct Opt {
|
||||
pub volumes: Vec<String>,
|
||||
pub address: String,
|
||||
pub server_domains: Vec<String>,
|
||||
pub access_key: Option<String>,
|
||||
pub access_key_file: Option<PathBuf>,
|
||||
pub secret_key: Option<String>,
|
||||
pub secret_key_file: Option<PathBuf>,
|
||||
pub console_enable: bool,
|
||||
pub console_address: String,
|
||||
pub obs_endpoint: String,
|
||||
pub tls_path: Option<String>,
|
||||
pub license: Option<String>,
|
||||
pub region: Option<String>,
|
||||
pub kms_enable: bool,
|
||||
pub kms_backend: String,
|
||||
pub kms_key_dir: Option<String>,
|
||||
pub kms_vault_address: Option<String>,
|
||||
pub kms_vault_token: Option<String>,
|
||||
pub kms_default_key_id: Option<String>,
|
||||
pub buffer_profile_disable: bool,
|
||||
pub buffer_profile: String,
|
||||
}
|
||||
|
||||
impl Opt {
|
||||
fn from_server_opts(o: ServerOpts) -> Self {
|
||||
Self {
|
||||
volumes: o.volumes,
|
||||
address: o.address,
|
||||
server_domains: o.server_domains,
|
||||
access_key: o.access_key,
|
||||
access_key_file: o.access_key_file,
|
||||
secret_key: o.secret_key,
|
||||
secret_key_file: o.secret_key_file,
|
||||
console_enable: o.console_enable,
|
||||
console_address: o.console_address,
|
||||
obs_endpoint: o.obs_endpoint,
|
||||
tls_path: o.tls_path,
|
||||
license: o.license,
|
||||
region: o.region,
|
||||
kms_enable: o.kms_enable,
|
||||
kms_backend: o.kms_backend,
|
||||
kms_key_dir: o.kms_key_dir,
|
||||
kms_vault_address: o.kms_vault_address,
|
||||
kms_vault_token: o.kms_vault_token,
|
||||
kms_default_key_id: o.kms_default_key_id,
|
||||
buffer_profile_disable: o.buffer_profile_disable,
|
||||
buffer_profile: o.buffer_profile,
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse from preprocessed args. Supports both `rustfs <volume>` and `rustfs server <volume>`.
|
||||
pub fn parse_from<I, T>(args: I) -> Self
|
||||
where
|
||||
I: IntoIterator<Item = T>,
|
||||
T: Into<std::ffi::OsString> + Clone,
|
||||
{
|
||||
let _ = apply_external_env_compat();
|
||||
let args: Vec<String> = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect();
|
||||
let args = preprocess_args_for_legacy(args);
|
||||
let cli = Cli::parse_from(args);
|
||||
let Commands::Server(opts) = cli.command.expect("server is the default subcommand");
|
||||
Self::from_server_opts(opts)
|
||||
}
|
||||
|
||||
/// Try parse from args, returns error on invalid input.
|
||||
#[allow(dead_code)] // used in config_test
|
||||
pub fn try_parse_from<I, T>(args: I) -> Result<Self, clap::Error>
|
||||
where
|
||||
I: IntoIterator<Item = T>,
|
||||
T: Into<std::ffi::OsString> + Clone,
|
||||
{
|
||||
let _ = apply_external_env_compat();
|
||||
let args: Vec<String> = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect();
|
||||
let args = preprocess_args_for_legacy(args);
|
||||
let cli = Cli::try_parse_from(args)?;
|
||||
let Commands::Server(opts) = cli.command.expect("server is the default subcommand");
|
||||
Ok(Self::from_server_opts(opts))
|
||||
}
|
||||
|
||||
/// Parse from env::args(). Used by Config::parse().
|
||||
fn parse() -> Self {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
Self::parse_from(args)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Config {
|
||||
/// DIR points to a directory on a filesystem.
|
||||
@@ -227,11 +356,7 @@ pub struct Config {
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// parse the command line arguments and environment arguments from [`Opt`] and convert them
|
||||
/// into a ready to use [`Config`]
|
||||
///
|
||||
/// This includes some intermediate checks for mutually exclusive options
|
||||
pub fn parse() -> std::io::Result<Self> {
|
||||
fn from_opt(opt: Opt) -> std::io::Result<Self> {
|
||||
let Opt {
|
||||
volumes,
|
||||
address,
|
||||
@@ -254,7 +379,7 @@ impl Config {
|
||||
kms_default_key_id,
|
||||
buffer_profile_disable,
|
||||
buffer_profile,
|
||||
} = Opt::parse();
|
||||
} = opt;
|
||||
|
||||
let access_key = access_key
|
||||
.map(Ok)
|
||||
@@ -262,6 +387,7 @@ impl Config {
|
||||
let path = access_key_file.as_ref()?;
|
||||
Some(std::fs::read_to_string(path))
|
||||
})
|
||||
.or_else(|| get_env_opt_str("RUSTFS_ROOT_USER").map(Ok))
|
||||
.transpose()?
|
||||
.unwrap_or_else(|| {
|
||||
// neither argument was specified ... using default
|
||||
@@ -276,6 +402,7 @@ impl Config {
|
||||
let path = secret_key_file.as_ref()?;
|
||||
Some(std::fs::read_to_string(path))
|
||||
})
|
||||
.or_else(|| get_env_opt_str("RUSTFS_ROOT_PASSWORD").map(Ok))
|
||||
.transpose()?
|
||||
.unwrap_or_else(|| {
|
||||
// neither argument was specified ... using default
|
||||
@@ -309,6 +436,16 @@ impl Config {
|
||||
buffer_profile,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse the command line arguments and environment arguments from [`Opt`] and convert them
|
||||
/// into a ready to use [`Config`].
|
||||
///
|
||||
/// Supports both `rustfs <volume>` (legacy) and `rustfs server <volume>`.
|
||||
///
|
||||
/// This includes some intermediate checks for mutually exclusive options.
|
||||
pub fn parse() -> std::io::Result<Self> {
|
||||
Self::from_opt(Opt::parse())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Config {
|
||||
|
||||
+73
-1
@@ -50,6 +50,7 @@ use rustfs_credentials::init_global_action_credentials;
|
||||
use rustfs_ecstore::store::init_lock_clients;
|
||||
use rustfs_ecstore::{
|
||||
bucket::metadata_sys::init_bucket_metadata_sys,
|
||||
bucket::migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
|
||||
bucket::replication::{get_global_replication_pool, init_background_replication},
|
||||
config as ecconfig,
|
||||
endpoints::EndpointServerPools,
|
||||
@@ -69,7 +70,9 @@ use rustfs_iam::{init_iam_sys, init_oidc_sys};
|
||||
use rustfs_metrics::init_metrics_system;
|
||||
use rustfs_obs::{init_obs, set_global_guard};
|
||||
use rustfs_scanner::init_data_scanner;
|
||||
use rustfs_utils::{get_env_bool_with_aliases, net::parse_and_resolve_address};
|
||||
use rustfs_utils::{
|
||||
ExternalEnvCompatReport, apply_external_env_compat, get_env_bool_with_aliases, net::parse_and_resolve_address,
|
||||
};
|
||||
use rustls::crypto::aws_lc_rs::default_provider;
|
||||
use std::io::{Error, Result};
|
||||
use std::sync::Arc;
|
||||
@@ -98,6 +101,10 @@ static GLOBAL: profiling::allocator::TracingAllocator<mimalloc::MiMalloc> =
|
||||
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||
|
||||
fn main() {
|
||||
if let Err(err) = bootstrap_external_prefix_compat() {
|
||||
eprintln!("[WARN] Failed to bootstrap external-prefix compatibility: {err}");
|
||||
}
|
||||
|
||||
let runtime = server::tokio_runtime_builder()
|
||||
.build()
|
||||
.expect("Failed to build Tokio runtime");
|
||||
@@ -108,6 +115,41 @@ fn main() {
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn bootstrap_external_prefix_compat() -> Result<()> {
|
||||
let env_compat_report = apply_external_env_compat();
|
||||
if env_compat_report.conflict_count() > 0 {
|
||||
// RUSTFS_* is the canonical namespace in this codebase, so on key conflicts we keep RUSTFS_*
|
||||
// to preserve explicit user/operator overrides and avoid changing existing runtime behavior.
|
||||
eprintln!(
|
||||
"[WARN] Found {} source/RUSTFS_ conflict(s), keeping RUSTFS_ values: {}",
|
||||
env_compat_report.conflict_count(),
|
||||
env_compat_report.conflict_keys.join(", ")
|
||||
);
|
||||
}
|
||||
|
||||
if env_compat_report.mapped_count() == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
eprintln!(
|
||||
"[INFO] Applying external-prefix compatibility in-process for {} variable(s): {}",
|
||||
env_compat_report.mapped_count(),
|
||||
format_external_prefix_mappings(&env_compat_report)
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn format_external_prefix_mappings(report: &ExternalEnvCompatReport) -> String {
|
||||
report
|
||||
.mapped_pairs
|
||||
.iter()
|
||||
.map(|(source_key, rustfs_key)| format!("{source_key}->{rustfs_key}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
|
||||
async fn async_main() -> Result<()> {
|
||||
// Parse the obtained parameters
|
||||
let config = config::Config::parse()?;
|
||||
@@ -298,6 +340,7 @@ async fn run(config: config::Config) -> Result<()> {
|
||||
})?;
|
||||
|
||||
ecconfig::init();
|
||||
ecconfig::try_migrate_server_config(store.clone()).await;
|
||||
|
||||
// // Initialize global configuration system
|
||||
let mut retry_count = 0;
|
||||
@@ -398,10 +441,13 @@ async fn run(config: config::Config) -> Result<()> {
|
||||
// Collect bucket names into a vector
|
||||
let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
|
||||
try_migrate_bucket_metadata(store.clone()).await;
|
||||
|
||||
if let Some(pool) = get_global_replication_pool() {
|
||||
pool.init_resync(ctx.clone(), buckets.clone()).await?;
|
||||
}
|
||||
|
||||
try_migrate_iam_config(store.clone()).await;
|
||||
init_bucket_metadata_sys(store.clone(), buckets.clone()).await;
|
||||
|
||||
// 3. Initialize IAM System (Blocking load)
|
||||
@@ -634,3 +680,29 @@ async fn handle_shutdown(
|
||||
state_manager.update(ServiceState::Stopped);
|
||||
info!(target: "rustfs::main::handle_shutdown", "Server stopped successfully.");
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn format_external_prefix_mappings_lists_mapped_pairs() {
|
||||
let report = ExternalEnvCompatReport {
|
||||
mapped_pairs: vec![
|
||||
("MINIO_ROOT_USER".to_string(), "RUSTFS_ROOT_USER".to_string()),
|
||||
(
|
||||
"MINIO_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(),
|
||||
"RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(),
|
||||
),
|
||||
],
|
||||
conflict_keys: Vec::new(),
|
||||
};
|
||||
|
||||
let formatted = format_external_prefix_mappings(&report);
|
||||
|
||||
assert_eq!(
|
||||
formatted,
|
||||
"MINIO_ROOT_USER->RUSTFS_ROOT_USER, MINIO_NOTIFY_WEBHOOK_ENABLE_PRIMARY->RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::server::{
|
||||
ReadinessGateLayer, RemoteAddr, ServiceState, ServiceStateManager,
|
||||
compress::{CompressionConfig, CompressionPredicate},
|
||||
hybrid::hybrid,
|
||||
layer::{ConditionalCorsLayer, ObjectAttributesEtagFixLayer, RedirectLayer},
|
||||
layer::{AdminChunkedContentLengthCompatLayer, ConditionalCorsLayer, ObjectAttributesEtagFixLayer, RedirectLayer},
|
||||
};
|
||||
use crate::storage;
|
||||
use crate::storage::tonic_service::make_server;
|
||||
@@ -621,6 +621,7 @@ fn process_connection(
|
||||
None
|
||||
})
|
||||
.layer(SetRequestIdLayer::x_request_id(MakeRequestUuid))
|
||||
.layer(AdminChunkedContentLengthCompatLayer)
|
||||
.layer(CatchPanicLayer::new())
|
||||
// CRITICAL: Insert ReadinessGateLayer before business logic
|
||||
// This stops requests from hitting IAMAuth or Storage if they are not ready.
|
||||
|
||||
+109
-2
@@ -15,13 +15,14 @@
|
||||
use crate::admin::console::is_console_path;
|
||||
use crate::server::cors;
|
||||
use crate::server::hybrid::HybridBody;
|
||||
use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, RPC_PREFIX, RUSTFS_ADMIN_PREFIX};
|
||||
use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX, MINIO_ADMIN_V3_PREFIX, RPC_PREFIX, RUSTFS_ADMIN_PREFIX};
|
||||
use crate::storage::apply_cors_headers;
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderValue, Method, Request as HttpRequest, Response, StatusCode};
|
||||
use http_body::Body;
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Incoming;
|
||||
use rustfs_utils::get_env_opt_str;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
@@ -98,6 +99,64 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AdminChunkedContentLengthCompatLayer;
|
||||
|
||||
impl<S> Layer<S> for AdminChunkedContentLengthCompatLayer {
|
||||
type Service = AdminChunkedContentLengthCompatService<S>;
|
||||
|
||||
fn layer(&self, inner: S) -> Self::Service {
|
||||
AdminChunkedContentLengthCompatService { inner }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AdminChunkedContentLengthCompatService<S> {
|
||||
inner: S,
|
||||
}
|
||||
|
||||
impl<S, ResBody> Service<HttpRequest<Incoming>> for AdminChunkedContentLengthCompatService<S>
|
||||
where
|
||||
S: Service<HttpRequest<Incoming>, Response = Response<ResBody>> + Clone + Send + 'static,
|
||||
S::Future: Send + 'static,
|
||||
S::Error: Into<Box<dyn std::error::Error + Send + Sync>> + Send + 'static,
|
||||
ResBody: Send + 'static,
|
||||
{
|
||||
type Response = Response<ResBody>;
|
||||
type Error = Box<dyn std::error::Error + Send + Sync>;
|
||||
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx).map_err(Into::into)
|
||||
}
|
||||
|
||||
fn call(&mut self, mut req: HttpRequest<Incoming>) -> Self::Future {
|
||||
if should_force_zero_content_length_for_admin_empty_body(&req) {
|
||||
req.headers_mut()
|
||||
.insert(http::header::CONTENT_LENGTH, HeaderValue::from_static("0"));
|
||||
}
|
||||
|
||||
let mut inner = self.inner.clone();
|
||||
Box::pin(async move { inner.call(req).await.map_err(Into::into) })
|
||||
}
|
||||
}
|
||||
|
||||
fn should_force_zero_content_length_for_admin_empty_body<B>(req: &HttpRequest<B>) -> bool {
|
||||
req.method() == Method::PUT
|
||||
&& is_empty_body_admin_put_path(req.uri().path())
|
||||
&& !req.headers().contains_key(http::header::CONTENT_LENGTH)
|
||||
}
|
||||
|
||||
fn is_empty_body_admin_put_path(path: &str) -> bool {
|
||||
matches!(
|
||||
path,
|
||||
"/minio/admin/v3/set-user-status"
|
||||
| "/minio/admin/v3/set-group-status"
|
||||
| "/rustfs/admin/v3/set-user-status"
|
||||
| "/rustfs/admin/v3/set-group-status"
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ObjectAttributesEtagFixLayer;
|
||||
|
||||
@@ -220,7 +279,9 @@ fn is_object_attributes_request(req: &HttpRequest<Incoming>) -> bool {
|
||||
|
||||
let path = req.uri().path();
|
||||
if path.starts_with(ADMIN_PREFIX)
|
||||
|| path.starts_with(MINIO_ADMIN_PREFIX)
|
||||
|| path.starts_with(RUSTFS_ADMIN_PREFIX)
|
||||
|| path.starts_with(MINIO_ADMIN_V3_PREFIX)
|
||||
|| path.starts_with(CONSOLE_PREFIX)
|
||||
|| path.starts_with(RPC_PREFIX)
|
||||
{
|
||||
@@ -253,7 +314,7 @@ pub struct ConditionalCorsLayer {
|
||||
|
||||
impl ConditionalCorsLayer {
|
||||
pub fn new() -> Self {
|
||||
let cors_origins = std::env::var("RUSTFS_CORS_ALLOWED_ORIGINS").ok().filter(|s| !s.is_empty());
|
||||
let cors_origins = get_env_opt_str("RUSTFS_CORS_ALLOWED_ORIGINS").filter(|s| !s.is_empty());
|
||||
Self { cors_origins }
|
||||
}
|
||||
|
||||
@@ -263,6 +324,7 @@ impl ConditionalCorsLayer {
|
||||
fn is_s3_path(path: &str) -> bool {
|
||||
// Exclude Admin, Console, RPC, and configured special paths
|
||||
!path.starts_with(ADMIN_PREFIX)
|
||||
&& !path.starts_with(MINIO_ADMIN_PREFIX)
|
||||
&& !path.starts_with(RPC_PREFIX)
|
||||
&& !is_console_path(path)
|
||||
&& !Self::EXCLUDED_EXACT_PATHS.contains(&path)
|
||||
@@ -501,8 +563,44 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::Request;
|
||||
use http_body_util::BodyExt;
|
||||
use http_body_util::Full;
|
||||
use temp_env::with_var;
|
||||
|
||||
#[test]
|
||||
fn admin_chunked_put_without_content_length_is_normalized() {
|
||||
let request = Request::builder()
|
||||
.method(Method::PUT)
|
||||
.uri("/minio/admin/v3/set-user-status?accessKey=test&status=enabled")
|
||||
.body(())
|
||||
.expect("request");
|
||||
|
||||
assert!(should_force_zero_content_length_for_admin_empty_body(&request));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_request_with_explicit_content_length_is_left_unchanged() {
|
||||
let request = Request::builder()
|
||||
.method(Method::PUT)
|
||||
.uri("/minio/admin/v3/set-group-status?group=test&status=enabled")
|
||||
.header(http::header::CONTENT_LENGTH, "0")
|
||||
.body(())
|
||||
.expect("request");
|
||||
|
||||
assert!(!should_force_zero_content_length_for_admin_empty_body(&request));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_admin_chunked_put_is_not_normalized() {
|
||||
let request = Request::builder()
|
||||
.method(Method::PUT)
|
||||
.uri("/bucket/object")
|
||||
.body(())
|
||||
.expect("request");
|
||||
|
||||
assert!(!should_force_zero_content_length_for_admin_empty_body(&request));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_strip_quotes_from_first_etag_removes_quotes() {
|
||||
@@ -553,6 +651,7 @@ mod tests {
|
||||
assert!(ConditionalCorsLayer::is_s3_path("/my-bucket/key"));
|
||||
assert!(ConditionalCorsLayer::is_s3_path("/"));
|
||||
assert!(!ConditionalCorsLayer::is_s3_path("/rustfs/admin/v3/info"));
|
||||
assert!(!ConditionalCorsLayer::is_s3_path("/minio/admin/v3/info"));
|
||||
assert!(!ConditionalCorsLayer::is_s3_path("/health"));
|
||||
assert!(!ConditionalCorsLayer::is_s3_path("/health/ready"));
|
||||
}
|
||||
@@ -594,6 +693,14 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_conditional_cors_layer_reads_env() {
|
||||
with_var("RUSTFS_CORS_ALLOWED_ORIGINS", Some("https://allowed.com"), || {
|
||||
let cors = ConditionalCorsLayer::new();
|
||||
assert_eq!(cors.cors_origins.as_deref(), Some("https://allowed.com"));
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_resolve_s3_options_cors_headers_no_headers_without_match() {
|
||||
let mut req_headers = HeaderMap::new();
|
||||
|
||||
@@ -36,10 +36,17 @@ pub(crate) const HEALTH_READY_PATH: &str = "/health/ready";
|
||||
/// such as configuration, monitoring, and management.
|
||||
pub(crate) const ADMIN_PREFIX: &str = "/rustfs/admin";
|
||||
|
||||
/// MinIO-compatible administrative prefix accepted by RustFS.
|
||||
/// This alias allows stock MinIO admin tooling to reach RustFS handlers.
|
||||
pub(crate) const MINIO_ADMIN_PREFIX: &str = "/minio/admin";
|
||||
|
||||
/// Environment variable name for overriding the default
|
||||
/// administrative prefix path.
|
||||
pub(crate) const RUSTFS_ADMIN_PREFIX: &str = "/rustfs/admin/v3";
|
||||
|
||||
/// MinIO-compatible admin API prefix accepted by RustFS.
|
||||
pub(crate) const MINIO_ADMIN_V3_PREFIX: &str = "/minio/admin/v3";
|
||||
|
||||
/// Predefined console prefix for RustFS server routes.
|
||||
/// This prefix is used for endpoints that handle console-related tasks
|
||||
/// such as user interface and management.
|
||||
|
||||
@@ -102,9 +102,11 @@ where
|
||||
|
||||
// 2) Prefix matching: the entire set of route prefixes (including their subpaths)
|
||||
let is_prefix_probe = path.starts_with(crate::server::RUSTFS_ADMIN_PREFIX)
|
||||
|| path.starts_with(crate::server::MINIO_ADMIN_V3_PREFIX)
|
||||
|| path.starts_with(crate::server::CONSOLE_PREFIX)
|
||||
|| path.starts_with(crate::server::RPC_PREFIX)
|
||||
|| path.starts_with(crate::server::ADMIN_PREFIX)
|
||||
|| path.starts_with(crate::server::MINIO_ADMIN_PREFIX)
|
||||
|| path.starts_with(crate::server::TONIC_PREFIX);
|
||||
|
||||
let is_probe = is_exact_probe || is_prefix_probe;
|
||||
|
||||
@@ -23,30 +23,11 @@ use rustfs_ecstore::{
|
||||
};
|
||||
use rustfs_s3_common::{S3Operation, record_s3_op};
|
||||
use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error};
|
||||
use std::{fmt::Debug, sync::LazyLock};
|
||||
use std::fmt::Debug;
|
||||
use tokio::io::{AsyncRead, AsyncSeek};
|
||||
use tracing::{debug, error, instrument, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEFAULT_OWNER_ID: &str = "rustfsadmin";
|
||||
const DEFAULT_OWNER_DISPLAY_NAME: &str = "RustFS Tester";
|
||||
|
||||
pub(crate) static RUSTFS_OWNER: LazyLock<Owner> = LazyLock::new(|| Owner {
|
||||
display_name: Some(DEFAULT_OWNER_DISPLAY_NAME.to_owned()),
|
||||
id: Some(DEFAULT_OWNER_ID.to_owned()),
|
||||
});
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct StoredOwner {
|
||||
pub(crate) id: String,
|
||||
}
|
||||
|
||||
pub(crate) fn default_owner() -> StoredOwner {
|
||||
StoredOwner {
|
||||
id: DEFAULT_OWNER_ID.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FS {
|
||||
// pub store: ECStore,
|
||||
|
||||
@@ -31,7 +31,7 @@ use rustfs_targets::EventName;
|
||||
use rustfs_targets::arn::{TargetID, TargetIDError};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
|
||||
RESERVED_METADATA_PREFIX_LOWER,
|
||||
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, insert_str,
|
||||
};
|
||||
use s3s::dto::{
|
||||
Delimiter, LambdaFunctionConfiguration, NotificationConfigurationFilter, ObjectLockConfiguration, ObjectLockEnabled,
|
||||
@@ -284,8 +284,9 @@ pub(crate) fn parse_object_lock_retention(retention: Option<ObjectLockRetention>
|
||||
// This is intentional behavior. Empty string represents "retention cleared" which is different from "retention never set". Consistent with minio
|
||||
eval_metadata.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), mode);
|
||||
eval_metadata.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), retain_until_date);
|
||||
eval_metadata.insert(
|
||||
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-retention-timestamp"),
|
||||
insert_str(
|
||||
&mut eval_metadata,
|
||||
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
||||
format!("{}.{:09}Z", now.format(&Rfc3339).unwrap(), now.nanosecond()),
|
||||
);
|
||||
}
|
||||
@@ -310,8 +311,9 @@ pub(crate) fn parse_object_lock_legal_hold(legal_hold: Option<ObjectLockLegalHol
|
||||
let now = OffsetDateTime::now_utc();
|
||||
// This is intentional behavior. Empty string represents "status cleared" which is different from "status never set".
|
||||
eval_metadata.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), status);
|
||||
eval_metadata.insert(
|
||||
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-legalhold-timestamp"),
|
||||
insert_str(
|
||||
&mut eval_metadata,
|
||||
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
|
||||
format!("{}.{:09}Z", now.format(&Rfc3339).unwrap(), now.nanosecond()),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ mod tests {
|
||||
use crate::config::workload_profiles::WorkloadProfile;
|
||||
use crate::server::cors;
|
||||
use crate::storage::ecfs::FS;
|
||||
use crate::storage::ecfs::RUSTFS_OWNER;
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use crate::storage::{
|
||||
apply_cors_headers, check_preconditions, get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal,
|
||||
matches_origin_pattern, parse_etag, parse_object_lock_legal_hold, parse_object_lock_retention,
|
||||
@@ -29,7 +29,10 @@ mod tests {
|
||||
use rustfs_ecstore::bucket::{metadata::BucketMetadata, metadata_sys};
|
||||
use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
use rustfs_ecstore::store_api::ObjectInfo;
|
||||
use rustfs_utils::http::{AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, RESERVED_METADATA_PREFIX_LOWER};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
||||
contains_key_str,
|
||||
};
|
||||
use rustfs_zip::CompressionFormat;
|
||||
use s3s::dto::{
|
||||
CORSConfiguration, CORSRule, Delimiter, LambdaFunctionConfiguration, ObjectLockLegalHold, ObjectLockLegalHoldStatus,
|
||||
@@ -68,11 +71,15 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rustfs_owner_constant() {
|
||||
// Test that RUSTFS_OWNER constant is properly defined
|
||||
assert!(!RUSTFS_OWNER.display_name.as_ref().unwrap().is_empty());
|
||||
assert!(!RUSTFS_OWNER.id.as_ref().unwrap().is_empty());
|
||||
assert_eq!(RUSTFS_OWNER.display_name.as_ref().unwrap(), "RustFS Tester");
|
||||
fn test_rustfs_owner_helpers_are_stable() {
|
||||
let owner = rustfs_owner();
|
||||
let initiator = rustfs_initiator();
|
||||
|
||||
assert!(!owner.display_name.as_ref().unwrap().is_empty());
|
||||
assert!(!owner.id.as_ref().unwrap().is_empty());
|
||||
assert_eq!(owner.display_name.as_deref(), Some("rustfs"));
|
||||
assert_eq!(initiator.display_name, owner.display_name);
|
||||
assert_eq!(initiator.id, owner.id);
|
||||
}
|
||||
|
||||
// Note: Most S3 API methods require complex setup with global state, storage backend,
|
||||
@@ -440,9 +447,7 @@ mod tests {
|
||||
compliance_metadata.get("x-amz-object-lock-retain-until-date").unwrap(),
|
||||
"2030-01-01T00:00:00Z"
|
||||
);
|
||||
assert!(
|
||||
compliance_metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-retention-timestamp"))
|
||||
);
|
||||
assert!(contains_key_str(&compliance_metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP));
|
||||
|
||||
// [3] Normal case: Retention with valid GOVERNANCE mode (future date)
|
||||
let valid_governance_retention = ObjectLockRetention {
|
||||
@@ -502,7 +507,7 @@ mod tests {
|
||||
};
|
||||
let on_metadata = parse_object_lock_legal_hold(Some(valid_on_legal_hold)).unwrap();
|
||||
assert_eq!(on_metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).unwrap(), "ON");
|
||||
assert!(on_metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "objectlock-legalhold-timestamp")));
|
||||
assert!(contains_key_str(&on_metadata, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP));
|
||||
|
||||
// [3] Normal case: Legal hold with valid OFF status
|
||||
let valid_off_legal_hold = ObjectLockLegalHold {
|
||||
|
||||
@@ -18,7 +18,11 @@ use rustfs_ecstore::error::Result;
|
||||
use rustfs_ecstore::error::StorageError;
|
||||
use rustfs_utils::http::AMZ_META_UNENCRYPTED_CONTENT_LENGTH;
|
||||
use rustfs_utils::http::AMZ_META_UNENCRYPTED_CONTENT_MD5;
|
||||
use rustfs_utils::http::RUSTFS_FORCE_DELETE;
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_FORCE_DELETE, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_SOURCE_DELETEMARKER,
|
||||
SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, get_header, insert_header_map,
|
||||
is_encryption_metadata_key, is_internal_key,
|
||||
};
|
||||
use s3s::header::X_AMZ_OBJECT_LOCK_MODE;
|
||||
use s3s::header::X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE;
|
||||
|
||||
@@ -28,14 +32,6 @@ use rustfs_ecstore::store_api::{HTTPPreconditions, HTTPRangeSpec, ObjectOptions}
|
||||
use rustfs_policy::service_type::ServiceType;
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
use rustfs_utils::http::AMZ_CONTENT_SHA256;
|
||||
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
|
||||
use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_DELETE_MARKER;
|
||||
use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_REQUEST;
|
||||
use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM;
|
||||
use rustfs_utils::http::RUSTFS_BUCKET_SOURCE_MTIME;
|
||||
use rustfs_utils::http::RUSTFS_BUCKET_SOURCE_VERSION_ID;
|
||||
use rustfs_utils::http::RUSTFS_ENCRYPTION_LOWER;
|
||||
use rustfs_utils::http::RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE;
|
||||
use rustfs_utils::path::is_dir_object;
|
||||
use s3s::{S3Result, s3_error};
|
||||
use std::collections::HashMap;
|
||||
@@ -48,7 +44,8 @@ use crate::auth::get_query_param;
|
||||
use crate::auth::get_request_auth_type_with_query;
|
||||
use crate::auth::is_request_presigned_signature_v4_with_query;
|
||||
|
||||
const REPLICATION_REQUEST_TRUE: HeaderValue = HeaderValue::from_static("true");
|
||||
#[cfg(test)]
|
||||
use rustfs_utils::http::insert_header;
|
||||
|
||||
/// Creates options for deleting an object in a bucket.
|
||||
pub async fn del_opts(
|
||||
@@ -62,9 +59,7 @@ pub async fn del_opts(
|
||||
let version_suspended = BucketVersioningSys::suspended(bucket).await;
|
||||
|
||||
let vid = if vid.is_none() {
|
||||
headers
|
||||
.get(RUSTFS_BUCKET_SOURCE_VERSION_ID)
|
||||
.and_then(|v| v.to_str().ok().map(|s| s.to_owned()))
|
||||
get_header(headers, SUFFIX_SOURCE_VERSION_ID).map(|s| s.into_owned())
|
||||
} else {
|
||||
vid
|
||||
};
|
||||
@@ -94,9 +89,8 @@ pub async fn del_opts(
|
||||
StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string())
|
||||
})?;
|
||||
|
||||
opts.delete_prefix = headers
|
||||
.get(RUSTFS_FORCE_DELETE)
|
||||
.map(|v| v.to_str().unwrap_or_default() == "true")
|
||||
opts.delete_prefix = get_header(headers, SUFFIX_FORCE_DELETE)
|
||||
.map(|v| v.as_ref() == "true")
|
||||
.unwrap_or_default();
|
||||
|
||||
opts.version_id = {
|
||||
@@ -109,9 +103,8 @@ pub async fn del_opts(
|
||||
opts.version_suspended = version_suspended;
|
||||
opts.versioned = versioned;
|
||||
|
||||
opts.delete_marker = headers
|
||||
.get(RUSTFS_BUCKET_REPLICATION_DELETE_MARKER)
|
||||
.map(|v| v.to_str().unwrap() == "true")
|
||||
opts.delete_marker = get_header(headers, SUFFIX_SOURCE_DELETEMARKER)
|
||||
.map(|v| v.as_ref() == "true")
|
||||
.unwrap_or_default();
|
||||
|
||||
fill_conditional_writes_opts_from_header(headers, &mut opts)?;
|
||||
@@ -214,9 +207,7 @@ pub async fn put_opts(
|
||||
let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
|
||||
|
||||
let vid = if vid.is_none() {
|
||||
headers
|
||||
.get(RUSTFS_BUCKET_SOURCE_VERSION_ID)
|
||||
.and_then(|v| v.to_str().ok().map(|s| s.to_owned()))
|
||||
get_header(headers, SUFFIX_SOURCE_VERSION_ID).map(|s| s.into_owned())
|
||||
} else {
|
||||
vid
|
||||
};
|
||||
@@ -252,23 +243,21 @@ pub fn get_complete_multipart_upload_opts(headers: &HeaderMap<HeaderValue>) -> s
|
||||
let mut user_defined = HashMap::new();
|
||||
|
||||
let mut replication_request = false;
|
||||
if headers.get(RUSTFS_BUCKET_REPLICATION_REQUEST) == Some(&REPLICATION_REQUEST_TRUE) {
|
||||
if get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") {
|
||||
replication_request = true;
|
||||
if let Some(actual_size_str) = headers
|
||||
.get(RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE)
|
||||
.and_then(|h| h.to_str().ok())
|
||||
{
|
||||
user_defined.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}Actual-Object-Size"), actual_size_str.to_string());
|
||||
if let Some(actual_size_str) = get_header(headers, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE) {
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::SUFFIX_ACTUAL_OBJECT_SIZE_CAP,
|
||||
actual_size_str.into_owned(),
|
||||
);
|
||||
} else {
|
||||
tracing::warn!("Failed to get or parse {} header", RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE);
|
||||
tracing::warn!("Failed to get or parse replication actual object size header (x-rustfs-* or x-minio-*)");
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(v) = headers.get(RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM) {
|
||||
user_defined.insert(
|
||||
RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM.to_string(),
|
||||
v.to_str().unwrap_or_default().to_owned(),
|
||||
);
|
||||
if let Some(v) = get_header(headers, SUFFIX_REPLICATION_SSEC_CRC) {
|
||||
insert_header_map(&mut user_defined, SUFFIX_REPLICATION_SSEC_CRC, v.into_owned());
|
||||
}
|
||||
|
||||
let mut opts = ObjectOptions {
|
||||
@@ -299,26 +288,14 @@ pub fn copy_src_opts(_bucket: &str, _object: &str, headers: &HeaderMap<HeaderVal
|
||||
|
||||
pub fn put_opts_from_headers(headers: &HeaderMap<HeaderValue>, metadata: HashMap<String, String>) -> Result<ObjectOptions> {
|
||||
let mut opts = get_default_opts(headers, metadata, false)?;
|
||||
if headers.get(RUSTFS_BUCKET_REPLICATION_REQUEST) == Some(&REPLICATION_REQUEST_TRUE) {
|
||||
if get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") {
|
||||
opts.replication_request = true;
|
||||
if let Some(v) = headers.get(RUSTFS_BUCKET_SOURCE_MTIME) {
|
||||
match v.to_str() {
|
||||
Ok(s) => {
|
||||
let trimmed_s = s.trim();
|
||||
match time::OffsetDateTime::parse(trimmed_s, &time::format_description::well_known::Rfc3339) {
|
||||
Ok(mtime) => opts.mod_time = Some(mtime),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Invalid X-RustFS-Source-Mtime value '{}' (replication request=true): {}",
|
||||
trimmed_s,
|
||||
e
|
||||
);
|
||||
opts.mod_time = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(v) = get_header(headers, SUFFIX_SOURCE_MTIME) {
|
||||
let trimmed_s = v.trim();
|
||||
match time::OffsetDateTime::parse(trimmed_s, &time::format_description::well_known::Rfc3339) {
|
||||
Ok(mtime) => opts.mod_time = Some(mtime),
|
||||
Err(e) => {
|
||||
tracing::warn!("X-RustFS-Source-Mtime header is not valid UTF-8 (replication request=true): {}", e);
|
||||
tracing::warn!("Invalid source-mtime value '{}' (replication request=true): {}", trimmed_s, e);
|
||||
opts.mod_time = None;
|
||||
}
|
||||
}
|
||||
@@ -379,12 +356,17 @@ pub fn extract_metadata_from_mime_with_object_name(
|
||||
skip_content_type: bool,
|
||||
object_name: Option<&str>,
|
||||
) {
|
||||
const USER_METADATA_PREFIXES: &[&str] = &["x-amz-meta-", "x-rustfs-meta-", "x-minio-meta-"];
|
||||
|
||||
for (k, v) in headers.iter() {
|
||||
if k.as_str() == "content-type" && skip_content_type {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(key) = k.as_str().strip_prefix("x-amz-meta-") {
|
||||
if let Some(key) = USER_METADATA_PREFIXES
|
||||
.iter()
|
||||
.find_map(|prefix| k.as_str().strip_prefix(prefix))
|
||||
{
|
||||
if key.is_empty() {
|
||||
continue;
|
||||
}
|
||||
@@ -393,11 +375,6 @@ pub fn extract_metadata_from_mime_with_object_name(
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(key) = k.as_str().strip_prefix("x-rustfs-meta-") {
|
||||
metadata.insert(key.to_owned(), String::from_utf8_lossy(v.as_bytes()).to_string());
|
||||
continue;
|
||||
}
|
||||
|
||||
for hd in SUPPORTED_HEADERS.iter() {
|
||||
if k.as_str() == *hd {
|
||||
let raw = String::from_utf8_lossy(v.as_bytes()).to_string();
|
||||
@@ -446,13 +423,13 @@ pub(crate) fn filter_object_metadata(metadata: &HashMap<String, String>) -> Opti
|
||||
let mut filtered_metadata = HashMap::new();
|
||||
for (k, v) in metadata {
|
||||
let lower_key = k.to_ascii_lowercase();
|
||||
// Skip internal/reserved metadata
|
||||
if lower_key.starts_with(RESERVED_METADATA_PREFIX_LOWER) {
|
||||
// Skip internal/reserved metadata (x-rustfs-internal-* or x-minio-internal-*)
|
||||
if is_internal_key(&lower_key) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip internal encryption metadata
|
||||
if lower_key.starts_with(RUSTFS_ENCRYPTION_LOWER) {
|
||||
// Skip internal encryption metadata (x-rustfs-encryption-* or x-minio-encryption-*)
|
||||
if is_encryption_metadata_key(&lower_key) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -804,28 +781,28 @@ mod tests {
|
||||
let mut headers = create_test_headers();
|
||||
let metadata = create_test_metadata();
|
||||
|
||||
// Test without RUSTFS_FORCE_DELETE header - should default to false
|
||||
// Test without force-delete header - should default to false
|
||||
let result = del_opts("test-bucket", "test-object", None, &headers, metadata.clone()).await;
|
||||
assert!(result.is_ok());
|
||||
let opts = result.unwrap();
|
||||
assert!(!opts.delete_prefix);
|
||||
|
||||
// Test with RUSTFS_FORCE_DELETE header set to "true"
|
||||
headers.insert(RUSTFS_FORCE_DELETE, HeaderValue::from_static("true"));
|
||||
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
|
||||
let result = del_opts("test-bucket", "test-object", None, &headers, metadata.clone()).await;
|
||||
assert!(result.is_ok());
|
||||
let opts = result.unwrap();
|
||||
assert!(opts.delete_prefix);
|
||||
|
||||
// Test with RUSTFS_FORCE_DELETE header set to "false"
|
||||
headers.insert(RUSTFS_FORCE_DELETE, HeaderValue::from_static("false"));
|
||||
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "false");
|
||||
let result = del_opts("test-bucket", "test-object", None, &headers, metadata.clone()).await;
|
||||
assert!(result.is_ok());
|
||||
let opts = result.unwrap();
|
||||
assert!(!opts.delete_prefix);
|
||||
|
||||
// Test with RUSTFS_FORCE_DELETE header set to other value
|
||||
headers.insert(RUSTFS_FORCE_DELETE, HeaderValue::from_static("maybe"));
|
||||
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "maybe");
|
||||
let result = del_opts("test-bucket", "test-object", None, &headers, metadata).await;
|
||||
assert!(result.is_ok());
|
||||
let opts = result.unwrap();
|
||||
@@ -994,9 +971,9 @@ mod tests {
|
||||
#[test]
|
||||
fn test_put_opts_from_headers_with_replication_request() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE.clone());
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
let valid_mtime = "2024-05-20T10:30:00+08:00";
|
||||
headers.insert(RUSTFS_BUCKET_SOURCE_MTIME, HeaderValue::from_static(valid_mtime));
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, valid_mtime);
|
||||
|
||||
let metadata = HashMap::new();
|
||||
|
||||
@@ -1011,8 +988,8 @@ mod tests {
|
||||
assert_eq!(opts.mod_time, Some(expected_mtime));
|
||||
|
||||
let mut headers_invalid_mtime = HeaderMap::new();
|
||||
headers_invalid_mtime.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE.clone());
|
||||
headers_invalid_mtime.insert(RUSTFS_BUCKET_SOURCE_MTIME, HeaderValue::from_static("invalid-time"));
|
||||
insert_header(&mut headers_invalid_mtime, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
insert_header(&mut headers_invalid_mtime, SUFFIX_SOURCE_MTIME, "invalid-time");
|
||||
let result_invalid = put_opts_from_headers(&headers_invalid_mtime, HashMap::new());
|
||||
assert!(result_invalid.is_ok());
|
||||
let opts_invalid = result_invalid.unwrap();
|
||||
@@ -1090,6 +1067,21 @@ mod tests {
|
||||
assert_eq!(metadata.get("category"), Some(&"documents".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_metadata_from_mime_minio_meta() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-minio-meta-origin", HeaderValue::from_static("gateway"));
|
||||
headers.insert("x-minio-meta-source-id", HeaderValue::from_static("abc123"));
|
||||
headers.insert("x-minio-meta-", HeaderValue::from_static("empty-key"));
|
||||
|
||||
let mut metadata = HashMap::new();
|
||||
extract_metadata_from_mime(&headers, &mut metadata);
|
||||
|
||||
assert_eq!(metadata.get("origin"), Some(&"gateway".to_string()));
|
||||
assert_eq!(metadata.get("source-id"), Some(&"abc123".to_string()));
|
||||
assert!(!metadata.contains_key(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_metadata_from_mime_supported_headers() {
|
||||
let mut headers = HeaderMap::new();
|
||||
@@ -1249,6 +1241,7 @@ mod tests {
|
||||
headers.insert("content-type", HeaderValue::from_static("application/xml"));
|
||||
headers.insert("x-amz-meta-version", HeaderValue::from_static("1.0"));
|
||||
headers.insert("x-rustfs-meta-source", HeaderValue::from_static("upload"));
|
||||
headers.insert("x-minio-meta-origin", HeaderValue::from_static("replication"));
|
||||
headers.insert("cache-control", HeaderValue::from_static("public"));
|
||||
headers.insert("authorization", HeaderValue::from_static("Bearer xyz")); // Should be ignored
|
||||
|
||||
@@ -1257,6 +1250,7 @@ mod tests {
|
||||
assert_eq!(metadata.get("content-type"), Some(&"application/xml".to_string()));
|
||||
assert_eq!(metadata.get("version"), Some(&"1.0".to_string()));
|
||||
assert_eq!(metadata.get("source"), Some(&"upload".to_string()));
|
||||
assert_eq!(metadata.get("origin"), Some(&"replication".to_string()));
|
||||
assert_eq!(metadata.get("cache-control"), Some(&"public".to_string()));
|
||||
assert!(!metadata.contains_key("authorization"));
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use rustfs_ecstore::store_api::{BucketInfo, ListObjectVersionsInfo, ListObjectsV2Info};
|
||||
use s3s::dto::{
|
||||
Bucket, CommonPrefix, DeleteMarkerEntry, EncodingType, ListBucketsOutput, ListObjectVersionsOutput, ListObjectsOutput,
|
||||
ListObjectsV2Output, Object, ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Owner, Timestamp,
|
||||
ListObjectsV2Output, Object, ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Timestamp,
|
||||
};
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
use tracing::debug;
|
||||
@@ -265,10 +265,7 @@ pub(crate) fn build_list_objects_v2_output(
|
||||
};
|
||||
|
||||
if fetch_owner {
|
||||
obj.owner = Some(Owner {
|
||||
display_name: Some("rustfs".to_owned()),
|
||||
id: Some("v0.1".to_owned()),
|
||||
});
|
||||
obj.owner = Some(rustfs_owner());
|
||||
}
|
||||
|
||||
obj
|
||||
@@ -482,10 +479,16 @@ mod tests {
|
||||
|
||||
let contents = output.contents.as_ref().expect("contents should exist");
|
||||
let common_prefixes = output.common_prefixes.as_ref().expect("common prefixes should exist");
|
||||
let expected_owner = rustfs_owner();
|
||||
|
||||
assert_eq!(contents[0].key.as_deref(), Some("dir%20a/file%2Bb%25.txt"));
|
||||
assert_eq!(common_prefixes[0].prefix.as_deref(), Some("prefix%20a/sub%2B"));
|
||||
assert!(contents[0].owner.is_some());
|
||||
assert_eq!(
|
||||
contents[0].owner.as_ref().and_then(|owner| owner.display_name.clone()),
|
||||
expected_owner.display_name
|
||||
);
|
||||
assert_eq!(contents[0].owner.as_ref().and_then(|owner| owner.id.clone()), expected_owner.id);
|
||||
assert_eq!(output.encoding_type.as_ref().map(EncodingType::as_str), Some(EncodingType::URL));
|
||||
}
|
||||
|
||||
|
||||
@@ -184,6 +184,7 @@ mod tests {
|
||||
build_list_multipart_uploads_output, build_list_parts_output, parse_list_multipart_uploads_params,
|
||||
parse_list_parts_params,
|
||||
};
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use rustfs_ecstore::set_disk::MAX_PARTS_COUNT;
|
||||
use rustfs_ecstore::store_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo};
|
||||
@@ -226,8 +227,8 @@ mod tests {
|
||||
assert_eq!(parts[0].part_number, Some(1));
|
||||
assert_eq!(parts[0].size, Some(11));
|
||||
assert_eq!(parts[0].e_tag, Some(to_s3s_etag("etag-1")));
|
||||
assert!(output.owner.is_some());
|
||||
assert!(output.initiator.is_some());
|
||||
assert_eq!(output.owner, Some(rustfs_owner()));
|
||||
assert_eq!(output.initiator, Some(rustfs_initiator()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -88,6 +88,7 @@ use rustfs_kms::{
|
||||
types::{EncryptionMetadata, ObjectEncryptionContext},
|
||||
};
|
||||
use rustfs_rio::{DecryptReader, EncryptReader, HardLimitReader, Reader, WarpReader};
|
||||
use rustfs_utils::get_env_opt_str;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::ServerSideEncryption;
|
||||
use std::collections::HashMap;
|
||||
@@ -1397,8 +1398,8 @@ impl TestSseDekProvider {
|
||||
/// Uses RUSTFS_SSE_S3_MASTER_KEY (base64 32-byte) if set; otherwise a built-in default.
|
||||
/// Allows PUT/GET to work without KMS (backward compatible).
|
||||
pub fn new_for_local_sse() -> Self {
|
||||
let master_key = match std::env::var("RUSTFS_SSE_S3_MASTER_KEY") {
|
||||
Ok(v) if !v.trim().is_empty() => match BASE64_STANDARD.decode(v.trim()) {
|
||||
let master_key = match get_env_opt_str("RUSTFS_SSE_S3_MASTER_KEY") {
|
||||
Some(v) if !v.trim().is_empty() => match BASE64_STANDARD.decode(v.trim()) {
|
||||
Ok(decoded) if decoded.len() == 32 => {
|
||||
let mut arr = [0u8; 32];
|
||||
arr.copy_from_slice(&decoded[..32]);
|
||||
|
||||
Reference in New Issue
Block a user