build: update docker config and refine s3s region handling (#1976)

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
heihutu
2026-02-27 01:21:12 +08:00
committed by GitHub
parent eb07f084cb
commit d983638391
43 changed files with 1009 additions and 852 deletions
+4 -1
View File
@@ -324,7 +324,10 @@ impl Operation for ListTargetsArns {
.clone()
.ok_or_else(|| s3_error!(InvalidRequest, "region not found"))?;
let data_target_arn_list: Vec<_> = active_targets.iter().map(|id| id.to_arn(&region).to_string()).collect();
let data_target_arn_list: Vec<_> = active_targets
.iter()
.map(|id| id.to_arn(region.as_str()).to_string())
.collect();
let data = serde_json::to_vec(&data_target_arn_list)
.map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?;
+1 -1
View File
@@ -242,7 +242,7 @@ impl Operation for AdminOperation {
#[derive(Debug, Clone)]
pub struct Extra {
pub credentials: Option<s3s::auth::Credentials>,
pub region: Option<String>,
pub region: Option<s3s::region::Region>,
pub service: Option<String>,
}
+16 -11
View File
@@ -30,6 +30,7 @@ use crate::storage::*;
use futures::StreamExt;
use http::StatusCode;
use metrics::counter;
use rustfs_config::RUSTFS_REGION;
use rustfs_ecstore::bucket::{
lifecycle::bucket_lifecycle_ops::validate_transition_tier,
metadata::{
@@ -57,6 +58,7 @@ use rustfs_targets::{
use rustfs_utils::http::RUSTFS_FORCE_DELETE;
use rustfs_utils::string::parse_bool;
use s3s::dto::*;
use s3s::region::Region;
use s3s::xml;
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
use std::{fmt::Display, sync::Arc};
@@ -73,8 +75,8 @@ fn to_internal_error(err: impl Display) -> S3Error {
S3Error::with_message(S3ErrorCode::InternalError, format!("{err}"))
}
fn resolve_notification_region(global_region: Option<String>, request_region: Option<String>) -> String {
global_region.unwrap_or_else(|| request_region.unwrap_or_default())
fn resolve_notification_region(global_region: Option<Region>, request_region: Option<Region>) -> Region {
global_region.unwrap_or_else(|| request_region.unwrap_or_else(|| Region::new(RUSTFS_REGION.into()).expect("valid region")))
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -165,7 +167,7 @@ impl DefaultBucketUsecase {
self.context.clone()
}
fn global_region(&self) -> Option<String> {
fn global_region(&self) -> Option<Region> {
self.context.as_ref().and_then(|context| context.region().get())
}
@@ -431,7 +433,7 @@ impl DefaultBucketUsecase {
if let Some(region) = self.global_region() {
return Ok(S3Response::new(GetBucketLocationOutput {
location_constraint: Some(BucketLocationConstraint::from(region)),
location_constraint: Some(BucketLocationConstraint::from(region.to_string())),
}));
}
@@ -1230,9 +1232,9 @@ impl DefaultBucketUsecase {
let event_rules =
event_rules_result.map_err(|e| s3_error!(InvalidArgument, "Invalid ARN in notification configuration: {e}"))?;
warn!("notify event rules: {:?}", &event_rules);
let region_clone = region.clone();
notify
.add_event_specific_rules(&bucket, &region, &event_rules)
.add_event_specific_rules(&bucket, region_clone.as_str(), &event_rules)
.await
.map_err(|e| s3_error!(InternalError, "Failed to add rules: {e}"))?;
@@ -1800,20 +1802,23 @@ mod tests {
#[test]
fn resolve_notification_region_prefers_global_region() {
let region = resolve_notification_region(Some("us-east-1".to_string()), Some("ap-southeast-1".to_string()));
let binding = resolve_notification_region(Some("us-east-1".parse().unwrap()), Some("ap-southeast-1".parse().unwrap()));
let region = binding.as_str();
assert_eq!(region, "us-east-1");
}
#[test]
fn resolve_notification_region_falls_back_to_request_region() {
let region = resolve_notification_region(None, Some("ap-southeast-1".to_string()));
let binding = resolve_notification_region(None, Some("ap-southeast-1".parse().unwrap()));
let region = binding.as_str();
assert_eq!(region, "ap-southeast-1");
}
#[test]
fn resolve_notification_region_defaults_to_empty() {
let region = resolve_notification_region(None, None);
assert!(region.is_empty());
fn resolve_notification_region_defaults_value() {
let binding = resolve_notification_region(None, None);
let region = binding.as_str();
assert_eq!(region, RUSTFS_REGION);
}
#[tokio::test]
+2 -2
View File
@@ -75,7 +75,7 @@ pub trait EndpointsInterface: Send + Sync {
/// Region interface for application-layer use-cases.
pub trait RegionInterface: Send + Sync {
fn get(&self) -> Option<String>;
fn get(&self) -> Option<s3s::region::Region>;
}
/// Tier config interface for application-layer and admin handlers.
@@ -190,7 +190,7 @@ impl EndpointsInterface for EndpointsHandle {
pub struct RegionHandle;
impl RegionInterface for RegionHandle {
fn get(&self) -> Option<String> {
fn get(&self) -> Option<s3s::region::Region> {
get_global_region()
}
}
+5 -4
View File
@@ -27,6 +27,7 @@ use crate::storage::options::{
use crate::storage::*;
use bytes::Bytes;
use futures::StreamExt;
use rustfs_config::RUSTFS_REGION;
use rustfs_ecstore::StorageAPI;
use rustfs_ecstore::bucket::quota::checker::QuotaChecker;
use rustfs_ecstore::bucket::{
@@ -164,7 +165,7 @@ impl DefaultMultipartUsecase {
self.context.as_ref().and_then(|context| context.bucket_metadata().handle())
}
fn global_region(&self) -> Option<String> {
fn global_region(&self) -> Option<s3s::region::Region> {
self.context.as_ref().and_then(|context| context.region().get())
}
@@ -422,12 +423,12 @@ impl DefaultMultipartUsecase {
}
}
let region = self.global_region().unwrap_or_else(|| "us-east-1".to_string());
let region = self.global_region().unwrap_or_else(|| RUSTFS_REGION.parse().unwrap());
let output = CompleteMultipartUploadOutput {
bucket: Some(bucket.clone()),
key: Some(key.clone()),
e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)),
location: Some(region.clone()),
location: Some(region.to_string()),
server_side_encryption: server_side_encryption.clone(),
ssekms_key_id: ssekms_key_id.clone(),
checksum_crc32: checksum_crc32.clone(),
@@ -448,7 +449,7 @@ impl DefaultMultipartUsecase {
bucket: Some(bucket.clone()),
key: Some(key.clone()),
e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)),
location: Some(region),
location: Some(region.to_string()),
server_side_encryption,
ssekms_key_id,
checksum_crc32,
+2 -2
View File
@@ -276,7 +276,7 @@ pub fn get_condition_values(
header: &HeaderMap,
cred: &Credentials,
version_id: Option<&str>,
region: Option<&str>,
region: Option<s3s::region::Region>,
remote_addr: Option<std::net::SocketAddr>,
) -> HashMap<String, Vec<String>> {
let username = if cred.is_temp() || cred.is_service_account() {
@@ -362,7 +362,7 @@ pub fn get_condition_values(
}
if let Some(lc) = region
&& !lc.is_empty()
&& !lc.as_str().is_empty()
{
args.insert("LocationConstraint".to_owned(), vec![lc.to_string()]);
}
+7 -12
View File
@@ -15,8 +15,10 @@
use clap::Parser;
use clap::builder::NonEmptyStringValueParser;
use const_str::concat;
use rustfs_config::RUSTFS_REGION;
use std::path::PathBuf;
use std::string::ToString;
shadow_rs::shadow!(build);
pub mod workload_profiles;
@@ -191,8 +193,10 @@ pub struct Config {
/// tls path for rustfs API and console.
pub tls_path: Option<String>,
/// License key for enterprise features
pub license: Option<String>,
/// Region for the server, used for signing and other region-specific behavior
pub region: Option<String>,
/// Enable KMS encryption for server-side encryption
@@ -280,6 +284,9 @@ impl Config {
.trim()
.to_string();
// Region is optional, but if not set, we should default to "rustfs-global-0" for signing compatibility with AWS S3 clients
let region = region.or_else(|| Some(RUSTFS_REGION.to_string()));
Ok(Config {
volumes,
address,
@@ -329,15 +336,3 @@ impl std::fmt::Debug for Config {
.finish()
}
}
// lazy_static::lazy_static! {
// pub(crate) static ref OPT: OnceLock<Opt> = OnceLock::new();
// }
// pub fn init_config(opt: Opt) {
// OPT.set(opt).expect("Failed to set global config");
// }
// pub fn get_config() -> &'static Opt {
// OPT.get().expect("Global config not initialized")
// }
+9 -8
View File
@@ -93,14 +93,15 @@ pub(crate) fn init_update_check() {
/// * `buckets` - A vector of bucket names to process
#[instrument(skip_all)]
pub(crate) async fn add_bucket_notification_configuration(buckets: Vec<String>) {
let region_opt = rustfs_ecstore::global::get_global_region();
let region = match region_opt {
Some(ref r) if !r.is_empty() => r,
_ => {
let global_region = rustfs_ecstore::global::get_global_region();
let region = global_region
.as_ref()
.filter(|r| !r.as_str().is_empty())
.map(|r| r.as_str())
.unwrap_or_else(|| {
warn!("Global region is not set; attempting notification configuration for all buckets with an empty region.");
""
}
};
});
for bucket in buckets.iter() {
let has_notification_config = metadata_sys::get_notification_config(bucket).await.unwrap_or_else(|err| {
warn!("get_notification_config err {:?}", err);
@@ -368,7 +369,7 @@ pub async fn init_ftp_system() -> Result<Option<tokio::sync::broadcast::Sender<(
// Create FTP server with protocol storage client
let fs = crate::storage::ecfs::FS::new();
let storage_client = ProtocolStorageClient::new(fs);
let server: FtpsServer<crate::protocols::ProtocolStorageClient> = FtpsServer::new(config, storage_client).await?;
let server: FtpsServer<ProtocolStorageClient> = FtpsServer::new(config, storage_client).await?;
// Log server configuration
info!(
@@ -451,7 +452,7 @@ pub async fn init_ftps_system() -> Result<Option<tokio::sync::broadcast::Sender<
// Create FTPS server with protocol storage client
let fs = crate::storage::ecfs::FS::new();
let storage_client = ProtocolStorageClient::new(fs);
let server: FtpsServer<crate::protocols::ProtocolStorageClient> = FtpsServer::new(config, storage_client).await?;
let server: FtpsServer<ProtocolStorageClient> = FtpsServer::new(config, storage_client).await?;
// Log server configuration
info!(
+4 -1
View File
@@ -164,7 +164,10 @@ async fn run(config: config::Config) -> Result<()> {
let readiness = Arc::new(GlobalReadiness::new());
if let Some(region) = &config.region {
rustfs_ecstore::global::set_global_region(region.clone());
let region = region
.parse()
.map_err(|e| Error::other(format!("invalid region {}: {e}", region)))?;
rustfs_ecstore::global::set_global_region(region);
}
let server_addr = parse_and_resolve_address(config.address.as_str()).map_err(Error::other)?;
+2 -2
View File
@@ -44,7 +44,7 @@ pub(crate) struct ReqInfo {
pub bucket: Option<String>,
pub object: Option<String>,
pub version_id: Option<String>,
pub region: Option<String>,
pub region: Option<s3s::region::Region>,
}
#[derive(Clone, Copy)]
@@ -352,7 +352,7 @@ pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3R
&req.headers,
&rustfs_credentials::Credentials::default(),
req_info.version_id.as_deref(),
req.region.as_deref(),
req.region.clone(),
remote_addr,
);
let bucket_name = req_info.bucket.as_deref().unwrap_or("");