gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-16 18:08:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(quota): enforce durable hard quota reservations (#6058)

Browse Source 
* fix(quota): enforce durable hard quota reservations

* fix(quota): close reservation bypasses

* fix(quota): isolate tests and box object futures

* fix(quota): close legacy and deferred settlement bypasses

* fix(app): keep object futures off caller stacks

* fix(metrics): preserve object operation labels

* fix(logging): retain GET trace guard contract
This commit is contained in:
cxymds
2026-08-14 14:26:00 +08:00
committed by GitHub
parent 307f50ee1b
commit d60a77b750
35 changed files with 4183 additions and 425 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/protocols/src/swift/handler.rs
+24 -18
View File
@@ -748,9 +748,10 @@ async fn handle_authenticated_request(
}
for (key, value) in info.user_defined.iter() {
if key != "content-type" {
let header_name = format!("x-object-meta-{}", key);
response = response.header(header_name, value.as_str());
if key != "content-type"
&& let Some(key) = object::swift_response_user_metadata_key(key)
{
response = response.header(format!("x-object-meta-{key}"), value.as_str());
}
}
@@ -817,9 +818,10 @@ async fn handle_authenticated_request(
// Add custom metadata headers (X-Object-Meta-*)
for (key, value) in info.user_defined.iter() {
if key != "content-type" {
let header_name = format!("x-object-meta-{}", key);
response = response.header(header_name, value.as_str());
if key != "content-type"
&& let Some(key) = object::swift_response_user_metadata_key(key)
{
response = response.header(format!("x-object-meta-{key}"), value.as_str());
}
}
@@ -856,9 +858,10 @@ async fn handle_authenticated_request(
// Add custom metadata headers (X-Object-Meta-*)
for (key, value) in info.user_defined.iter() {
if key != "content-type" {
let header_name = format!("x-object-meta-{}", key);
response = response.header(header_name, value.as_str());
if key != "content-type"
&& let Some(key) = object::swift_response_user_metadata_key(key)
{
response = response.header(format!("x-object-meta-{key}"), value.as_str());
}
}
@@ -1168,9 +1171,10 @@ async fn handle_object_get(
}
for (key, value) in info.user_defined.iter() {
if key != "content-type" {
let header_name = format!("x-object-meta-{}", key);
response = response.header(header_name, value.as_str());
if key != "content-type"
&& let Some(key) = object::swift_response_user_metadata_key(key)
{
response = response.header(format!("x-object-meta-{key}"), value.as_str());
}
}
@@ -1237,9 +1241,10 @@ async fn handle_object_get(
if key == "x-delete-at" {
// Add X-Delete-At header directly (not as X-Object-Meta-*)
response = response.header("x-delete-at", value.as_str());
} else if key != "content-type" {
let header_name = format!("x-object-meta-{}", key);
response = response.header(header_name, value.as_str());
} else if key != "content-type"
&& let Some(key) = object::swift_response_user_metadata_key(key)
{
response = response.header(format!("x-object-meta-{key}"), value.as_str());
}
}
@@ -1293,9 +1298,10 @@ async fn handle_object_head(
if key == "x-delete-at" {
// Add X-Delete-At header directly (not as X-Object-Meta-*)
response = response.header("x-delete-at", value.as_str());
} else if key != "content-type" {
let header_name = format!("x-object-meta-{}", key);
response = response.header(header_name, value.as_str());
} else if key != "content-type"
&& let Some(key) = object::swift_response_user_metadata_key(key)
{
response = response.header(format!("x-object-meta-{key}"), value.as_str());
}
}
crates/protocols/src/swift/object.rs
+81 -31
View File
@@ -67,10 +67,57 @@ const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object";
const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state";
const SWIFT_DELETE_AT_METADATA: &str = "x-delete-at";
const USER_METADATA_PREFIX: &str = "x-amz-meta-";
/// Maximum object size in bytes (5GB - Swift default)
const MAX_OBJECT_SIZE: i64 = 5 * 1024 * 1024 * 1024;
fn stored_swift_user_metadata_key(key: &str) -> String {
if rustfs_utils::http::is_internal_key(key)
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-amz-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-rustfs-encryption-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-minio-encryption-")
{
format!("{USER_METADATA_PREFIX}{key}")
} else {
key.to_string()
}
}
pub(super) fn swift_response_user_metadata_key(key: &str) -> Option<&str> {
if rustfs_utils::http::is_internal_key(key)
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-rustfs-encryption-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(key, "x-minio-encryption-")
{
return None;
}
if let Some(unescaped) = key.strip_prefix(USER_METADATA_PREFIX)
&& (rustfs_utils::http::is_internal_key(unescaped)
|| rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-amz-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-rustfs-encryption-")
|| rustfs_utils::http::starts_with_ignore_ascii_case(unescaped, "x-minio-encryption-"))
{
return Some(unescaped);
}
Some(key)
}
fn swift_user_metadata(headers: &HeaderMap) -> Option<HashMap<String, String>> {
let mut metadata = HashMap::new();
let mut present = false;
for (header_name, header_value) in headers.iter() {
let header_name = header_name.as_str().to_lowercase();
let Some(key) = header_name.strip_prefix("x-object-meta-") else {
continue;
};
present = true;
if let Ok(value) = header_value.to_str() {
metadata.insert(stored_swift_user_metadata_key(key), value.to_string());
}
}
present.then_some(metadata)
}
/// Object key translator for Swift object names
///
/// Handles URL encoding/decoding and path normalization for Swift object keys.
@@ -303,15 +350,7 @@ where
let bucket = mapper.swift_to_s3_bucket(container, &project_id);
// 5. Extract Swift metadata from X-Object-Meta-* headers
let mut user_metadata = HashMap::new();
for (header_name, header_value) in headers.iter() {
let header_str = header_name.as_str().to_lowercase();
if let Some(meta_key) = header_str.strip_prefix("x-object-meta-")
&& let Ok(value_str) = header_value.to_str()
{
user_metadata.insert(meta_key.to_string(), value_str.to_string());
}
}
let mut user_metadata = swift_user_metadata(headers).unwrap_or_default();
// 6. Extract Content-Type if provided
if let Some(content_type) = headers.get("content-type")
@@ -739,15 +778,7 @@ pub async fn update_object_metadata(
}
// 8. Extract new metadata from X-Object-Meta-* headers
let mut new_metadata = HashMap::new();
for (header_name, header_value) in headers.iter() {
let header_str = header_name.as_str().to_lowercase();
if let Some(meta_key) = header_str.strip_prefix("x-object-meta-")
&& let Ok(value_str) = header_value.to_str()
{
new_metadata.insert(meta_key.to_string(), value_str.to_string());
}
}
let mut new_metadata = swift_user_metadata(headers).unwrap_or_default();
// 9. Also update Content-Type if provided
if let Some(content_type) = headers.get("content-type")
@@ -889,19 +920,8 @@ pub async fn copy_object(
let mut new_metadata = (*src_info.user_defined).clone();
// 11. If custom metadata headers provided, use those instead (Swift behavior)
let mut has_custom_meta = false;
for (header_name, header_value) in headers.iter() {
let header_str = header_name.as_str().to_lowercase();
if let Some(meta_key) = header_str.strip_prefix("x-object-meta-") {
if !has_custom_meta {
// First custom meta header - clear source metadata
new_metadata.clear();
has_custom_meta = true;
}
if let Ok(value_str) = header_value.to_str() {
new_metadata.insert(meta_key.to_string(), value_str.to_string());
}
}
if let Some(custom_metadata) = swift_user_metadata(headers) {
new_metadata = custom_metadata;
}
// 12. Also check for Content-Type override
@@ -1123,6 +1143,36 @@ mod tests {
assert!(ObjectKeyMapper::validate_object_name("unicode-文件.txt").is_ok());
}
#[test]
fn swift_user_metadata_cannot_materialize_internal_storage_keys() {
let mut headers = HeaderMap::new();
headers.insert("x-object-meta-x-rustfs-internal-actual-size", "1".parse().expect("valid metadata value"));
headers.insert("x-object-meta-description", "safe".parse().expect("valid metadata value"));
let metadata = swift_user_metadata(&headers).expect("custom metadata should be detected");
assert_eq!(metadata.get("x-amz-meta-x-rustfs-internal-actual-size").map(String::as_str), Some("1"));
assert_eq!(metadata.get("description").map(String::as_str), Some("safe"));
assert!(!metadata.contains_key("x-rustfs-internal-actual-size"));
assert_eq!(
stored_swift_user_metadata_key("x-minio-encryption-original-size"),
"x-amz-meta-x-minio-encryption-original-size"
);
assert_eq!(stored_swift_user_metadata_key("description"), "description");
}
#[test]
fn swift_user_metadata_response_mapping_is_reversible_and_filters_internal_keys() {
assert_eq!(
swift_response_user_metadata_key("x-amz-meta-x-rustfs-internal-actual-size"),
Some("x-rustfs-internal-actual-size")
);
assert_eq!(swift_response_user_metadata_key("x-amz-meta-x-amz-checksum"), Some("x-amz-checksum"));
assert_eq!(swift_response_user_metadata_key("x-amz-meta-description"), Some("x-amz-meta-description"));
assert_eq!(swift_response_user_metadata_key("description"), Some("description"));
assert_eq!(swift_response_user_metadata_key("x-rustfs-internal-actual-size"), None);
assert_eq!(swift_response_user_metadata_key("x-minio-internal-actual-size"), None);
}
#[test]
fn test_validate_object_name_empty() {
let result = ObjectKeyMapper::validate_object_name("");