gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-12 16:16:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(storage): expose truthful storage class capabilities (#5172)

Browse Source
This commit is contained in:
cxymds
2026-07-24 15:28:03 +08:00
committed by GitHub
parent 6765aca3f9
commit 358caa23cb
19 changed files with 835 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/copy_object_metadata_test.rs
+6 -3
View File
@@ -65,7 +65,7 @@ mod tests {
.content_type("text/javascript; charset=utf-8")
.expires(source_expires)
.website_redirect_location("/source.html")
.storage_class(StorageClass::StandardIa)
.storage_class(StorageClass::ReducedRedundancy)
.metadata("mtime", "1777992333")
.metadata("stale", "must-be-removed")
.body(ByteStream::from_static(content))
@@ -158,7 +158,7 @@ mod tests {
.bucket(bucket)
.key("assets/explicit-storage-class.js")
.copy_source(format!("{bucket}/{key}"))
.storage_class(StorageClass::StandardIa)
.storage_class(StorageClass::ReducedRedundancy)
.send()
.await
.expect("CopyObject with an explicit storage class failed");
@@ -169,7 +169,10 @@ mod tests {
.send()
.await
.expect("HEAD failed after explicit storage class copy");
assert_eq!(explicit_storage_class_head.storage_class().map(StorageClass::as_str), Some("STANDARD_IA"));
assert_eq!(
explicit_storage_class_head.storage_class().map(StorageClass::as_str),
Some("REDUCED_REDUNDANCY")
);
client
.copy_object()
crates/e2e_test/src/lib.rs
+3
View File
@@ -203,6 +203,9 @@ mod copy_object_checksum_test;
#[cfg(test)]
mod multipart_storage_class_test;
#[cfg(test)]
mod storage_class_capability_test;
// S3 dummy-compat bucket API tests
#[cfg(test)]
mod bucket_logging_test;
crates/e2e_test/src/storage_class_capability_test.rs
+405
View File
@@ -0,0 +1,405 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Truthful storage-class write and discovery contract regressions.
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{ObjectAttributes, StorageClass};
use http::header::HOST;
use reqwest::StatusCode;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use serde_json::Value;
use std::error::Error;
use std::path::Path;
const UNSUPPORTED_AWS_CLASSES: [&str; 9] = [
"DEEP_ARCHIVE",
"EXPRESS_ONEZONE",
"GLACIER",
"GLACIER_IR",
"INTELLIGENT_TIERING",
"ONEZONE_IA",
"OUTPOSTS",
"SNOW",
"STANDARD_IA",
];
async fn assert_object_storage_class(
client: &Client,
bucket: &str,
key: &str,
expected: &str,
body: &[u8],
) -> Result<(), Box<dyn Error + Send + Sync>> {
let head = client.head_object().bucket(bucket).key(key).send().await?;
let expected_head = (expected != "STANDARD").then_some(expected);
assert_eq!(
head.storage_class().map(StorageClass::as_str),
expected_head,
"HeadObject must omit implicit STANDARD and report RRS"
);
let listed = client.list_objects_v2().bucket(bucket).prefix(key).send().await?;
let object = listed
.contents()
.iter()
.find(|object| object.key() == Some(key))
.ok_or("object missing from ListObjectsV2")?;
assert_eq!(object.storage_class().map(|storage_class| storage_class.as_str()), Some(expected));
let get = client.get_object().bucket(bucket).key(key).send().await?;
assert_eq!(
get.storage_class().map(StorageClass::as_str),
expected_head,
"GetObject must report the same effective storage class as HeadObject"
);
let downloaded = get.body.collect().await?.into_bytes();
assert_eq!(downloaded.as_ref(), body, "storage-class selection must not alter object bytes");
Ok(())
}
async fn mutate_xl_meta(
root: &str,
bucket: &str,
key: &str,
mutate: impl FnOnce(&mut rustfs_filemeta::MetaObject),
) -> Result<(), Box<dyn Error + Send + Sync>> {
let path = Path::new(root).join(bucket).join(key).join("xl.meta");
let bytes = tokio::fs::read(&path).await?;
let mut file_meta = rustfs_filemeta::FileMeta::load(&bytes)?;
let (index, mut version) = file_meta.find_version(None)?;
let object = version.object.as_mut().ok_or("fixture version is not an object")?;
mutate(object);
file_meta.versions[index] = rustfs_filemeta::FileMetaShallowVersion::try_from(version)?;
tokio::fs::write(path, file_meta.marshal_msg()?).await?;
Ok(())
}
async fn signed_admin_get(
env: &RustFSTestEnvironment,
path: &str,
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
let url = format!("{}{path}", env.url);
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
let request = http::Request::builder()
.method(http::Method::GET)
.uri(uri)
.header(HOST, authority)
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
.body(Body::empty())?;
let signed = sign_v4(request, 0, &env.access_key, &env.secret_key, "", "us-east-1");
let mut request = local_http_client().get(&url);
for (name, value) in signed.headers() {
request = request.header(name, value);
}
Ok(request.send().await?)
}
#[tokio::test]
async fn standard_and_rrs_are_supported_across_put_copy_and_multipart() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(Vec::new()).await?;
let client = env.create_s3_client();
let bucket = "storage-class-supported-contract";
env.create_test_bucket(bucket).await?;
client
.put_object()
.bucket(bucket)
.key("copy-source")
.body(ByteStream::from_static(b"copy-source-body"))
.send()
.await?;
for storage_class in [StorageClass::Standard, StorageClass::ReducedRedundancy] {
let class_name = storage_class.as_str().to_string();
let put_key = format!("put-{class_name}");
let put_body = format!("put-body-{class_name}").into_bytes();
client
.put_object()
.bucket(bucket)
.key(&put_key)
.storage_class(storage_class.clone())
.body(ByteStream::from(put_body.clone()))
.send()
.await?;
assert_object_storage_class(&client, bucket, &put_key, &class_name, &put_body).await?;
let copy_key = format!("copy-{class_name}");
client
.copy_object()
.bucket(bucket)
.key(&copy_key)
.copy_source(format!("{bucket}/copy-source"))
.storage_class(storage_class.clone())
.send()
.await?;
assert_object_storage_class(&client, bucket, &copy_key, &class_name, b"copy-source-body").await?;
let multipart_key = format!("multipart-{class_name}");
let created = client
.create_multipart_upload()
.bucket(bucket)
.key(&multipart_key)
.storage_class(storage_class)
.send()
.await?;
let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
let parts = client
.list_parts()
.bucket(bucket)
.key(&multipart_key)
.upload_id(upload_id)
.send()
.await?;
assert_eq!(parts.storage_class().map(StorageClass::as_str), Some(class_name.as_str()));
client
.abort_multipart_upload()
.bucket(bucket)
.key(&multipart_key)
.upload_id(upload_id)
.send()
.await?;
}
Ok(())
}
#[tokio::test]
async fn label_only_aws_classes_fail_before_put_copy_or_multipart_mutation() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(Vec::new()).await?;
let client = env.create_s3_client();
let bucket = "storage-class-unsupported-contract";
env.create_test_bucket(bucket).await?;
for key in ["put-guard", "copy-source", "copy-guard"] {
client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from(format!("original-{key}").into_bytes()))
.send()
.await?;
}
for unsupported in UNSUPPORTED_AWS_CLASSES {
let put_error = client
.put_object()
.bucket(bucket)
.key("put-guard")
.storage_class(StorageClass::from(unsupported))
.body(ByteStream::from(format!("rejected-put-{unsupported}").into_bytes()))
.send()
.await
.expect_err("label-only PUT storage class must be rejected");
assert_eq!(
put_error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("InvalidStorageClass"),
"PUT returned a different error for {unsupported}"
);
let copy_error = client
.copy_object()
.bucket(bucket)
.key("copy-guard")
.copy_source(format!("{bucket}/copy-source"))
.storage_class(StorageClass::from(unsupported))
.send()
.await
.expect_err("label-only CopyObject storage class must be rejected");
assert_eq!(
copy_error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("InvalidStorageClass"),
"CopyObject returned a different error for {unsupported}"
);
let multipart_key = format!("multipart-{unsupported}");
let multipart_error = client
.create_multipart_upload()
.bucket(bucket)
.key(&multipart_key)
.storage_class(StorageClass::from(unsupported))
.send()
.await
.expect_err("label-only CreateMultipartUpload storage class must be rejected");
assert_eq!(
multipart_error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("InvalidStorageClass"),
"CreateMultipartUpload returned a different error for {unsupported}"
);
}
let put_guard = client
.get_object()
.bucket(bucket)
.key("put-guard")
.send()
.await?
.body
.collect()
.await?
.into_bytes();
assert_eq!(put_guard.as_ref(), b"original-put-guard");
let copy_guard = client
.get_object()
.bucket(bucket)
.key("copy-guard")
.send()
.await?
.body
.collect()
.await?
.into_bytes();
assert_eq!(copy_guard.as_ref(), b"original-copy-guard");
let uploads = client.list_multipart_uploads().bucket(bucket).send().await?;
assert!(uploads.uploads().is_empty(), "unsupported classes must not create multipart sessions");
Ok(())
}
#[tokio::test]
async fn historical_label_only_metadata_is_standard_without_hiding_a_real_transition_tier()
-> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(Vec::new()).await?;
let client = env.create_s3_client();
let bucket = "storage-class-historical-contract";
let legacy_key = "legacy-label-only";
let transitioned_key = "real-transition-tier";
env.create_test_bucket(bucket).await?;
for key in [legacy_key, transitioned_key] {
client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"fixture-body"))
.send()
.await?;
}
env.stop_server();
mutate_xl_meta(&env.temp_dir, bucket, legacy_key, |object| {
object
.meta_user
.insert("x-amz-storage-class".to_string(), "STANDARD_IA".to_string());
})
.await?;
mutate_xl_meta(&env.temp_dir, bucket, transitioned_key, |object| {
object.set_transition(&rustfs_filemeta::FileInfo {
transition_status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
transition_tier: "STANDARD_IA".to_string(),
..Default::default()
});
})
.await?;
env.restart_server_preserving_data(Vec::new(), &[]).await?;
assert_object_storage_class(&client, bucket, legacy_key, "STANDARD", b"fixture-body").await?;
let legacy_attributes = client
.get_object_attributes()
.bucket(bucket)
.key(legacy_key)
.object_attributes(ObjectAttributes::StorageClass)
.send()
.await?;
assert_eq!(legacy_attributes.storage_class().map(StorageClass::as_str), Some("STANDARD"));
let versions = client.list_object_versions().bucket(bucket).prefix(legacy_key).send().await?;
let legacy_version = versions
.versions()
.iter()
.find(|version| version.key() == Some(legacy_key))
.ok_or("legacy fixture missing from ListObjectVersions")?;
assert_eq!(legacy_version.storage_class().map(|class| class.as_str()), Some("STANDARD"));
let transitioned_head = client.head_object().bucket(bucket).key(transitioned_key).send().await?;
assert_eq!(transitioned_head.storage_class().map(StorageClass::as_str), Some("STANDARD_IA"));
let transitioned_attributes = client
.get_object_attributes()
.bucket(bucket)
.key(transitioned_key)
.object_attributes(ObjectAttributes::StorageClass)
.send()
.await?;
assert_eq!(transitioned_attributes.storage_class().map(StorageClass::as_str), Some("STANDARD_IA"));
let transitioned_list = client
.list_objects_v2()
.bucket(bucket)
.prefix(transitioned_key)
.send()
.await?;
assert_eq!(
transitioned_list.contents()[0].storage_class().map(|class| class.as_str()),
Some("STANDARD_IA")
);
let transitioned_versions = client
.list_object_versions()
.bucket(bucket)
.prefix(transitioned_key)
.send()
.await?;
assert_eq!(
transitioned_versions.versions()[0]
.storage_class()
.map(|class| class.as_str()),
Some("STANDARD_IA")
);
Ok(())
}
#[tokio::test]
async fn authenticated_runtime_capabilities_publish_the_versioned_storage_class_contract()
-> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(Vec::new()).await?;
let path = "/rustfs/admin/v4/runtime/capabilities";
let unsigned = local_http_client().get(format!("{}{path}", env.url)).send().await?;
assert_eq!(unsigned.status(), StatusCode::FORBIDDEN);
let unsigned_body = unsigned.text().await?;
assert!(
!unsigned_body.contains("supported_write_classes"),
"the capability contract must not bypass admin authentication"
);
let response = signed_admin_get(&env, path).await?;
assert_eq!(response.status(), StatusCode::OK);
let body: Value = response.json().await?;
assert_eq!(body["storage_classes"]["contract_version"], 1);
assert_eq!(
body["storage_classes"]["supported_write_classes"],
serde_json::json!(["STANDARD", "REDUCED_REDUNDANCY"])
);
assert_eq!(body["storage_classes"]["unsupported_write_error"], "InvalidStorageClass");
assert_eq!(body["storage_classes"]["legacy_label_behavior"], "normalized_to_effective_class");
Ok(())
}
}
crates/ecstore/src/api/mod.rs
+6 -5
View File
@@ -252,11 +252,12 @@ pub mod config {
pub mod storageclass {
pub use crate::config::storageclass::{
CLASS_RRS, CLASS_STANDARD, Config, DEEP_ARCHIVE, DEFAULT_INLINE_BLOCK, DEFAULT_KVS, DEFAULT_RRS_PARITY,
EXPRESS_ONEZONE, GLACIER, GLACIER_IR, INLINE_BLOCK, INLINE_BLOCK_ENV, INTELLIGENT_TIERING, MIN_PARITY_DRIVES,
ONEZONE_IA, OPTIMIZE, OPTIMIZE_ENV, OUTPOSTS, RRS, RRS_ENV, SCHEME_PREFIX, SNOW, STANDARD, STANDARD_ENV, STANDARD_IA,
StorageClass, default_parity_count, lookup_config, lookup_config_for_pools, parse_storage_class, validate_parity,
validate_parity_inner,
CAPABILITY_CONTRACT_VERSION, CLASS_RRS, CLASS_STANDARD, Config, DEEP_ARCHIVE, DEFAULT_INLINE_BLOCK, DEFAULT_KVS,
DEFAULT_RRS_PARITY, EXPRESS_ONEZONE, GLACIER, GLACIER_IR, INLINE_BLOCK, INLINE_BLOCK_ENV, INTELLIGENT_TIERING,
LEGACY_LABEL_BEHAVIOR, MIN_PARITY_DRIVES, ONEZONE_IA, OPTIMIZE, OPTIMIZE_ENV, OUTPOSTS, RRS, RRS_ENV, SCHEME_PREFIX,
SNOW, STANDARD, STANDARD_ENV, STANDARD_IA, SUPPORTED_WRITE_CLASSES, StorageClass, UNSUPPORTED_WRITE_ERROR,
default_parity_count, effective_class, is_supported_write_class, lookup_config, lookup_config_for_pools,
parse_storage_class, validate_parity, validate_parity_inner,
};
}
crates/ecstore/src/config/storageclass.rs
+69
View File
@@ -46,6 +46,34 @@ pub const OUTPOSTS: &str = "OUTPOSTS";
pub const SNOW: &str = "SNOW";
pub const STANDARD_IA: &str = "STANDARD_IA";
/// Version of the client-discoverable storage-class write contract.
pub const CAPABILITY_CONTRACT_VERSION: u32 = 1;
/// Storage classes whose write semantics RustFS implements.
pub const SUPPORTED_WRITE_CLASSES: [&str; 2] = [STANDARD, RRS];
/// Stable S3 error code returned for unsupported write classes.
pub const UNSUPPORTED_WRITE_ERROR: &str = "InvalidStorageClass";
/// Compatibility behavior applied to historical label-only object metadata.
pub const LEGACY_LABEL_BEHAVIOR: &str = "normalized_to_effective_class";
/// Returns whether a client may select this storage class for a write.
pub fn is_supported_write_class(storage_class: &str) -> bool {
SUPPORTED_WRITE_CLASSES.contains(&storage_class)
}
/// Resolves the storage class that truthfully describes the stored object.
///
/// A completed lifecycle transition is a real storage tier and therefore keeps
/// its tier name. For local objects, only RRS has distinct layout semantics;
/// historical AWS class labels otherwise describe the effective STANDARD
/// layout.
pub fn effective_class<'a>(stored_class: Option<&'a str>, transitioned_tier: Option<&'a str>) -> &'a str {
if let Some(tier) = transitioned_tier.filter(|tier| !tier.is_empty()) {
return tier;
}
if stored_class == Some(RRS) { RRS } else { STANDARD }
}
// Standard constants for config info storage class
pub const CLASS_STANDARD: &str = "standard";
pub const CLASS_RRS: &str = "rrs";
@@ -556,6 +584,47 @@ mod tests {
}
}
#[test]
fn write_capability_contract_only_accepts_implemented_layouts() {
assert_eq!(SUPPORTED_WRITE_CLASSES, [STANDARD, RRS]);
assert!(is_supported_write_class(STANDARD));
assert!(is_supported_write_class(RRS));
for label_only_class in [
DEEP_ARCHIVE,
EXPRESS_ONEZONE,
GLACIER,
GLACIER_IR,
INTELLIGENT_TIERING,
ONEZONE_IA,
OUTPOSTS,
SNOW,
STANDARD_IA,
] {
assert!(
!is_supported_write_class(label_only_class),
"{label_only_class} must not be advertised as a supported write class"
);
}
assert!(!is_supported_write_class(""));
assert!(!is_supported_write_class("standard"));
assert!(!is_supported_write_class("UNKNOWN"));
}
#[test]
fn effective_class_normalizes_legacy_labels_and_preserves_real_tiers() {
assert_eq!(effective_class(None, None), STANDARD);
assert_eq!(effective_class(Some(STANDARD), None), STANDARD);
assert_eq!(effective_class(Some(RRS), None), RRS);
assert_eq!(effective_class(Some(STANDARD_IA), None), STANDARD);
assert_eq!(effective_class(Some(GLACIER), None), STANDARD);
assert_eq!(effective_class(Some("UNKNOWN"), None), STANDARD);
assert_eq!(effective_class(Some(STANDARD_IA), Some("WARM-TIER")), "WARM-TIER");
assert_eq!(effective_class(Some(STANDARD), Some(STANDARD_IA)), STANDARD_IA);
assert_eq!(effective_class(Some(RRS), Some("CUSTOM-RRS-TIER")), "CUSTOM-RRS-TIER");
}
#[test]
fn automatic_parity_is_resolved_per_pool() {
let cfg = lookup_config_for_pools_with_env(&KVS::new(), &[4, 2], no_env_overrides())
crates/ecstore/src/object_api/types.rs
+60 -9
View File
@@ -478,15 +478,14 @@ impl ObjectInfo {
v
};
// Extract storage class from metadata, default to STANDARD if not found
let storage_class = if !fi.transition_tier.is_empty() {
Some(fi.transition_tier.clone())
} else {
fi.metadata
.get(AMZ_STORAGE_CLASS)
.cloned()
.or_else(|| Some(storageclass::STANDARD.to_string()))
};
let storage_class = Some(
storageclass::effective_class(
fi.metadata.get(AMZ_STORAGE_CLASS).map(String::as_str),
(fi.transition_status == rustfs_filemeta::TRANSITION_COMPLETE && !fi.transition_tier.is_empty())
.then_some(fi.transition_tier.as_str()),
)
.to_string(),
);
let mut restore_ongoing = false;
let mut restore_expires = None;
@@ -1145,6 +1144,58 @@ mod tests {
assert_eq!(info.replication_decision, "arn=true;false;arn:replication::1:dest;rule-id");
}
#[test]
fn from_file_info_reports_effective_storage_class_for_legacy_metadata() {
for legacy_label in [
storageclass::STANDARD_IA,
storageclass::ONEZONE_IA,
storageclass::INTELLIGENT_TIERING,
storageclass::GLACIER,
] {
let fi = FileInfo {
metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), legacy_label.to_string())]),
..Default::default()
};
let info = ObjectInfo::from_file_info(&fi, "bucket", "legacy-object", true);
assert_eq!(
info.storage_class.as_deref(),
Some(storageclass::STANDARD),
"{legacy_label} was only a label and must report the effective STANDARD layout"
);
}
}
#[test]
fn from_file_info_preserves_transitioned_tier_storage_class() {
let fi = FileInfo {
metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string())]),
transition_tier: "WARM-TIER".to_string(),
transition_status: TRANSITION_COMPLETE.to_string(),
..Default::default()
};
let info = ObjectInfo::from_file_info(&fi, "bucket", "transitioned-object", true);
assert_eq!(info.storage_class.as_deref(), Some("WARM-TIER"));
assert_eq!(info.transitioned_object.tier, "WARM-TIER");
}
#[test]
fn from_file_info_ignores_a_tier_name_without_a_completed_transition() {
let fi = FileInfo {
metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string())]),
transition_tier: "WARM-TIER".to_string(),
..Default::default()
};
let info = ObjectInfo::from_file_info(&fi, "bucket", "incomplete-transition", true);
assert_eq!(info.storage_class.as_deref(), Some(storageclass::STANDARD));
assert_eq!(info.transitioned_object.tier, "WARM-TIER");
}
#[test]
fn get_actual_size_uses_compressed_parts_actual_size_when_metadata_missing() {
let user_defined = {
crates/ecstore/src/set_disk/mod.rs
+2 -28
View File
@@ -4408,20 +4408,7 @@ pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<String>, if_m
/// Validates if the given storage class is supported
pub fn is_valid_storage_class(storage_class: &str) -> bool {
matches!(
storage_class,
storageclass::STANDARD
| storageclass::RRS
| storageclass::DEEP_ARCHIVE
| storageclass::EXPRESS_ONEZONE
| storageclass::GLACIER
| storageclass::GLACIER_IR
| storageclass::INTELLIGENT_TIERING
| storageclass::ONEZONE_IA
| storageclass::OUTPOSTS
| storageclass::SNOW
| storageclass::STANDARD_IA
)
storageclass::is_supported_write_class(storage_class)
}
/// Returns true if the storage class is a cold storage tier that requires special handling
@@ -7796,23 +7783,10 @@ mod tests {
#[test]
fn test_is_valid_storage_class() {
// Test valid storage classes
assert!(is_valid_storage_class(storageclass::STANDARD));
assert!(is_valid_storage_class(storageclass::RRS));
assert!(is_valid_storage_class(storageclass::DEEP_ARCHIVE));
assert!(is_valid_storage_class(storageclass::EXPRESS_ONEZONE));
assert!(is_valid_storage_class(storageclass::GLACIER));
assert!(is_valid_storage_class(storageclass::GLACIER_IR));
assert!(is_valid_storage_class(storageclass::INTELLIGENT_TIERING));
assert!(is_valid_storage_class(storageclass::ONEZONE_IA));
assert!(is_valid_storage_class(storageclass::OUTPOSTS));
assert!(is_valid_storage_class(storageclass::SNOW));
assert!(is_valid_storage_class(storageclass::STANDARD_IA));
// Test invalid storage classes
assert!(!is_valid_storage_class(storageclass::STANDARD_IA));
assert!(!is_valid_storage_class("INVALID"));
assert!(!is_valid_storage_class(""));
assert!(!is_valid_storage_class("standard")); // lowercase
}
#[test]
crates/ecstore/src/store/list_objects.rs
+66 -8
View File
@@ -309,7 +309,8 @@ const MAX_LIST_OBJECTS_METADATA_FAST_STALENESS_MS: u64 = 60_000;
const LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY: &str = "walker_key_only";
const LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY: &str = "persistent_key_only";
const LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY_DEFAULT_GENERATION: &str = "persistent-key-only";
const PERSISTENT_KEY_ONLY_INDEX_HEADER: &str = "# rustfs-listobjects-key-only-v1";
const PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION: u8 = 2;
const PERSISTENT_KEY_ONLY_INDEX_HEADER: &str = "# rustfs-listobjects-key-only-v2";
const PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER: &str = "# bucket=";
const PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER: &str = "# generation=";
const PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER: &str = "# checkpoint_high_water_mark=";
@@ -502,6 +503,7 @@ struct PersistentKeyOnlyIndexCache {
#[derive(Debug, Clone, PartialEq, Eq)]
struct PersistentKeyOnlyIndex {
format_version: u8,
bucket: Option<String>,
generation: String,
checkpoint_high_water_mark: u64,
@@ -1410,6 +1412,7 @@ fn parse_persistent_list_metadata_object(line: &str) -> Option<PersistentListMet
}
fn parse_persistent_key_only_index(contents: &str) -> PersistentKeyOnlyIndex {
let mut format_version = 0;
let mut bucket = None;
let mut generation = None;
let mut checkpoint_high_water_mark = None;
@@ -1421,6 +1424,10 @@ fn parse_persistent_key_only_index(contents: &str) -> PersistentKeyOnlyIndex {
if line.is_empty() {
continue;
}
if line == PERSISTENT_KEY_ONLY_INDEX_HEADER {
format_version = PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION;
continue;
}
if let Some(object) = parse_persistent_list_metadata_object(line) {
keys.push(object.name.clone());
objects.push(object);
@@ -1455,6 +1462,7 @@ fn parse_persistent_key_only_index(contents: &str) -> PersistentKeyOnlyIndex {
let checkpoint_high_water_mark = checkpoint_high_water_mark.unwrap_or_else(|| u64::try_from(keys.len()).unwrap_or(u64::MAX));
PersistentKeyOnlyIndex {
format_version,
bucket,
generation: generation.unwrap_or_else(|| LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY_DEFAULT_GENERATION.to_owned()),
checkpoint_high_water_mark,
@@ -1485,6 +1493,9 @@ fn persistent_key_only_index_matches_provider(
bucket: &str,
provider_state: &ListObjectsIndexProviderState,
) -> bool {
if index.format_version != PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION {
return false;
}
if index.bucket.as_deref().is_some_and(|index_bucket| index_bucket != bucket) {
return false;
}
@@ -1579,6 +1590,7 @@ async fn write_persistent_key_only_index_with_metadata(
tokio::fs::rename(&tmp_path, path).await.map_err(Error::Io)?;
Ok(PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some(bucket.to_owned()),
generation: generation.to_owned(),
checkpoint_high_water_mark,
@@ -6687,13 +6699,13 @@ mod test {
ListObjectsIndexProviderState, ListPathOptions, ListPathRawOptions, ListSourceMode, ListingEntryResolution,
ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend,
NamespaceMutationJournalSnapshot, NamespaceMutationJournalStatus, PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER,
PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER,
PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, PersistentListMetadataObject, RUSTFS_META_BUCKET,
VerifiedIndexCandidateStats, VersionMarker, current_list_objects_mutation_sequence,
encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum,
fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects, latest_listing_object_quorum,
latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key, list_merged_entry_channel,
list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates,
PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER, PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex,
PersistentListMetadataObject, RUSTFS_META_BUCKET, VerifiedIndexCandidateStats, VersionMarker,
current_list_objects_mutation_sequence, encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum,
expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects,
latest_listing_object_quorum, latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key,
list_merged_entry_channel, list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates,
list_objects_from_verified_index_candidates, list_objects_from_verified_index_candidates_with_optional_stats,
list_objects_from_verified_index_candidates_with_stats, list_objects_index_mode_from_env,
list_objects_index_provider_from_env, list_objects_index_provider_state_from_env, list_objects_key_only_provider_health,
@@ -8041,6 +8053,7 @@ mod test {
#[test]
fn metadata_fast_requires_complete_metadata_snapshot() {
let index = PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some("bucket".to_string()),
generation: "generation-42".to_string(),
checkpoint_high_water_mark: 42,
@@ -8057,6 +8070,7 @@ mod test {
assert!(!persistent_key_only_index_has_complete_metadata_snapshot(&index));
let complete = PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some("bucket".to_string()),
generation: "generation-42".to_string(),
checkpoint_high_water_mark: 42,
@@ -8402,6 +8416,7 @@ mod test {
#[test]
fn persistent_key_only_index_provider_match_rejects_configured_generation_mismatch() {
let index = PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some("bucket".to_string()),
generation: "generation-old".to_string(),
checkpoint_high_water_mark: 42,
@@ -8422,9 +8437,50 @@ mod test {
assert!(!persistent_key_only_index_matches_provider(&index, "other-bucket", &matching_provider));
}
#[test]
fn persistent_key_only_index_provider_match_rejects_legacy_format_without_configured_generation() {
let legacy = parse_persistent_key_only_index(&format!(
"# rustfs-listobjects-key-only-v1\n\
{PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER}bucket\n\
{PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER}generation-old\n\
{PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER}42\n\
# object\tbGVnYWN5\t1\t-\t-\tU1RBTkRBUkRfSUE=\n"
));
let provider =
ListObjectsIndexProviderState::persistent_key_only(Some(PathBuf::from("/tmp/persistent-key-only.index")), None);
assert_eq!(legacy.format_version, 0);
assert!(!persistent_key_only_index_matches_provider(&legacy, "bucket", &provider));
}
#[test]
fn current_persistent_snapshot_preserves_an_effective_transition_tier_name() {
let object = PersistentListMetadataObject::from_object_info(&ObjectInfo {
bucket: "bucket".to_string(),
name: "transitioned".to_string(),
storage_class: Some("STANDARD_IA".to_string()),
..Default::default()
});
let index = parse_persistent_key_only_index(&format!(
"{PERSISTENT_KEY_ONLY_INDEX_HEADER}\n\
{PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER}bucket\n\
{PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER}generation-current\n\
{PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER}42\n\
{}\n",
encode_persistent_list_metadata_object(&object)
));
let provider =
ListObjectsIndexProviderState::persistent_key_only(Some(PathBuf::from("/tmp/persistent-key-only.index")), None);
assert_eq!(index.format_version, PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION);
assert!(persistent_key_only_index_matches_provider(&index, "bucket", &provider));
assert_eq!(index.objects[0].to_object_info("bucket").storage_class.as_deref(), Some("STANDARD_IA"));
}
#[test]
fn persistent_key_only_index_health_uses_snapshot_generation_and_checkpoint() {
let index = PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some("bucket".to_string()),
generation: "generation-42".to_string(),
checkpoint_high_water_mark: 42,
@@ -8450,6 +8506,7 @@ mod test {
#[test]
fn persistent_key_only_index_health_reports_lagging_mutation_checkpoint() {
let index = PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some("bucket".to_string()),
generation: "generation-42".to_string(),
checkpoint_high_water_mark: 42,
@@ -8476,6 +8533,7 @@ mod test {
#[test]
fn persistent_key_only_index_health_reports_degraded_journal() {
let index = PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some("bucket".to_string()),
generation: "generation-42".to_string(),
checkpoint_high_water_mark: 42,