From 358caa23cb6d95122a3d950ed09738f32ede0317 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 24 Jul 2026 15:28:03 +0800 Subject: [PATCH] fix(storage): expose truthful storage class capabilities (#5172) --- .config/nextest.toml | 2 +- .../e2e_test/src/copy_object_metadata_test.rs | 9 +- crates/e2e_test/src/lib.rs | 3 + .../src/storage_class_capability_test.rs | 405 ++++++++++++++++++ crates/ecstore/src/api/mod.rs | 11 +- crates/ecstore/src/config/storageclass.rs | 69 +++ crates/ecstore/src/object_api/types.rs | 69 ++- crates/ecstore/src/set_disk/mod.rs | 30 +- crates/ecstore/src/store/list_objects.rs | 74 +++- docs/architecture/erasure-coding.md | 3 + .../runtime-capability-contracts.md | 30 ++ docs/testing/e2e-suite-inventory.md | 3 +- rustfs/src/admin/handlers/system.rs | 33 ++ rustfs/src/admin/storage_api.rs | 4 + rustfs/src/app/object_usecase.rs | 76 +++- rustfs/src/app/storage_api.rs | 4 +- rustfs/src/storage/s3_api/bucket.rs | 63 +++ rustfs/src/storage/s3_api/multipart.rs | 24 +- rustfs/src/storage/storage_api.rs | 4 + 19 files changed, 835 insertions(+), 81 deletions(-) create mode 100644 crates/e2e_test/src/storage_class_capability_test.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 08e91255a..087d90944 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -197,7 +197,7 @@ test-group = 'ecstore-serial-flaky' [profile.e2e-smoke] default-filter = """ package(e2e_test) & ( - test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools)_test::|^fake_s3_target::/) + test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|multipart_storage_class|storage_class_capability|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools)_test::|^fake_s3_target::/) | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/) | test(/^reliant::lifecycle::/) | test(/^reliant::tiering::/) diff --git a/crates/e2e_test/src/copy_object_metadata_test.rs b/crates/e2e_test/src/copy_object_metadata_test.rs index 6827d8520..cfd257f06 100644 --- a/crates/e2e_test/src/copy_object_metadata_test.rs +++ b/crates/e2e_test/src/copy_object_metadata_test.rs @@ -65,7 +65,7 @@ mod tests { .content_type("text/javascript; charset=utf-8") .expires(source_expires) .website_redirect_location("/source.html") - .storage_class(StorageClass::StandardIa) + .storage_class(StorageClass::ReducedRedundancy) .metadata("mtime", "1777992333") .metadata("stale", "must-be-removed") .body(ByteStream::from_static(content)) @@ -158,7 +158,7 @@ mod tests { .bucket(bucket) .key("assets/explicit-storage-class.js") .copy_source(format!("{bucket}/{key}")) - .storage_class(StorageClass::StandardIa) + .storage_class(StorageClass::ReducedRedundancy) .send() .await .expect("CopyObject with an explicit storage class failed"); @@ -169,7 +169,10 @@ mod tests { .send() .await .expect("HEAD failed after explicit storage class copy"); - assert_eq!(explicit_storage_class_head.storage_class().map(StorageClass::as_str), Some("STANDARD_IA")); + assert_eq!( + explicit_storage_class_head.storage_class().map(StorageClass::as_str), + Some("REDUCED_REDUNDANCY") + ); client .copy_object() diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index 24b92f085..c279cd47b 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -203,6 +203,9 @@ mod copy_object_checksum_test; #[cfg(test)] mod multipart_storage_class_test; +#[cfg(test)] +mod storage_class_capability_test; + // S3 dummy-compat bucket API tests #[cfg(test)] mod bucket_logging_test; diff --git a/crates/e2e_test/src/storage_class_capability_test.rs b/crates/e2e_test/src/storage_class_capability_test.rs new file mode 100644 index 000000000..24470be23 --- /dev/null +++ b/crates/e2e_test/src/storage_class_capability_test.rs @@ -0,0 +1,405 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Truthful storage-class write and discovery contract regressions. + +#[cfg(test)] +mod tests { + use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; + use aws_sdk_s3::Client; + use aws_sdk_s3::error::ProvideErrorMetadata; + use aws_sdk_s3::primitives::ByteStream; + use aws_sdk_s3::types::{ObjectAttributes, StorageClass}; + use http::header::HOST; + use reqwest::StatusCode; + use rustfs_signer::constants::UNSIGNED_PAYLOAD; + use rustfs_signer::sign_v4; + use s3s::Body; + use serde_json::Value; + use std::error::Error; + use std::path::Path; + + const UNSUPPORTED_AWS_CLASSES: [&str; 9] = [ + "DEEP_ARCHIVE", + "EXPRESS_ONEZONE", + "GLACIER", + "GLACIER_IR", + "INTELLIGENT_TIERING", + "ONEZONE_IA", + "OUTPOSTS", + "SNOW", + "STANDARD_IA", + ]; + + async fn assert_object_storage_class( + client: &Client, + bucket: &str, + key: &str, + expected: &str, + body: &[u8], + ) -> Result<(), Box> { + let head = client.head_object().bucket(bucket).key(key).send().await?; + let expected_head = (expected != "STANDARD").then_some(expected); + assert_eq!( + head.storage_class().map(StorageClass::as_str), + expected_head, + "HeadObject must omit implicit STANDARD and report RRS" + ); + + let listed = client.list_objects_v2().bucket(bucket).prefix(key).send().await?; + let object = listed + .contents() + .iter() + .find(|object| object.key() == Some(key)) + .ok_or("object missing from ListObjectsV2")?; + assert_eq!(object.storage_class().map(|storage_class| storage_class.as_str()), Some(expected)); + + let get = client.get_object().bucket(bucket).key(key).send().await?; + assert_eq!( + get.storage_class().map(StorageClass::as_str), + expected_head, + "GetObject must report the same effective storage class as HeadObject" + ); + let downloaded = get.body.collect().await?.into_bytes(); + assert_eq!(downloaded.as_ref(), body, "storage-class selection must not alter object bytes"); + Ok(()) + } + + async fn mutate_xl_meta( + root: &str, + bucket: &str, + key: &str, + mutate: impl FnOnce(&mut rustfs_filemeta::MetaObject), + ) -> Result<(), Box> { + let path = Path::new(root).join(bucket).join(key).join("xl.meta"); + let bytes = tokio::fs::read(&path).await?; + let mut file_meta = rustfs_filemeta::FileMeta::load(&bytes)?; + let (index, mut version) = file_meta.find_version(None)?; + let object = version.object.as_mut().ok_or("fixture version is not an object")?; + mutate(object); + file_meta.versions[index] = rustfs_filemeta::FileMetaShallowVersion::try_from(version)?; + tokio::fs::write(path, file_meta.marshal_msg()?).await?; + Ok(()) + } + + async fn signed_admin_get( + env: &RustFSTestEnvironment, + path: &str, + ) -> Result> { + let url = format!("{}{path}", env.url); + let uri = url.parse::()?; + let authority = uri.authority().ok_or("admin URL missing authority")?.to_string(); + let request = http::Request::builder() + .method(http::Method::GET) + .uri(uri) + .header(HOST, authority) + .header("x-amz-content-sha256", UNSIGNED_PAYLOAD) + .body(Body::empty())?; + let signed = sign_v4(request, 0, &env.access_key, &env.secret_key, "", "us-east-1"); + + let mut request = local_http_client().get(&url); + for (name, value) in signed.headers() { + request = request.header(name, value); + } + Ok(request.send().await?) + } + + #[tokio::test] + async fn standard_and_rrs_are_supported_across_put_copy_and_multipart() -> Result<(), Box> { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(Vec::new()).await?; + let client = env.create_s3_client(); + let bucket = "storage-class-supported-contract"; + env.create_test_bucket(bucket).await?; + + client + .put_object() + .bucket(bucket) + .key("copy-source") + .body(ByteStream::from_static(b"copy-source-body")) + .send() + .await?; + + for storage_class in [StorageClass::Standard, StorageClass::ReducedRedundancy] { + let class_name = storage_class.as_str().to_string(); + let put_key = format!("put-{class_name}"); + let put_body = format!("put-body-{class_name}").into_bytes(); + client + .put_object() + .bucket(bucket) + .key(&put_key) + .storage_class(storage_class.clone()) + .body(ByteStream::from(put_body.clone())) + .send() + .await?; + assert_object_storage_class(&client, bucket, &put_key, &class_name, &put_body).await?; + + let copy_key = format!("copy-{class_name}"); + client + .copy_object() + .bucket(bucket) + .key(©_key) + .copy_source(format!("{bucket}/copy-source")) + .storage_class(storage_class.clone()) + .send() + .await?; + assert_object_storage_class(&client, bucket, ©_key, &class_name, b"copy-source-body").await?; + + let multipart_key = format!("multipart-{class_name}"); + let created = client + .create_multipart_upload() + .bucket(bucket) + .key(&multipart_key) + .storage_class(storage_class) + .send() + .await?; + let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?; + let parts = client + .list_parts() + .bucket(bucket) + .key(&multipart_key) + .upload_id(upload_id) + .send() + .await?; + assert_eq!(parts.storage_class().map(StorageClass::as_str), Some(class_name.as_str())); + client + .abort_multipart_upload() + .bucket(bucket) + .key(&multipart_key) + .upload_id(upload_id) + .send() + .await?; + } + + Ok(()) + } + + #[tokio::test] + async fn label_only_aws_classes_fail_before_put_copy_or_multipart_mutation() -> Result<(), Box> { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(Vec::new()).await?; + let client = env.create_s3_client(); + let bucket = "storage-class-unsupported-contract"; + env.create_test_bucket(bucket).await?; + + for key in ["put-guard", "copy-source", "copy-guard"] { + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from(format!("original-{key}").into_bytes())) + .send() + .await?; + } + + for unsupported in UNSUPPORTED_AWS_CLASSES { + let put_error = client + .put_object() + .bucket(bucket) + .key("put-guard") + .storage_class(StorageClass::from(unsupported)) + .body(ByteStream::from(format!("rejected-put-{unsupported}").into_bytes())) + .send() + .await + .expect_err("label-only PUT storage class must be rejected"); + assert_eq!( + put_error.as_service_error().and_then(ProvideErrorMetadata::code), + Some("InvalidStorageClass"), + "PUT returned a different error for {unsupported}" + ); + + let copy_error = client + .copy_object() + .bucket(bucket) + .key("copy-guard") + .copy_source(format!("{bucket}/copy-source")) + .storage_class(StorageClass::from(unsupported)) + .send() + .await + .expect_err("label-only CopyObject storage class must be rejected"); + assert_eq!( + copy_error.as_service_error().and_then(ProvideErrorMetadata::code), + Some("InvalidStorageClass"), + "CopyObject returned a different error for {unsupported}" + ); + + let multipart_key = format!("multipart-{unsupported}"); + let multipart_error = client + .create_multipart_upload() + .bucket(bucket) + .key(&multipart_key) + .storage_class(StorageClass::from(unsupported)) + .send() + .await + .expect_err("label-only CreateMultipartUpload storage class must be rejected"); + assert_eq!( + multipart_error.as_service_error().and_then(ProvideErrorMetadata::code), + Some("InvalidStorageClass"), + "CreateMultipartUpload returned a different error for {unsupported}" + ); + } + + let put_guard = client + .get_object() + .bucket(bucket) + .key("put-guard") + .send() + .await? + .body + .collect() + .await? + .into_bytes(); + assert_eq!(put_guard.as_ref(), b"original-put-guard"); + + let copy_guard = client + .get_object() + .bucket(bucket) + .key("copy-guard") + .send() + .await? + .body + .collect() + .await? + .into_bytes(); + assert_eq!(copy_guard.as_ref(), b"original-copy-guard"); + + let uploads = client.list_multipart_uploads().bucket(bucket).send().await?; + assert!(uploads.uploads().is_empty(), "unsupported classes must not create multipart sessions"); + Ok(()) + } + + #[tokio::test] + async fn historical_label_only_metadata_is_standard_without_hiding_a_real_transition_tier() + -> Result<(), Box> { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(Vec::new()).await?; + let client = env.create_s3_client(); + let bucket = "storage-class-historical-contract"; + let legacy_key = "legacy-label-only"; + let transitioned_key = "real-transition-tier"; + env.create_test_bucket(bucket).await?; + + for key in [legacy_key, transitioned_key] { + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"fixture-body")) + .send() + .await?; + } + + env.stop_server(); + mutate_xl_meta(&env.temp_dir, bucket, legacy_key, |object| { + object + .meta_user + .insert("x-amz-storage-class".to_string(), "STANDARD_IA".to_string()); + }) + .await?; + mutate_xl_meta(&env.temp_dir, bucket, transitioned_key, |object| { + object.set_transition(&rustfs_filemeta::FileInfo { + transition_status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(), + transition_tier: "STANDARD_IA".to_string(), + ..Default::default() + }); + }) + .await?; + env.restart_server_preserving_data(Vec::new(), &[]).await?; + + assert_object_storage_class(&client, bucket, legacy_key, "STANDARD", b"fixture-body").await?; + + let legacy_attributes = client + .get_object_attributes() + .bucket(bucket) + .key(legacy_key) + .object_attributes(ObjectAttributes::StorageClass) + .send() + .await?; + assert_eq!(legacy_attributes.storage_class().map(StorageClass::as_str), Some("STANDARD")); + + let versions = client.list_object_versions().bucket(bucket).prefix(legacy_key).send().await?; + let legacy_version = versions + .versions() + .iter() + .find(|version| version.key() == Some(legacy_key)) + .ok_or("legacy fixture missing from ListObjectVersions")?; + assert_eq!(legacy_version.storage_class().map(|class| class.as_str()), Some("STANDARD")); + + let transitioned_head = client.head_object().bucket(bucket).key(transitioned_key).send().await?; + assert_eq!(transitioned_head.storage_class().map(StorageClass::as_str), Some("STANDARD_IA")); + let transitioned_attributes = client + .get_object_attributes() + .bucket(bucket) + .key(transitioned_key) + .object_attributes(ObjectAttributes::StorageClass) + .send() + .await?; + assert_eq!(transitioned_attributes.storage_class().map(StorageClass::as_str), Some("STANDARD_IA")); + let transitioned_list = client + .list_objects_v2() + .bucket(bucket) + .prefix(transitioned_key) + .send() + .await?; + assert_eq!( + transitioned_list.contents()[0].storage_class().map(|class| class.as_str()), + Some("STANDARD_IA") + ); + let transitioned_versions = client + .list_object_versions() + .bucket(bucket) + .prefix(transitioned_key) + .send() + .await?; + assert_eq!( + transitioned_versions.versions()[0] + .storage_class() + .map(|class| class.as_str()), + Some("STANDARD_IA") + ); + Ok(()) + } + + #[tokio::test] + async fn authenticated_runtime_capabilities_publish_the_versioned_storage_class_contract() + -> Result<(), Box> { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(Vec::new()).await?; + let path = "/rustfs/admin/v4/runtime/capabilities"; + + let unsigned = local_http_client().get(format!("{}{path}", env.url)).send().await?; + assert_eq!(unsigned.status(), StatusCode::FORBIDDEN); + let unsigned_body = unsigned.text().await?; + assert!( + !unsigned_body.contains("supported_write_classes"), + "the capability contract must not bypass admin authentication" + ); + + let response = signed_admin_get(&env, path).await?; + assert_eq!(response.status(), StatusCode::OK); + let body: Value = response.json().await?; + assert_eq!(body["storage_classes"]["contract_version"], 1); + assert_eq!( + body["storage_classes"]["supported_write_classes"], + serde_json::json!(["STANDARD", "REDUCED_REDUNDANCY"]) + ); + assert_eq!(body["storage_classes"]["unsupported_write_error"], "InvalidStorageClass"); + assert_eq!(body["storage_classes"]["legacy_label_behavior"], "normalized_to_effective_class"); + Ok(()) + } +} diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 4f707c4eb..f16de641a 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -252,11 +252,12 @@ pub mod config { pub mod storageclass { pub use crate::config::storageclass::{ - CLASS_RRS, CLASS_STANDARD, Config, DEEP_ARCHIVE, DEFAULT_INLINE_BLOCK, DEFAULT_KVS, DEFAULT_RRS_PARITY, - EXPRESS_ONEZONE, GLACIER, GLACIER_IR, INLINE_BLOCK, INLINE_BLOCK_ENV, INTELLIGENT_TIERING, MIN_PARITY_DRIVES, - ONEZONE_IA, OPTIMIZE, OPTIMIZE_ENV, OUTPOSTS, RRS, RRS_ENV, SCHEME_PREFIX, SNOW, STANDARD, STANDARD_ENV, STANDARD_IA, - StorageClass, default_parity_count, lookup_config, lookup_config_for_pools, parse_storage_class, validate_parity, - validate_parity_inner, + CAPABILITY_CONTRACT_VERSION, CLASS_RRS, CLASS_STANDARD, Config, DEEP_ARCHIVE, DEFAULT_INLINE_BLOCK, DEFAULT_KVS, + DEFAULT_RRS_PARITY, EXPRESS_ONEZONE, GLACIER, GLACIER_IR, INLINE_BLOCK, INLINE_BLOCK_ENV, INTELLIGENT_TIERING, + LEGACY_LABEL_BEHAVIOR, MIN_PARITY_DRIVES, ONEZONE_IA, OPTIMIZE, OPTIMIZE_ENV, OUTPOSTS, RRS, RRS_ENV, SCHEME_PREFIX, + SNOW, STANDARD, STANDARD_ENV, STANDARD_IA, SUPPORTED_WRITE_CLASSES, StorageClass, UNSUPPORTED_WRITE_ERROR, + default_parity_count, effective_class, is_supported_write_class, lookup_config, lookup_config_for_pools, + parse_storage_class, validate_parity, validate_parity_inner, }; } diff --git a/crates/ecstore/src/config/storageclass.rs b/crates/ecstore/src/config/storageclass.rs index d14322901..11af87adf 100644 --- a/crates/ecstore/src/config/storageclass.rs +++ b/crates/ecstore/src/config/storageclass.rs @@ -46,6 +46,34 @@ pub const OUTPOSTS: &str = "OUTPOSTS"; pub const SNOW: &str = "SNOW"; pub const STANDARD_IA: &str = "STANDARD_IA"; +/// Version of the client-discoverable storage-class write contract. +pub const CAPABILITY_CONTRACT_VERSION: u32 = 1; +/// Storage classes whose write semantics RustFS implements. +pub const SUPPORTED_WRITE_CLASSES: [&str; 2] = [STANDARD, RRS]; +/// Stable S3 error code returned for unsupported write classes. +pub const UNSUPPORTED_WRITE_ERROR: &str = "InvalidStorageClass"; +/// Compatibility behavior applied to historical label-only object metadata. +pub const LEGACY_LABEL_BEHAVIOR: &str = "normalized_to_effective_class"; + +/// Returns whether a client may select this storage class for a write. +pub fn is_supported_write_class(storage_class: &str) -> bool { + SUPPORTED_WRITE_CLASSES.contains(&storage_class) +} + +/// Resolves the storage class that truthfully describes the stored object. +/// +/// A completed lifecycle transition is a real storage tier and therefore keeps +/// its tier name. For local objects, only RRS has distinct layout semantics; +/// historical AWS class labels otherwise describe the effective STANDARD +/// layout. +pub fn effective_class<'a>(stored_class: Option<&'a str>, transitioned_tier: Option<&'a str>) -> &'a str { + if let Some(tier) = transitioned_tier.filter(|tier| !tier.is_empty()) { + return tier; + } + + if stored_class == Some(RRS) { RRS } else { STANDARD } +} + // Standard constants for config info storage class pub const CLASS_STANDARD: &str = "standard"; pub const CLASS_RRS: &str = "rrs"; @@ -556,6 +584,47 @@ mod tests { } } + #[test] + fn write_capability_contract_only_accepts_implemented_layouts() { + assert_eq!(SUPPORTED_WRITE_CLASSES, [STANDARD, RRS]); + assert!(is_supported_write_class(STANDARD)); + assert!(is_supported_write_class(RRS)); + + for label_only_class in [ + DEEP_ARCHIVE, + EXPRESS_ONEZONE, + GLACIER, + GLACIER_IR, + INTELLIGENT_TIERING, + ONEZONE_IA, + OUTPOSTS, + SNOW, + STANDARD_IA, + ] { + assert!( + !is_supported_write_class(label_only_class), + "{label_only_class} must not be advertised as a supported write class" + ); + } + assert!(!is_supported_write_class("")); + assert!(!is_supported_write_class("standard")); + assert!(!is_supported_write_class("UNKNOWN")); + } + + #[test] + fn effective_class_normalizes_legacy_labels_and_preserves_real_tiers() { + assert_eq!(effective_class(None, None), STANDARD); + assert_eq!(effective_class(Some(STANDARD), None), STANDARD); + assert_eq!(effective_class(Some(RRS), None), RRS); + assert_eq!(effective_class(Some(STANDARD_IA), None), STANDARD); + assert_eq!(effective_class(Some(GLACIER), None), STANDARD); + assert_eq!(effective_class(Some("UNKNOWN"), None), STANDARD); + + assert_eq!(effective_class(Some(STANDARD_IA), Some("WARM-TIER")), "WARM-TIER"); + assert_eq!(effective_class(Some(STANDARD), Some(STANDARD_IA)), STANDARD_IA); + assert_eq!(effective_class(Some(RRS), Some("CUSTOM-RRS-TIER")), "CUSTOM-RRS-TIER"); + } + #[test] fn automatic_parity_is_resolved_per_pool() { let cfg = lookup_config_for_pools_with_env(&KVS::new(), &[4, 2], no_env_overrides()) diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 7091f9b49..a53cc16ae 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -478,15 +478,14 @@ impl ObjectInfo { v }; - // Extract storage class from metadata, default to STANDARD if not found - let storage_class = if !fi.transition_tier.is_empty() { - Some(fi.transition_tier.clone()) - } else { - fi.metadata - .get(AMZ_STORAGE_CLASS) - .cloned() - .or_else(|| Some(storageclass::STANDARD.to_string())) - }; + let storage_class = Some( + storageclass::effective_class( + fi.metadata.get(AMZ_STORAGE_CLASS).map(String::as_str), + (fi.transition_status == rustfs_filemeta::TRANSITION_COMPLETE && !fi.transition_tier.is_empty()) + .then_some(fi.transition_tier.as_str()), + ) + .to_string(), + ); let mut restore_ongoing = false; let mut restore_expires = None; @@ -1145,6 +1144,58 @@ mod tests { assert_eq!(info.replication_decision, "arn=true;false;arn:replication::1:dest;rule-id"); } + #[test] + fn from_file_info_reports_effective_storage_class_for_legacy_metadata() { + for legacy_label in [ + storageclass::STANDARD_IA, + storageclass::ONEZONE_IA, + storageclass::INTELLIGENT_TIERING, + storageclass::GLACIER, + ] { + let fi = FileInfo { + metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), legacy_label.to_string())]), + ..Default::default() + }; + + let info = ObjectInfo::from_file_info(&fi, "bucket", "legacy-object", true); + + assert_eq!( + info.storage_class.as_deref(), + Some(storageclass::STANDARD), + "{legacy_label} was only a label and must report the effective STANDARD layout" + ); + } + } + + #[test] + fn from_file_info_preserves_transitioned_tier_storage_class() { + let fi = FileInfo { + metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string())]), + transition_tier: "WARM-TIER".to_string(), + transition_status: TRANSITION_COMPLETE.to_string(), + ..Default::default() + }; + + let info = ObjectInfo::from_file_info(&fi, "bucket", "transitioned-object", true); + + assert_eq!(info.storage_class.as_deref(), Some("WARM-TIER")); + assert_eq!(info.transitioned_object.tier, "WARM-TIER"); + } + + #[test] + fn from_file_info_ignores_a_tier_name_without_a_completed_transition() { + let fi = FileInfo { + metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string())]), + transition_tier: "WARM-TIER".to_string(), + ..Default::default() + }; + + let info = ObjectInfo::from_file_info(&fi, "bucket", "incomplete-transition", true); + + assert_eq!(info.storage_class.as_deref(), Some(storageclass::STANDARD)); + assert_eq!(info.transitioned_object.tier, "WARM-TIER"); + } + #[test] fn get_actual_size_uses_compressed_parts_actual_size_when_metadata_missing() { let user_defined = { diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 405c7230c..9cb0c690e 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -4408,20 +4408,7 @@ pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option, if_m /// Validates if the given storage class is supported pub fn is_valid_storage_class(storage_class: &str) -> bool { - matches!( - storage_class, - storageclass::STANDARD - | storageclass::RRS - | storageclass::DEEP_ARCHIVE - | storageclass::EXPRESS_ONEZONE - | storageclass::GLACIER - | storageclass::GLACIER_IR - | storageclass::INTELLIGENT_TIERING - | storageclass::ONEZONE_IA - | storageclass::OUTPOSTS - | storageclass::SNOW - | storageclass::STANDARD_IA - ) + storageclass::is_supported_write_class(storage_class) } /// Returns true if the storage class is a cold storage tier that requires special handling @@ -7796,23 +7783,10 @@ mod tests { #[test] fn test_is_valid_storage_class() { - // Test valid storage classes assert!(is_valid_storage_class(storageclass::STANDARD)); assert!(is_valid_storage_class(storageclass::RRS)); - assert!(is_valid_storage_class(storageclass::DEEP_ARCHIVE)); - assert!(is_valid_storage_class(storageclass::EXPRESS_ONEZONE)); - assert!(is_valid_storage_class(storageclass::GLACIER)); - assert!(is_valid_storage_class(storageclass::GLACIER_IR)); - assert!(is_valid_storage_class(storageclass::INTELLIGENT_TIERING)); - assert!(is_valid_storage_class(storageclass::ONEZONE_IA)); - assert!(is_valid_storage_class(storageclass::OUTPOSTS)); - assert!(is_valid_storage_class(storageclass::SNOW)); - assert!(is_valid_storage_class(storageclass::STANDARD_IA)); - - // Test invalid storage classes + assert!(!is_valid_storage_class(storageclass::STANDARD_IA)); assert!(!is_valid_storage_class("INVALID")); - assert!(!is_valid_storage_class("")); - assert!(!is_valid_storage_class("standard")); // lowercase } #[test] diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index f7c323006..11f50c148 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -309,7 +309,8 @@ const MAX_LIST_OBJECTS_METADATA_FAST_STALENESS_MS: u64 = 60_000; const LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY: &str = "walker_key_only"; const LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY: &str = "persistent_key_only"; const LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY_DEFAULT_GENERATION: &str = "persistent-key-only"; -const PERSISTENT_KEY_ONLY_INDEX_HEADER: &str = "# rustfs-listobjects-key-only-v1"; +const PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION: u8 = 2; +const PERSISTENT_KEY_ONLY_INDEX_HEADER: &str = "# rustfs-listobjects-key-only-v2"; const PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER: &str = "# bucket="; const PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER: &str = "# generation="; const PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER: &str = "# checkpoint_high_water_mark="; @@ -502,6 +503,7 @@ struct PersistentKeyOnlyIndexCache { #[derive(Debug, Clone, PartialEq, Eq)] struct PersistentKeyOnlyIndex { + format_version: u8, bucket: Option, generation: String, checkpoint_high_water_mark: u64, @@ -1410,6 +1412,7 @@ fn parse_persistent_list_metadata_object(line: &str) -> Option PersistentKeyOnlyIndex { + let mut format_version = 0; let mut bucket = None; let mut generation = None; let mut checkpoint_high_water_mark = None; @@ -1421,6 +1424,10 @@ fn parse_persistent_key_only_index(contents: &str) -> PersistentKeyOnlyIndex { if line.is_empty() { continue; } + if line == PERSISTENT_KEY_ONLY_INDEX_HEADER { + format_version = PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION; + continue; + } if let Some(object) = parse_persistent_list_metadata_object(line) { keys.push(object.name.clone()); objects.push(object); @@ -1455,6 +1462,7 @@ fn parse_persistent_key_only_index(contents: &str) -> PersistentKeyOnlyIndex { let checkpoint_high_water_mark = checkpoint_high_water_mark.unwrap_or_else(|| u64::try_from(keys.len()).unwrap_or(u64::MAX)); PersistentKeyOnlyIndex { + format_version, bucket, generation: generation.unwrap_or_else(|| LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY_DEFAULT_GENERATION.to_owned()), checkpoint_high_water_mark, @@ -1485,6 +1493,9 @@ fn persistent_key_only_index_matches_provider( bucket: &str, provider_state: &ListObjectsIndexProviderState, ) -> bool { + if index.format_version != PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION { + return false; + } if index.bucket.as_deref().is_some_and(|index_bucket| index_bucket != bucket) { return false; } @@ -1579,6 +1590,7 @@ async fn write_persistent_key_only_index_with_metadata( tokio::fs::rename(&tmp_path, path).await.map_err(Error::Io)?; Ok(PersistentKeyOnlyIndex { + format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION, bucket: Some(bucket.to_owned()), generation: generation.to_owned(), checkpoint_high_water_mark, @@ -6687,13 +6699,13 @@ mod test { ListObjectsIndexProviderState, ListPathOptions, ListPathRawOptions, ListSourceMode, ListingEntryResolution, ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend, NamespaceMutationJournalSnapshot, NamespaceMutationJournalStatus, PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER, - PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER, - PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, PersistentListMetadataObject, RUSTFS_META_BUCKET, - VerifiedIndexCandidateStats, VersionMarker, current_list_objects_mutation_sequence, - encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum, - fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects, latest_listing_object_quorum, - latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key, list_merged_entry_channel, - list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates, + PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION, + PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER, PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, + PersistentListMetadataObject, RUSTFS_META_BUCKET, VerifiedIndexCandidateStats, VersionMarker, + current_list_objects_mutation_sequence, encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum, + expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects, + latest_listing_object_quorum, latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key, + list_merged_entry_channel, list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates, list_objects_from_verified_index_candidates, list_objects_from_verified_index_candidates_with_optional_stats, list_objects_from_verified_index_candidates_with_stats, list_objects_index_mode_from_env, list_objects_index_provider_from_env, list_objects_index_provider_state_from_env, list_objects_key_only_provider_health, @@ -8041,6 +8053,7 @@ mod test { #[test] fn metadata_fast_requires_complete_metadata_snapshot() { let index = PersistentKeyOnlyIndex { + format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION, bucket: Some("bucket".to_string()), generation: "generation-42".to_string(), checkpoint_high_water_mark: 42, @@ -8057,6 +8070,7 @@ mod test { assert!(!persistent_key_only_index_has_complete_metadata_snapshot(&index)); let complete = PersistentKeyOnlyIndex { + format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION, bucket: Some("bucket".to_string()), generation: "generation-42".to_string(), checkpoint_high_water_mark: 42, @@ -8402,6 +8416,7 @@ mod test { #[test] fn persistent_key_only_index_provider_match_rejects_configured_generation_mismatch() { let index = PersistentKeyOnlyIndex { + format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION, bucket: Some("bucket".to_string()), generation: "generation-old".to_string(), checkpoint_high_water_mark: 42, @@ -8422,9 +8437,50 @@ mod test { assert!(!persistent_key_only_index_matches_provider(&index, "other-bucket", &matching_provider)); } + #[test] + fn persistent_key_only_index_provider_match_rejects_legacy_format_without_configured_generation() { + let legacy = parse_persistent_key_only_index(&format!( + "# rustfs-listobjects-key-only-v1\n\ + {PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER}bucket\n\ + {PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER}generation-old\n\ + {PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER}42\n\ + # object\tbGVnYWN5\t1\t-\t-\tU1RBTkRBUkRfSUE=\n" + )); + let provider = + ListObjectsIndexProviderState::persistent_key_only(Some(PathBuf::from("/tmp/persistent-key-only.index")), None); + + assert_eq!(legacy.format_version, 0); + assert!(!persistent_key_only_index_matches_provider(&legacy, "bucket", &provider)); + } + + #[test] + fn current_persistent_snapshot_preserves_an_effective_transition_tier_name() { + let object = PersistentListMetadataObject::from_object_info(&ObjectInfo { + bucket: "bucket".to_string(), + name: "transitioned".to_string(), + storage_class: Some("STANDARD_IA".to_string()), + ..Default::default() + }); + let index = parse_persistent_key_only_index(&format!( + "{PERSISTENT_KEY_ONLY_INDEX_HEADER}\n\ + {PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER}bucket\n\ + {PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER}generation-current\n\ + {PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER}42\n\ + {}\n", + encode_persistent_list_metadata_object(&object) + )); + let provider = + ListObjectsIndexProviderState::persistent_key_only(Some(PathBuf::from("/tmp/persistent-key-only.index")), None); + + assert_eq!(index.format_version, PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION); + assert!(persistent_key_only_index_matches_provider(&index, "bucket", &provider)); + assert_eq!(index.objects[0].to_object_info("bucket").storage_class.as_deref(), Some("STANDARD_IA")); + } + #[test] fn persistent_key_only_index_health_uses_snapshot_generation_and_checkpoint() { let index = PersistentKeyOnlyIndex { + format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION, bucket: Some("bucket".to_string()), generation: "generation-42".to_string(), checkpoint_high_water_mark: 42, @@ -8450,6 +8506,7 @@ mod test { #[test] fn persistent_key_only_index_health_reports_lagging_mutation_checkpoint() { let index = PersistentKeyOnlyIndex { + format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION, bucket: Some("bucket".to_string()), generation: "generation-42".to_string(), checkpoint_high_water_mark: 42, @@ -8476,6 +8533,7 @@ mod test { #[test] fn persistent_key_only_index_health_reports_degraded_journal() { let index = PersistentKeyOnlyIndex { + format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION, bucket: Some("bucket".to_string()), generation: "generation-42".to_string(), checkpoint_high_water_mark: 42, diff --git a/docs/architecture/erasure-coding.md b/docs/architecture/erasure-coding.md index b35dd904c..c5f1b4e11 100644 --- a/docs/architecture/erasure-coding.md +++ b/docs/architecture/erasure-coding.md @@ -75,6 +75,9 @@ Default parity by drive count — `default_parity_count(N)` ([storageclass.rs](. Two storage classes: `STANDARD` (SC) and `REDUCED_REDUNDANCY` (RRS) ([storageclass.rs](../../crates/ecstore/src/config/storageclass.rs)), configured as `"EC:"` via the `standard` / `rrs` config keys or the `RUSTFS_STORAGE_CLASS_STANDARD` / `RUSTFS_STORAGE_CLASS_RRS` env overrides. Absent config falls back to `default_parity_count` (SC) and `1`, or `0` on a single drive (RRS). +- **INVARIANT — truthful client write classes.** S3 PUT, CopyObject, and CreateMultipartUpload accept only `STANDARD` and `REDUCED_REDUNDANCY`. AWS labels such as `STANDARD_IA`, `ONEZONE_IA`, `INTELLIGENT_TIERING`, `GLACIER`, and `DEEP_ARCHIVE` are rejected with `InvalidStorageClass` because RustFS does not implement their advertised access, retrieval, or archival semantics. The supported allowlist and stable error identifier are owned by [storageclass.rs](../../crates/ecstore/src/config/storageclass.rs), not duplicated by individual handlers. +- **INVARIANT — historical label normalization.** Non-transitioned objects written by older RustFS versions may contain an AWS storage-class label without distinct physical semantics. Read and listing responses report the effective local layout (`STANDARD`, or `REDUCED_REDUNDANCY` when that layout was selected) instead of repeating a label-only promise. A completed lifecycle transition is different: its real transition tier name is preserved. Persistent metadata-fast list snapshots use the `rustfs-listobjects-key-only-v2` header; older or unknown snapshot formats are rebuilt from normalized `ObjectInfo` values instead of being served, because their stored label lacks enough information to distinguish historical metadata from a real transition tier. + - **INVARIANT — parity bounds.** Parity must satisfy `parity ≤ N/2` for both classes, and `SC parity ≥ RRS parity` when both are non-zero ([storageclass.rs](../../crates/ecstore/src/config/storageclass.rs), `validate_parity` / `validate_parity_inner`). Enforcement nuance to be aware of: `validate_parity_inner` (the path a user-configured `EC:` storage class flows through) only applies the `parity ≤ N/2` check for `N > 2`, so degenerate small-set values (e.g. `EC:2` on `N = 2`, giving `data_blocks = 0`) are not caught there; the standalone `validate_parity` enforces the bound unconditionally but is applied only to the resolved default parity. A change that lets user-configured parity reach a write path must not assume the `≤ N/2` bound was enforced for `N ≤ 2`. Parity `0` is permitted (single-drive / capacity setups); there is no non-zero minimum. - **INVARIANT — per-pool validity.** Each pool's resolved parity must be valid for **that pool's own drive count**. A heterogeneous deployment (pools of different widths) must resolve parity per pool; applying one pool's parity to a narrower pool can drive `data_blocks = N − parity` to `0` and make encoding impossible. - Baseline defect: `main` computes `common_parity_drives` from the **first** pool only and applies it to every pool ([store/init.rs](../../crates/ecstore/src/store/init.rs), `ec_drives_no_config` at [store/init_format.rs](../../crates/ecstore/src/store/init_format.rs)); this is issue #4801 (a smaller later pool panics with `TooFewDataShards`). The correct rule is per-pool resolution. diff --git a/docs/architecture/runtime-capability-contracts.md b/docs/architecture/runtime-capability-contracts.md index 29eb6f56c..f67069d19 100644 --- a/docs/architecture/runtime-capability-contracts.md +++ b/docs/architecture/runtime-capability-contracts.md @@ -59,3 +59,33 @@ owners through read-only providers: Unsupported or unavailable runtime capabilities are reported as `unsupported` or `unknown` contract states instead of activating fallback behavior. + +## Storage-Class Write Contract + +Authenticated clients discover the storage-class write contract from +`GET /rustfs/admin/v4/runtime/capabilities`. The additive +`storage_classes` object is versioned independently from the route: + +```json +{ + "storage_classes": { + "contract_version": 1, + "supported_write_classes": ["STANDARD", "REDUCED_REDUNDANCY"], + "unsupported_write_error": "InvalidStorageClass", + "legacy_label_behavior": "normalized_to_effective_class" + } +} +``` + +`supported_write_classes` is the complete client-selectable write allowlist. +Any other value fails before object or multipart mutation with the stable S3 +error named by `unsupported_write_error`. `legacy_label_behavior` means +non-transitioned historical label-only metadata is reported as its effective +local class; actual lifecycle transition tier names remain unchanged. + +The values are sourced from +[`crates/ecstore/src/config/storageclass.rs`](../../crates/ecstore/src/config/storageclass.rs), +which also owns write validation and response normalization. Consumers must +branch on `contract_version` before assigning meaning to future fields. The +admin route continues to require `ServerInfoAdminAction`; capability discovery +does not weaken authentication or authorization. diff --git a/docs/testing/e2e-suite-inventory.md b/docs/testing/e2e-suite-inventory.md index 92e02fb14..c84e34a02 100644 --- a/docs/testing/e2e-suite-inventory.md +++ b/docs/testing/e2e-suite-inventory.md @@ -79,10 +79,11 @@ | snowball_auto_extract_test | 6 | | | special_chars_test | 14 | ✅ | | stale_multipart_cleanup_cluster_test | 1 | | +| storage_class_capability_test | 4 | ✅ | | tls_gen | 3 | | | tls_hot_reload_test | 1 | ✅ | | version_id_regression_test | 10 | ✅ | `notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above. -**Total listed: 475 tests across 65 modules · PR smoke subset: 122 tests / 30 modules** (28 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-24. +**Total listed: 479 tests across 66 modules · PR smoke subset: 126 tests / 31 modules** (29 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-24. diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index f37bbc7ff..bf03f61fd 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -24,6 +24,7 @@ use crate::admin::runtime_sources::{ use crate::admin::storage_api::cluster::{ CapabilityState, CapabilityStatus, ObservabilitySnapshotProvider, TopologySnapshot, TopologySnapshotProvider, }; +use crate::admin::storage_api::storageclass as storage_class_contract; use crate::auth::{check_key_valid, get_session_token}; use crate::runtime_capabilities::{EndpointTopologySnapshotProvider, RustFsObservabilitySnapshotProvider}; use crate::server::{ADMIN_PREFIX, RemoteAddr}; @@ -635,6 +636,7 @@ pub struct RuntimeCapabilitiesSummary { #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct RuntimeCapabilitiesResponse { pub summary: RuntimeCapabilitiesSummary, + pub storage_classes: StorageClassCapabilities, pub cluster_snapshot_path: String, pub cluster_snapshot_summary: Option, pub observability: crate::admin::storage_api::cluster::ObservabilitySnapshot, @@ -643,6 +645,25 @@ pub struct RuntimeCapabilitiesResponse { pub topology_status: CapabilityStatus, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct StorageClassCapabilities { + pub contract_version: u32, + pub supported_write_classes: [&'static str; 2], + pub unsupported_write_error: &'static str, + pub legacy_label_behavior: &'static str, +} + +impl StorageClassCapabilities { + fn current() -> Self { + Self { + contract_version: storage_class_contract::CAPABILITY_CONTRACT_VERSION, + supported_write_classes: storage_class_contract::SUPPORTED_WRITE_CLASSES, + unsupported_write_error: storage_class_contract::UNSUPPORTED_WRITE_ERROR, + legacy_label_behavior: storage_class_contract::LEGACY_LABEL_BEHAVIOR, + } + } +} + pub struct RuntimeCapabilitiesHandler {} pub(crate) async fn build_runtime_capabilities_response() @@ -669,6 +690,7 @@ pub(crate) async fn build_runtime_capabilities_response() Ok(RuntimeCapabilitiesResponse { summary, + storage_classes: StorageClassCapabilities::current(), cluster_snapshot_path: usecase.cluster_snapshot_route().to_string(), cluster_snapshot_summary: cluster_snapshot_discovery.summary, observability, @@ -931,11 +953,22 @@ mod tests { assert_eq!(response.summary.site_replication_info.state, CapabilityState::Supported); assert_eq!(response.summary.site_replication_edit.state, CapabilityState::Supported); assert_eq!(response.summary.site_replication_resync.state, CapabilityState::Supported); + assert_eq!(response.storage_classes.contract_version, 1); + assert_eq!(response.storage_classes.supported_write_classes, ["STANDARD", "REDUCED_REDUNDANCY"]); + assert_eq!(response.storage_classes.unsupported_write_error, "InvalidStorageClass"); + assert_eq!(response.storage_classes.legacy_label_behavior, "normalized_to_effective_class"); let value = serde_json::to_value(response).expect("runtime capability response should serialize"); assert_eq!(value["summary"]["site_replication_info"]["state"], "supported"); assert_eq!(value["summary"]["site_replication_edit"]["state"], "supported"); assert_eq!(value["summary"]["site_replication_resync"]["state"], "supported"); + assert_eq!(value["storage_classes"]["contract_version"], 1); + assert_eq!( + value["storage_classes"]["supported_write_classes"], + json!(["STANDARD", "REDUCED_REDUNDANCY"]) + ); + assert_eq!(value["storage_classes"]["unsupported_write_error"], "InvalidStorageClass"); + assert_eq!(value["storage_classes"]["legacy_label_behavior"], "normalized_to_effective_class"); } #[test] diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index eaa916932..33c6f9651 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -390,9 +390,11 @@ pub(crate) mod versioning_sys { } pub(crate) mod storageclass { + pub(crate) const CAPABILITY_CONTRACT_VERSION: u32 = super::ecstore_config::storageclass::CAPABILITY_CONTRACT_VERSION; #[cfg(test)] pub(crate) const CLASS_STANDARD: &str = super::ecstore_config::storageclass::CLASS_STANDARD; pub(crate) const INLINE_BLOCK_ENV: &str = super::ecstore_config::storageclass::INLINE_BLOCK_ENV; + pub(crate) const LEGACY_LABEL_BEHAVIOR: &str = super::ecstore_config::storageclass::LEGACY_LABEL_BEHAVIOR; pub(crate) const OPTIMIZE_ENV: &str = super::ecstore_config::storageclass::OPTIMIZE_ENV; #[cfg(test)] pub(crate) const RRS: &str = super::ecstore_config::storageclass::RRS; @@ -400,6 +402,8 @@ pub(crate) mod storageclass { #[cfg(test)] pub(crate) const STANDARD: &str = super::ecstore_config::storageclass::STANDARD; pub(crate) const STANDARD_ENV: &str = super::ecstore_config::storageclass::STANDARD_ENV; + pub(crate) const SUPPORTED_WRITE_CLASSES: [&str; 2] = super::ecstore_config::storageclass::SUPPORTED_WRITE_CLASSES; + pub(crate) const UNSUPPORTED_WRITE_ERROR: &str = super::ecstore_config::storageclass::UNSUPPORTED_WRITE_ERROR; pub(crate) type Config = super::ecstore_config::storageclass::Config; diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 4b812e94c..45cb1ab0f 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -2778,11 +2778,16 @@ fn apply_put_request_metadata( } fn response_storage_class(info: &ObjectInfo, metadata: &HashMap) -> Option { - info.storage_class - .clone() - .or_else(|| metadata.get(AMZ_STORAGE_CLASS).cloned()) - .filter(|storage_class| !storage_class.is_empty() && storage_class != storageclass::STANDARD) - .map(StorageClass::from) + let stored_class = info + .storage_class + .as_deref() + .or_else(|| metadata.get(AMZ_STORAGE_CLASS).map(String::as_str)); + let transitioned_tier = (info.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE + && !info.transitioned_object.tier.is_empty()) + .then_some(info.transitioned_object.tier.as_str()); + let effective_class = storageclass::effective_class(stored_class, transitioned_tier); + + (effective_class != storageclass::STANDARD).then(|| StorageClass::from(effective_class.to_string())) } fn response_storage_class_for_object_attributes( @@ -2794,12 +2799,17 @@ fn response_storage_class_for_object_attributes( return None; } - info.storage_class - .clone() - .or_else(|| metadata.get(AMZ_STORAGE_CLASS).cloned()) - .or_else(|| Some(storageclass::STANDARD.to_string())) - .filter(|storage_class| !storage_class.is_empty()) - .map(StorageClass::from) + let stored_class = info + .storage_class + .as_deref() + .or_else(|| metadata.get(AMZ_STORAGE_CLASS).map(String::as_str)); + let transitioned_tier = (info.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE + && !info.transitioned_object.tier.is_empty()) + .then_some(info.transitioned_object.tier.as_str()); + + Some(StorageClass::from( + storageclass::effective_class(stored_class, transitioned_tier).to_string(), + )) } async fn apply_put_request_object_lock_opts( @@ -12286,7 +12296,7 @@ mod tests { } #[test] - fn response_storage_class_omits_standard_and_keeps_non_default() { + fn response_storage_class_reports_effective_layout_and_preserves_transition_tier() { let metadata = HashMap::new(); let standard_info = ObjectInfo { storage_class: Some(storageclass::STANDARD.to_string()), @@ -12297,16 +12307,39 @@ mod tests { let mut metadata = HashMap::new(); metadata.insert(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string()); - let infrequent_access_info = ObjectInfo { + let label_only_info = ObjectInfo { storage_class: Some(storageclass::STANDARD_IA.to_string()), user_defined: Arc::new(metadata.clone()), ..Default::default() }; + assert!( + response_storage_class(&label_only_info, &metadata).is_none(), + "historical STANDARD_IA labels must report the effective implicit STANDARD layout" + ); + + let rrs_info = ObjectInfo { + storage_class: Some(storageclass::RRS.to_string()), + ..Default::default() + }; assert_eq!( - response_storage_class(&infrequent_access_info, &metadata) + response_storage_class(&rrs_info, &HashMap::new()) .as_ref() .map(StorageClass::as_str), - Some(storageclass::STANDARD_IA) + Some(storageclass::RRS) + ); + + let mut transitioned_info = label_only_info; + transitioned_info.transitioned_object.tier = "WARM-TIER".to_string(); + assert!( + response_storage_class(&transitioned_info, &metadata).is_none(), + "a tier name without a completed transition must not override the effective local class" + ); + transitioned_info.transitioned_object.status = rustfs_filemeta::TRANSITION_COMPLETE.to_string(); + assert_eq!( + response_storage_class(&transitioned_info, &metadata) + .as_ref() + .map(StorageClass::as_str), + Some("WARM-TIER") ); let mut metadata = HashMap::new(); @@ -12337,6 +12370,17 @@ mod tests { .map(StorageClass::as_str), Some(storageclass::STANDARD) ); + + let legacy_info = ObjectInfo { + storage_class: Some(storageclass::STANDARD_IA.to_string()), + ..Default::default() + }; + assert_eq!( + response_storage_class_for_object_attributes(&legacy_info, &HashMap::new(), true) + .as_ref() + .map(StorageClass::as_str), + Some(storageclass::STANDARD) + ); } #[test] @@ -12575,7 +12619,7 @@ mod tests { }) .bucket("test-bucket".to_string()) .key("test-key".to_string()) - .storage_class(Some(StorageClass::from_static(storageclass::STANDARD_IA))) + .storage_class(Some(StorageClass::from_static(storageclass::RRS))) .build() .unwrap(); diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 9eef0fb74..c0a65439d 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -903,9 +903,9 @@ pub(crate) mod set_disk { } pub(crate) mod storage_class { - pub(crate) use crate::storage::storage_api::ecstore_config::storageclass::STANDARD; #[cfg(test)] - pub(crate) use crate::storage::storage_api::ecstore_config::storageclass::STANDARD_IA; + pub(crate) use crate::storage::storage_api::ecstore_config::storageclass::{RRS, STANDARD_IA}; + pub(crate) use crate::storage::storage_api::ecstore_config::storageclass::{STANDARD, effective_class}; } pub(crate) mod timeout_wrapper { diff --git a/rustfs/src/storage/s3_api/bucket.rs b/rustfs/src/storage/s3_api/bucket.rs index 1eb806955..e105adba4 100644 --- a/rustfs/src/storage/s3_api/bucket.rs +++ b/rustfs/src/storage/s3_api/bucket.rs @@ -449,6 +449,8 @@ mod tests { use crate::storage::s3_api::common::rustfs_owner; use crate::storage::storage_api::s3_api_consumer::bucket::StorageObjectInfo as ObjectInfo; use crate::storage::storage_api::s3_api_consumer::bucket::contract::bucket::BucketInfo; + use rustfs_filemeta::FileInfo; + use rustfs_utils::http::headers::AMZ_STORAGE_CLASS; use s3s::S3ErrorCode; use s3s::dto::{CommonPrefix, EncodingType, ListObjectsV2Output, Object}; use time::OffsetDateTime; @@ -654,6 +656,67 @@ mod tests { assert_eq!(output.common_prefixes.as_ref().map(std::vec::Vec::len), Some(2)); } + #[test] + fn list_responses_report_standard_for_legacy_label_only_file_metadata() { + let version_id = Uuid::parse_str("11111111-2222-3333-4444-555555555555").expect("fixture version ID should be valid"); + let file_info = FileInfo { + name: "legacy-object".to_string(), + version_id: Some(version_id), + metadata: std::collections::HashMap::from([(AMZ_STORAGE_CLASS.to_string(), "STANDARD_IA".to_string())]), + ..Default::default() + }; + let object_info = ObjectInfo::from_file_info(&file_info, "bucket", "legacy-object", true); + + let list_output = build_list_objects_v2_output( + ListObjectsV2Info { + objects: vec![object_info.clone()], + ..Default::default() + }, + false, + 1000, + "bucket".to_string(), + String::new(), + None, + None, + None, + None, + ); + assert_eq!( + list_output + .contents + .as_ref() + .and_then(|objects| objects.first()) + .and_then(|object| object.storage_class.as_ref()) + .map(|storage_class| storage_class.as_str()), + Some("STANDARD") + ); + + let versions_output = build_list_object_versions_output( + ListObjectVersionsInfo { + objects: vec![object_info], + ..Default::default() + }, + "bucket".to_string(), + &ListObjectVersionsParams { + prefix: String::new(), + delimiter: None, + key_marker: None, + version_id_marker: None, + max_keys: 1000, + }, + None, + ); + assert_eq!( + versions_output + .versions + .as_ref() + .and_then(|versions| versions.first()) + .and_then(|version| version.storage_class.as_ref()) + .map(|storage_class| storage_class.as_str()), + Some("STANDARD") + ); + } + #[test] fn test_list_objects_v2_url_encoding_preserves_slash() { let object_infos = ListObjectsV2Info { diff --git a/rustfs/src/storage/s3_api/multipart.rs b/rustfs/src/storage/s3_api/multipart.rs index fe9352ef4..ee2cc75b1 100644 --- a/rustfs/src/storage/s3_api/multipart.rs +++ b/rustfs/src/storage/s3_api/multipart.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; +use crate::storage::storage_api::effective_storage_class; use crate::storage::storage_api::s3_api_consumer::multipart::contract::multipart::{ ListMultipartsInfo, ListPartsInfo, MAX_MULTIPART_PART_NUMBER, }; @@ -61,11 +62,11 @@ pub(crate) fn build_list_parts_output(res: ListPartsInfo) -> ListPartsOutput { next_part_number_marker: res.next_part_number_marker.try_into().ok(), max_parts: res.max_parts.try_into().ok(), part_number_marker: res.part_number_marker.try_into().ok(), - storage_class: if res.storage_class.is_empty() { - None - } else { - Some(res.storage_class.into()) - }, + storage_class: Some( + effective_storage_class((!res.storage_class.is_empty()).then_some(res.storage_class.as_str()), None) + .to_string() + .into(), + ), ..Default::default() } } @@ -245,9 +246,9 @@ mod tests { } #[test] - fn test_list_parts_output_handles_empty_storage_class_and_overflow_markers() { + fn test_list_parts_output_normalizes_legacy_storage_class_and_handles_overflow_markers() { let input = ListPartsInfo { - storage_class: String::new(), + storage_class: "STANDARD_IA".to_string(), part_number_marker: usize::MAX, next_part_number_marker: usize::MAX, max_parts: usize::MAX, @@ -262,13 +263,20 @@ mod tests { let output = build_list_parts_output(input); let parts = output.parts.as_ref().expect("parts should be present"); - assert_eq!(output.storage_class, None); + assert_eq!(output.storage_class.as_ref().map(|value| value.as_str()), Some("STANDARD")); assert_eq!(output.part_number_marker, None); assert_eq!(output.next_part_number_marker, None); assert_eq!(output.max_parts, None); assert_eq!(parts.len(), 1); assert_eq!(parts[0].part_number, None); assert_eq!(parts[0].size, None); + + let output = build_list_parts_output(ListPartsInfo::default()); + assert_eq!( + output.storage_class.as_ref().map(|value| value.as_str()), + Some("STANDARD"), + "legacy uploads without a stored class must report their effective STANDARD layout" + ); } #[test] diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 793f631c6..0c9381a50 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -1512,6 +1512,10 @@ pub(crate) fn is_valid_storage_class(storage_class: &str) -> bool { ecstore_set_disk::is_valid_storage_class(storage_class) } +pub(crate) fn effective_storage_class<'a>(stored_class: Option<&'a str>, completed_transition_tier: Option<&'a str>) -> &'a str { + ecstore_config::storageclass::effective_class(stored_class, completed_transition_tier) +} + pub(crate) fn register_event_dispatch_hook(hook: F) -> bool where F: Fn(EventArgs) + Send + Sync + 'static,