gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-17 10:17:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(storage): expose truthful storage class capabilities (#5172)

Browse Source
This commit is contained in:
cxymds
2026-07-24 15:28:03 +08:00
committed by GitHub
parent 6765aca3f9
commit 358caa23cb
19 changed files with 835 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/api/mod.rs
+6 -5
View File
@@ -252,11 +252,12 @@ pub mod config {
pub mod storageclass {
pub use crate::config::storageclass::{
CLASS_RRS, CLASS_STANDARD, Config, DEEP_ARCHIVE, DEFAULT_INLINE_BLOCK, DEFAULT_KVS, DEFAULT_RRS_PARITY,
EXPRESS_ONEZONE, GLACIER, GLACIER_IR, INLINE_BLOCK, INLINE_BLOCK_ENV, INTELLIGENT_TIERING, MIN_PARITY_DRIVES,
ONEZONE_IA, OPTIMIZE, OPTIMIZE_ENV, OUTPOSTS, RRS, RRS_ENV, SCHEME_PREFIX, SNOW, STANDARD, STANDARD_ENV, STANDARD_IA,
StorageClass, default_parity_count, lookup_config, lookup_config_for_pools, parse_storage_class, validate_parity,
validate_parity_inner,
CAPABILITY_CONTRACT_VERSION, CLASS_RRS, CLASS_STANDARD, Config, DEEP_ARCHIVE, DEFAULT_INLINE_BLOCK, DEFAULT_KVS,
DEFAULT_RRS_PARITY, EXPRESS_ONEZONE, GLACIER, GLACIER_IR, INLINE_BLOCK, INLINE_BLOCK_ENV, INTELLIGENT_TIERING,
LEGACY_LABEL_BEHAVIOR, MIN_PARITY_DRIVES, ONEZONE_IA, OPTIMIZE, OPTIMIZE_ENV, OUTPOSTS, RRS, RRS_ENV, SCHEME_PREFIX,
SNOW, STANDARD, STANDARD_ENV, STANDARD_IA, SUPPORTED_WRITE_CLASSES, StorageClass, UNSUPPORTED_WRITE_ERROR,
default_parity_count, effective_class, is_supported_write_class, lookup_config, lookup_config_for_pools,
parse_storage_class, validate_parity, validate_parity_inner,
};
}
crates/ecstore/src/config/storageclass.rs
+69
View File
@@ -46,6 +46,34 @@ pub const OUTPOSTS: &str = "OUTPOSTS";
pub const SNOW: &str = "SNOW";
pub const STANDARD_IA: &str = "STANDARD_IA";
/// Version of the client-discoverable storage-class write contract.
pub const CAPABILITY_CONTRACT_VERSION: u32 = 1;
/// Storage classes whose write semantics RustFS implements.
pub const SUPPORTED_WRITE_CLASSES: [&str; 2] = [STANDARD, RRS];
/// Stable S3 error code returned for unsupported write classes.
pub const UNSUPPORTED_WRITE_ERROR: &str = "InvalidStorageClass";
/// Compatibility behavior applied to historical label-only object metadata.
pub const LEGACY_LABEL_BEHAVIOR: &str = "normalized_to_effective_class";
/// Returns whether a client may select this storage class for a write.
pub fn is_supported_write_class(storage_class: &str) -> bool {
SUPPORTED_WRITE_CLASSES.contains(&storage_class)
}
/// Resolves the storage class that truthfully describes the stored object.
///
/// A completed lifecycle transition is a real storage tier and therefore keeps
/// its tier name. For local objects, only RRS has distinct layout semantics;
/// historical AWS class labels otherwise describe the effective STANDARD
/// layout.
pub fn effective_class<'a>(stored_class: Option<&'a str>, transitioned_tier: Option<&'a str>) -> &'a str {
if let Some(tier) = transitioned_tier.filter(|tier| !tier.is_empty()) {
return tier;
}
if stored_class == Some(RRS) { RRS } else { STANDARD }
}
// Standard constants for config info storage class
pub const CLASS_STANDARD: &str = "standard";
pub const CLASS_RRS: &str = "rrs";
@@ -556,6 +584,47 @@ mod tests {
}
}
#[test]
fn write_capability_contract_only_accepts_implemented_layouts() {
assert_eq!(SUPPORTED_WRITE_CLASSES, [STANDARD, RRS]);
assert!(is_supported_write_class(STANDARD));
assert!(is_supported_write_class(RRS));
for label_only_class in [
DEEP_ARCHIVE,
EXPRESS_ONEZONE,
GLACIER,
GLACIER_IR,
INTELLIGENT_TIERING,
ONEZONE_IA,
OUTPOSTS,
SNOW,
STANDARD_IA,
] {
assert!(
!is_supported_write_class(label_only_class),
"{label_only_class} must not be advertised as a supported write class"
);
}
assert!(!is_supported_write_class(""));
assert!(!is_supported_write_class("standard"));
assert!(!is_supported_write_class("UNKNOWN"));
}
#[test]
fn effective_class_normalizes_legacy_labels_and_preserves_real_tiers() {
assert_eq!(effective_class(None, None), STANDARD);
assert_eq!(effective_class(Some(STANDARD), None), STANDARD);
assert_eq!(effective_class(Some(RRS), None), RRS);
assert_eq!(effective_class(Some(STANDARD_IA), None), STANDARD);
assert_eq!(effective_class(Some(GLACIER), None), STANDARD);
assert_eq!(effective_class(Some("UNKNOWN"), None), STANDARD);
assert_eq!(effective_class(Some(STANDARD_IA), Some("WARM-TIER")), "WARM-TIER");
assert_eq!(effective_class(Some(STANDARD), Some(STANDARD_IA)), STANDARD_IA);
assert_eq!(effective_class(Some(RRS), Some("CUSTOM-RRS-TIER")), "CUSTOM-RRS-TIER");
}
#[test]
fn automatic_parity_is_resolved_per_pool() {
let cfg = lookup_config_for_pools_with_env(&KVS::new(), &[4, 2], no_env_overrides())
crates/ecstore/src/object_api/types.rs
+60 -9
View File
@@ -478,15 +478,14 @@ impl ObjectInfo {
v
};
// Extract storage class from metadata, default to STANDARD if not found
let storage_class = if !fi.transition_tier.is_empty() {
Some(fi.transition_tier.clone())
} else {
fi.metadata
.get(AMZ_STORAGE_CLASS)
.cloned()
.or_else(|| Some(storageclass::STANDARD.to_string()))
};
let storage_class = Some(
storageclass::effective_class(
fi.metadata.get(AMZ_STORAGE_CLASS).map(String::as_str),
(fi.transition_status == rustfs_filemeta::TRANSITION_COMPLETE && !fi.transition_tier.is_empty())
.then_some(fi.transition_tier.as_str()),
)
.to_string(),
);
let mut restore_ongoing = false;
let mut restore_expires = None;
@@ -1145,6 +1144,58 @@ mod tests {
assert_eq!(info.replication_decision, "arn=true;false;arn:replication::1:dest;rule-id");
}
#[test]
fn from_file_info_reports_effective_storage_class_for_legacy_metadata() {
for legacy_label in [
storageclass::STANDARD_IA,
storageclass::ONEZONE_IA,
storageclass::INTELLIGENT_TIERING,
storageclass::GLACIER,
] {
let fi = FileInfo {
metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), legacy_label.to_string())]),
..Default::default()
};
let info = ObjectInfo::from_file_info(&fi, "bucket", "legacy-object", true);
assert_eq!(
info.storage_class.as_deref(),
Some(storageclass::STANDARD),
"{legacy_label} was only a label and must report the effective STANDARD layout"
);
}
}
#[test]
fn from_file_info_preserves_transitioned_tier_storage_class() {
let fi = FileInfo {
metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string())]),
transition_tier: "WARM-TIER".to_string(),
transition_status: TRANSITION_COMPLETE.to_string(),
..Default::default()
};
let info = ObjectInfo::from_file_info(&fi, "bucket", "transitioned-object", true);
assert_eq!(info.storage_class.as_deref(), Some("WARM-TIER"));
assert_eq!(info.transitioned_object.tier, "WARM-TIER");
}
#[test]
fn from_file_info_ignores_a_tier_name_without_a_completed_transition() {
let fi = FileInfo {
metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), storageclass::STANDARD_IA.to_string())]),
transition_tier: "WARM-TIER".to_string(),
..Default::default()
};
let info = ObjectInfo::from_file_info(&fi, "bucket", "incomplete-transition", true);
assert_eq!(info.storage_class.as_deref(), Some(storageclass::STANDARD));
assert_eq!(info.transitioned_object.tier, "WARM-TIER");
}
#[test]
fn get_actual_size_uses_compressed_parts_actual_size_when_metadata_missing() {
let user_defined = {
crates/ecstore/src/set_disk/mod.rs
+2 -28
View File
@@ -4408,20 +4408,7 @@ pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<String>, if_m
/// Validates if the given storage class is supported
pub fn is_valid_storage_class(storage_class: &str) -> bool {
matches!(
storage_class,
storageclass::STANDARD
| storageclass::RRS
| storageclass::DEEP_ARCHIVE
| storageclass::EXPRESS_ONEZONE
| storageclass::GLACIER
| storageclass::GLACIER_IR
| storageclass::INTELLIGENT_TIERING
| storageclass::ONEZONE_IA
| storageclass::OUTPOSTS
| storageclass::SNOW
| storageclass::STANDARD_IA
)
storageclass::is_supported_write_class(storage_class)
}
/// Returns true if the storage class is a cold storage tier that requires special handling
@@ -7796,23 +7783,10 @@ mod tests {
#[test]
fn test_is_valid_storage_class() {
// Test valid storage classes
assert!(is_valid_storage_class(storageclass::STANDARD));
assert!(is_valid_storage_class(storageclass::RRS));
assert!(is_valid_storage_class(storageclass::DEEP_ARCHIVE));
assert!(is_valid_storage_class(storageclass::EXPRESS_ONEZONE));
assert!(is_valid_storage_class(storageclass::GLACIER));
assert!(is_valid_storage_class(storageclass::GLACIER_IR));
assert!(is_valid_storage_class(storageclass::INTELLIGENT_TIERING));
assert!(is_valid_storage_class(storageclass::ONEZONE_IA));
assert!(is_valid_storage_class(storageclass::OUTPOSTS));
assert!(is_valid_storage_class(storageclass::SNOW));
assert!(is_valid_storage_class(storageclass::STANDARD_IA));
// Test invalid storage classes
assert!(!is_valid_storage_class(storageclass::STANDARD_IA));
assert!(!is_valid_storage_class("INVALID"));
assert!(!is_valid_storage_class(""));
assert!(!is_valid_storage_class("standard")); // lowercase
}
#[test]
crates/ecstore/src/store/list_objects.rs
+66 -8
View File
@@ -309,7 +309,8 @@ const MAX_LIST_OBJECTS_METADATA_FAST_STALENESS_MS: u64 = 60_000;
const LIST_OBJECTS_INDEX_PROVIDER_WALKER_KEY_ONLY: &str = "walker_key_only";
const LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY: &str = "persistent_key_only";
const LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY_DEFAULT_GENERATION: &str = "persistent-key-only";
const PERSISTENT_KEY_ONLY_INDEX_HEADER: &str = "# rustfs-listobjects-key-only-v1";
const PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION: u8 = 2;
const PERSISTENT_KEY_ONLY_INDEX_HEADER: &str = "# rustfs-listobjects-key-only-v2";
const PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER: &str = "# bucket=";
const PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER: &str = "# generation=";
const PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER: &str = "# checkpoint_high_water_mark=";
@@ -502,6 +503,7 @@ struct PersistentKeyOnlyIndexCache {
#[derive(Debug, Clone, PartialEq, Eq)]
struct PersistentKeyOnlyIndex {
format_version: u8,
bucket: Option<String>,
generation: String,
checkpoint_high_water_mark: u64,
@@ -1410,6 +1412,7 @@ fn parse_persistent_list_metadata_object(line: &str) -> Option<PersistentListMet
}
fn parse_persistent_key_only_index(contents: &str) -> PersistentKeyOnlyIndex {
let mut format_version = 0;
let mut bucket = None;
let mut generation = None;
let mut checkpoint_high_water_mark = None;
@@ -1421,6 +1424,10 @@ fn parse_persistent_key_only_index(contents: &str) -> PersistentKeyOnlyIndex {
if line.is_empty() {
continue;
}
if line == PERSISTENT_KEY_ONLY_INDEX_HEADER {
format_version = PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION;
continue;
}
if let Some(object) = parse_persistent_list_metadata_object(line) {
keys.push(object.name.clone());
objects.push(object);
@@ -1455,6 +1462,7 @@ fn parse_persistent_key_only_index(contents: &str) -> PersistentKeyOnlyIndex {
let checkpoint_high_water_mark = checkpoint_high_water_mark.unwrap_or_else(|| u64::try_from(keys.len()).unwrap_or(u64::MAX));
PersistentKeyOnlyIndex {
format_version,
bucket,
generation: generation.unwrap_or_else(|| LIST_OBJECTS_INDEX_PROVIDER_PERSISTENT_KEY_ONLY_DEFAULT_GENERATION.to_owned()),
checkpoint_high_water_mark,
@@ -1485,6 +1493,9 @@ fn persistent_key_only_index_matches_provider(
bucket: &str,
provider_state: &ListObjectsIndexProviderState,
) -> bool {
if index.format_version != PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION {
return false;
}
if index.bucket.as_deref().is_some_and(|index_bucket| index_bucket != bucket) {
return false;
}
@@ -1579,6 +1590,7 @@ async fn write_persistent_key_only_index_with_metadata(
tokio::fs::rename(&tmp_path, path).await.map_err(Error::Io)?;
Ok(PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some(bucket.to_owned()),
generation: generation.to_owned(),
checkpoint_high_water_mark,
@@ -6687,13 +6699,13 @@ mod test {
ListObjectsIndexProviderState, ListPathOptions, ListPathRawOptions, ListSourceMode, ListingEntryResolution,
ListingSupplement, ListingSupplementOptions, MAX_OBJECT_LIST, NamespaceMutationJournalBackend,
NamespaceMutationJournalSnapshot, NamespaceMutationJournalStatus, PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER,
PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER,
PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex, PersistentListMetadataObject, RUSTFS_META_BUCKET,
VerifiedIndexCandidateStats, VersionMarker, current_list_objects_mutation_sequence,
encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum, expand_ask_disks_for_object_quorum,
fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects, latest_listing_object_quorum,
latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key, list_merged_entry_channel,
list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates,
PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER, PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER, PERSISTENT_KEY_ONLY_INDEX_HEADER, PersistentKeyOnlyIndex,
PersistentListMetadataObject, RUSTFS_META_BUCKET, VerifiedIndexCandidateStats, VersionMarker,
current_list_objects_mutation_sequence, encode_persistent_list_metadata_object, enforce_latest_listing_write_quorum,
expand_ask_disks_for_object_quorum, fallback_entries_for_object, gather_results, latest_listing_allow_agreed_objects,
latest_listing_object_quorum, latest_listing_raw_min_disks, latest_listing_required_object_quorum, list_marker_key,
list_merged_entry_channel, list_metadata_resolution_params, list_objects_from_metadata_snapshot_candidates,
list_objects_from_verified_index_candidates, list_objects_from_verified_index_candidates_with_optional_stats,
list_objects_from_verified_index_candidates_with_stats, list_objects_index_mode_from_env,
list_objects_index_provider_from_env, list_objects_index_provider_state_from_env, list_objects_key_only_provider_health,
@@ -8041,6 +8053,7 @@ mod test {
#[test]
fn metadata_fast_requires_complete_metadata_snapshot() {
let index = PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some("bucket".to_string()),
generation: "generation-42".to_string(),
checkpoint_high_water_mark: 42,
@@ -8057,6 +8070,7 @@ mod test {
assert!(!persistent_key_only_index_has_complete_metadata_snapshot(&index));
let complete = PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some("bucket".to_string()),
generation: "generation-42".to_string(),
checkpoint_high_water_mark: 42,
@@ -8402,6 +8416,7 @@ mod test {
#[test]
fn persistent_key_only_index_provider_match_rejects_configured_generation_mismatch() {
let index = PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some("bucket".to_string()),
generation: "generation-old".to_string(),
checkpoint_high_water_mark: 42,
@@ -8422,9 +8437,50 @@ mod test {
assert!(!persistent_key_only_index_matches_provider(&index, "other-bucket", &matching_provider));
}
#[test]
fn persistent_key_only_index_provider_match_rejects_legacy_format_without_configured_generation() {
let legacy = parse_persistent_key_only_index(&format!(
"# rustfs-listobjects-key-only-v1\n\
{PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER}bucket\n\
{PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER}generation-old\n\
{PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER}42\n\
# object\tbGVnYWN5\t1\t-\t-\tU1RBTkRBUkRfSUE=\n"
));
let provider =
ListObjectsIndexProviderState::persistent_key_only(Some(PathBuf::from("/tmp/persistent-key-only.index")), None);
assert_eq!(legacy.format_version, 0);
assert!(!persistent_key_only_index_matches_provider(&legacy, "bucket", &provider));
}
#[test]
fn current_persistent_snapshot_preserves_an_effective_transition_tier_name() {
let object = PersistentListMetadataObject::from_object_info(&ObjectInfo {
bucket: "bucket".to_string(),
name: "transitioned".to_string(),
storage_class: Some("STANDARD_IA".to_string()),
..Default::default()
});
let index = parse_persistent_key_only_index(&format!(
"{PERSISTENT_KEY_ONLY_INDEX_HEADER}\n\
{PERSISTENT_KEY_ONLY_INDEX_BUCKET_HEADER}bucket\n\
{PERSISTENT_KEY_ONLY_INDEX_GENERATION_HEADER}generation-current\n\
{PERSISTENT_KEY_ONLY_INDEX_CHECKPOINT_HEADER}42\n\
{}\n",
encode_persistent_list_metadata_object(&object)
));
let provider =
ListObjectsIndexProviderState::persistent_key_only(Some(PathBuf::from("/tmp/persistent-key-only.index")), None);
assert_eq!(index.format_version, PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION);
assert!(persistent_key_only_index_matches_provider(&index, "bucket", &provider));
assert_eq!(index.objects[0].to_object_info("bucket").storage_class.as_deref(), Some("STANDARD_IA"));
}
#[test]
fn persistent_key_only_index_health_uses_snapshot_generation_and_checkpoint() {
let index = PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some("bucket".to_string()),
generation: "generation-42".to_string(),
checkpoint_high_water_mark: 42,
@@ -8450,6 +8506,7 @@ mod test {
#[test]
fn persistent_key_only_index_health_reports_lagging_mutation_checkpoint() {
let index = PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some("bucket".to_string()),
generation: "generation-42".to_string(),
checkpoint_high_water_mark: 42,
@@ -8476,6 +8533,7 @@ mod test {
#[test]
fn persistent_key_only_index_health_reports_degraded_journal() {
let index = PersistentKeyOnlyIndex {
format_version: PERSISTENT_KEY_ONLY_INDEX_FORMAT_VERSION,
bucket: Some("bucket".to_string()),
generation: "generation-42".to_string(),
checkpoint_high_water_mark: 42,