gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into reatang/fix-replication-switches

Browse Source
This commit is contained in:
Zhengchao An
2026-08-01 10:44:38 +08:00
committed by GitHub
parent cbfd29ccdc 4b6b6f14bd
commit 8c42499e83
247 changed files with 30765 additions and 4575 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/audit/Cargo.toml
+26
View File
@@ -25,7 +25,33 @@ documentation = "https://docs.rs/rustfs-audit/latest/rustfs_audit/"
keywords = ["audit", "target", "management", "fan-out", "RustFS"]
categories = ["web-programming", "development-tools", "asynchronous", "api-bindings"]
[features]
default = []
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"hotpath/futures",
"rustfs-config/hotpath",
"rustfs-s3-types/hotpath",
"rustfs-targets/hotpath",
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-config/hotpath-alloc",
"rustfs-s3-types/hotpath-alloc",
"rustfs-targets/hotpath-alloc",
]
hotpath-cpu = [
"hotpath",
"hotpath/hotpath-cpu",
"rustfs-config/hotpath-cpu",
"rustfs-s3-types/hotpath-cpu",
"rustfs-targets/hotpath-cpu",
]
[dependencies]
hotpath.workspace = true
rustfs-targets = { workspace = true }
rustfs-config = { workspace = true, features = ["audit", "server-config-model"] }
rustfs-s3-types = { workspace = true }
crates/checksums/Cargo.toml
+7
View File
@@ -28,7 +28,14 @@ documentation = "https://docs.rs/rustfs-checksums/latest/rustfs_checksum/"
[lints]
workspace = true
[features]
default = []
hotpath = ["hotpath/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
[dependencies]
hotpath.workspace = true
bytes = { workspace = true, features = ["serde"] }
crc-fast = { workspace = true }
http = { workspace = true }
crates/common/Cargo.toml
+7
View File
@@ -27,7 +27,14 @@ categories = ["web-programming", "development-tools", "data-structures"]
[lints]
workspace = true
[features]
default = []
hotpath = ["hotpath/hotpath", "hotpath/tokio"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
[dependencies]
hotpath.workspace = true
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
tonic = { workspace = true, features = ["gzip", "deflate"] }
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
crates/concurrency/Cargo.toml
+7
View File
@@ -13,7 +13,14 @@ categories = ["concurrency", "filesystem"]
[lints]
workspace = true
[features]
default = []
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-core/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-core/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-core/hotpath-cpu"]
[dependencies]
hotpath.workspace = true
# Internal crates
rustfs-io-core = { workspace = true }
serde = { workspace = true, features = ["derive"] }
crates/config/Cargo.toml
+4
View File
@@ -25,6 +25,7 @@ keywords = ["configuration", "settings", "management", "rustfs", "Minio"]
categories = ["web-programming", "development-tools", "config"]
[dependencies]
hotpath.workspace = true
const-str = { workspace = true, optional = true, features = ["std", "proc"] }
serde = { workspace = true, optional = true, features = ["derive"] }
serde_json = { workspace = true, optional = true, features = ["raw_value"] }
@@ -34,6 +35,9 @@ workspace = true
[features]
default = ["constants"]
hotpath = ["hotpath/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
audit = ["dep:const-str", "constants"]
constants = ["dep:const-str"]
notify = ["dep:const-str", "constants"]
crates/config/README.md
+4
View File
@@ -66,6 +66,10 @@ Current guidance:
- `RUSTFS_BROWSER_REDIRECT_URL` sets the externally reachable browser origin used for OIDC callback, console success redirect, and logout fallback URLs. Configure it to the public scheme and authority without a path, for example `https://console.example.com`. In load-balancer deployments, keep OIDC authorize and callback requests on the same backend node because the in-flight OIDC `state` is local to the RustFS node.
## Distributed endpoint locality
- `RUSTFS_LOCAL_ENDPOINT_HOST` identifies this server's host in a distributed `RUSTFS_VOLUMES` topology without resolving every peer during startup. Set it to exactly one host, without a scheme, port, or path. It is accepted only for orchestrated URL topologies and must match at least one endpoint on the RustFS server port; invalid or unmatched values fail startup. Leave it unset to retain DNS-based locality discovery.
## Scanner environment aliases
- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
crates/config/src/constants/app.rs
+4
View File
@@ -131,6 +131,10 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
/// Environment variable for server volumes.
pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
/// Environment variable identifying this server's host in distributed endpoint
/// lists without relying on DNS locality discovery.
pub const ENV_LOCAL_ENDPOINT_HOST: &str = "RUSTFS_LOCAL_ENDPOINT_HOST";
/// Environment variable to explicitly bypass local physical disk independence checks.
pub const ENV_UNSAFE_BYPASS_DISK_CHECK: &str = "RUSTFS_UNSAFE_BYPASS_DISK_CHECK";
crates/credentials/Cargo.toml
+7
View File
@@ -24,7 +24,14 @@ description = "Credentials management utilities for RustFS, enabling secure hand
keywords = ["rustfs", "Minio", "credentials", "authentication", "authorization"]
categories = ["web-programming", "development-tools", "data-structures", "security"]
[features]
default = []
hotpath = ["hotpath/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
[dependencies]
hotpath.workspace = true
base64-simd = { workspace = true }
hmac = { workspace = true }
rand = { workspace = true, features = ["serde"] }
crates/crypto/Cargo.toml
+4
View File
@@ -29,6 +29,7 @@ documentation = "https://docs.rs/rustfs-crypto/latest/rustfs_crypto/"
workspace = true
[dependencies]
hotpath.workspace = true
aes-gcm = { workspace = true, optional = true, features = ["rand_core"] }
argon2 = { workspace = true, optional = true }
chacha20poly1305 = { workspace = true, optional = true }
@@ -49,6 +50,9 @@ time = { workspace = true, features = ["parsing", "formatting", "macros", "serde
[features]
default = ["crypto", "fips"]
hotpath = ["hotpath/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
fips = []
crypto = [
"dep:aes-gcm",
crates/data-usage/Cargo.toml
+7
View File
@@ -27,7 +27,14 @@ categories = ["data-structures", "filesystem"]
[lints]
workspace = true
[features]
default = []
hotpath = ["hotpath/hotpath", "rustfs-filemeta/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-filemeta/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-filemeta/hotpath-cpu"]
[dependencies]
hotpath.workspace = true
serde = { workspace = true, features = ["derive"] }
rmp-serde = { workspace = true }
async-trait = { workspace = true }
crates/e2e_test/Cargo.toml
+48
View File
@@ -25,10 +25,58 @@ workspace = true
[features]
default = []
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"hotpath/futures",
"hotpath/reqwest-0-13",
"rustfs-config/hotpath",
"rustfs-credentials/hotpath",
"rustfs-data-usage/hotpath",
"rustfs-ecstore/hotpath",
"rustfs-filemeta/hotpath",
"rustfs-lock/hotpath",
"rustfs-madmin/hotpath",
"rustfs-protos/hotpath",
"rustfs-rio/hotpath",
"rustfs-signer/hotpath",
"rustfs-utils/hotpath",
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-config/hotpath-alloc",
"rustfs-credentials/hotpath-alloc",
"rustfs-data-usage/hotpath-alloc",
"rustfs-ecstore/hotpath-alloc",
"rustfs-filemeta/hotpath-alloc",
"rustfs-lock/hotpath-alloc",
"rustfs-madmin/hotpath-alloc",
"rustfs-protos/hotpath-alloc",
"rustfs-rio/hotpath-alloc",
"rustfs-signer/hotpath-alloc",
"rustfs-utils/hotpath-alloc",
]
hotpath-cpu = [
"hotpath",
"hotpath/hotpath-cpu",
"rustfs-config/hotpath-cpu",
"rustfs-credentials/hotpath-cpu",
"rustfs-data-usage/hotpath-cpu",
"rustfs-ecstore/hotpath-cpu",
"rustfs-filemeta/hotpath-cpu",
"rustfs-lock/hotpath-cpu",
"rustfs-madmin/hotpath-cpu",
"rustfs-protos/hotpath-cpu",
"rustfs-rio/hotpath-cpu",
"rustfs-signer/hotpath-cpu",
"rustfs-utils/hotpath-cpu",
]
ftps = []
sftp = []
[dependencies]
hotpath.workspace = true
rustfs-config = { workspace = true, features = ["constants"] }
rustfs-credentials.workspace = true
rustfs-ecstore.workspace = true
crates/e2e_test/src/copy_object_metadata_test.rs
+18 -15
View File
@@ -117,9 +117,14 @@ mod tests {
.key("assets/explicit-copy.js")
.copy_source(format!("{bucket}/{key}"))
.metadata_directive(MetadataDirective::Copy)
.customize()
.mutate_request(|request| {
request.headers_mut().insert("content-type", "application/octet-stream");
request.headers_mut().insert("x-amz-meta-request-only", "ignored");
})
.send()
.await
.expect("explicit COPY directive failed");
.expect("explicit COPY directive with request metadata failed");
let explicit_copy_head = client
.head_object()
.bucket(bucket)
@@ -128,6 +133,18 @@ mod tests {
.await
.expect("HEAD failed after explicit COPY");
assert_eq!(explicit_copy_head.cache_control(), Some("max-age=60"));
assert_eq!(explicit_copy_head.content_type(), Some("text/javascript; charset=utf-8"));
assert_eq!(
explicit_copy_head.metadata().and_then(|metadata| metadata.get("mtime")),
Some(&"1777992333".to_string())
);
assert_eq!(
explicit_copy_head
.metadata()
.and_then(|metadata| metadata.get("request-only")),
None,
"COPY must ignore request metadata"
);
assert_eq!(
explicit_copy_head.website_redirect_location(),
None,
@@ -571,20 +588,6 @@ mod tests {
Some("InvalidArgument")
);
let ignored_replacement = client
.copy_object()
.bucket(bucket)
.key(key)
.copy_source(format!("{bucket}/{key}"))
.content_type("application/ignored")
.send()
.await
.expect_err("Replacement fields without REPLACE should be rejected");
assert_eq!(
ignored_replacement.as_service_error().and_then(|error| error.code()),
Some("InvalidRequest")
);
let unchanged = client
.get_object()
.bucket(bucket)
crates/e2e_test/src/delete_marker_migration_semantics_test.rs
+28
View File
@@ -56,6 +56,21 @@ mod tests {
);
}
async fn assert_current_list_hides_delete_marker(client: &Client, bucket: &str, key: &str) {
let listed = client
.list_objects_v2()
.bucket(bucket)
.prefix(key)
.send()
.await
.expect("list current objects after delete marker");
assert!(
listed.contents().iter().all(|object| object.key() != Some(key)),
"ListObjectsV2 must hide an object whose latest version is a delete marker"
);
}
#[tokio::test]
#[serial]
async fn test_versioning_only_delete_marker_has_minio_compatible_visibility_for_migration_proof() {
@@ -94,6 +109,7 @@ mod tests {
assert_eq!(markers[0].version_id(), Some(delete_marker_version_id));
assert_eq!(markers[0].is_latest(), Some(true));
assert_current_get_is_delete_marker_not_found(&client, bucket, key).await;
assert_current_list_hides_delete_marker(&client, bucket, key).await;
}
#[tokio::test]
@@ -118,6 +134,17 @@ mod tests {
.await
.expect("put historical version");
let data_version_id = put.version_id().expect("put should return data version id");
let listed_before_delete = client
.list_objects_v2()
.bucket(bucket)
.prefix(key)
.send()
.await
.expect("list current object before creating delete marker");
assert!(
listed_before_delete.contents().iter().any(|object| object.key() == Some(key)),
"ListObjectsV2 must include the current object before it is deleted"
);
let delete_marker = client
.delete_object()
@@ -145,6 +172,7 @@ mod tests {
assert_eq!(markers[0].version_id(), Some(delete_marker_version_id));
assert_eq!(markers[0].is_latest(), Some(true));
assert_current_get_is_delete_marker_not_found(&client, bucket, key).await;
assert_current_list_hides_delete_marker(&client, bucket, key).await;
let historical = client
.get_object()
crates/e2e_test/src/inline_fast_path_cluster_test.rs
+12 -3
View File
@@ -2136,11 +2136,20 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
assert_eq!(terminal["bucket"].as_str(), Some(bucket.as_str()));
assert_eq!(terminal["prefix"].as_str(), Some(prefix));
assert_eq!(terminal["dry_run"].as_bool(), Some(false));
assert_eq!(
terminal["status"].as_str(),
Some("partial"),
let terminal_status = terminal["status"].as_str();
assert!(
matches!(terminal_status, Some("partial" | "unknown")),
"small transition queue should surface terminal backpressure: {terminal}"
);
if terminal_status == Some("unknown") {
let failure_reason = terminal["failure_reason"]
.as_str()
.ok_or_else(|| format!("unknown terminal status omitted failure_reason: {terminal}"))?;
assert!(
failure_reason.contains("worker result was not persisted before the transition queue drained"),
"unknown terminal status should identify lost worker-result persistence: {terminal}"
);
}
let skipped_queue_full = terminal["report"]["skipped_queue_full"]
.as_u64()
.ok_or_else(|| format!("terminal status omitted report.skipped_queue_full: {terminal}"))?;
crates/e2e_test/src/object_lock/object_lock_test.rs
+122
View File
@@ -26,6 +26,7 @@
use super::common::*;
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::{ByteStream, DateTimeFormat};
use aws_sdk_s3::types::{
CompletedMultipartUpload, CompletedPart, Delete, MetadataDirective, ObjectIdentifier, ObjectLockLegalHoldStatus,
@@ -2120,6 +2121,127 @@ async fn test_multipart_default_retention_fixed_at_create() {
// Versioning Auto-Enable Tests
// ============================================================================
#[tokio::test]
#[serial]
async fn test_unretained_object_lock_object_delete_and_bucket_cleanup() {
init_logging();
info!("🧪 Test: Unretained Object Lock object delete and bucket cleanup (Issue #5339)");
let mut env = ObjectLockTestEnvironment::new()
.await
.expect("failed to create Object Lock test environment");
env.start_rustfs().await.expect("failed to start RustFS");
let bucket = "test-object-lock-delete-cleanup";
let key = "unretained-object";
env.create_object_lock_bucket(bucket)
.await
.expect("failed to create Object Lock bucket");
let client = env.s3_client();
let put_response = client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"unretained data"))
.send()
.await
.expect("failed to upload unretained object");
let object_version_id = put_response
.version_id()
.expect("Object Lock buckets must create versioned objects")
.to_string();
let delete_response = client
.delete_object()
.bucket(bucket)
.key(key)
.send()
.await
.expect("failed to create delete marker");
assert_eq!(delete_response.delete_marker(), Some(true));
let delete_marker_version_id = delete_response
.version_id()
.expect("Deleting without a version ID must create a delete marker")
.to_string();
let get_error = client
.get_object()
.bucket(bucket)
.key(key)
.send()
.await
.expect_err("GET must not return an object hidden by a delete marker");
assert_eq!(get_error.raw_response().map(|response| response.status().as_u16()), Some(404));
assert_eq!(get_error.as_service_error().and_then(|error| error.code()), Some("NoSuchKey"));
let listed_objects = client
.list_objects_v2()
.bucket(bucket)
.send()
.await
.expect("failed to list current objects");
assert!(
listed_objects.contents().iter().all(|object| object.key() != Some(key)),
"ListObjectsV2 must hide objects whose latest version is a delete marker"
);
let listed_versions = client
.list_object_versions()
.bucket(bucket)
.send()
.await
.expect("failed to list object versions");
assert!(
listed_versions
.versions()
.iter()
.any(|version| version.key() == Some(key) && version.version_id() == Some(object_version_id.as_str())),
"The data version must remain until it is explicitly deleted"
);
assert!(
listed_versions
.delete_markers()
.iter()
.any(|marker| marker.key() == Some(key) && marker.version_id() == Some(delete_marker_version_id.as_str())),
"ListObjectVersions must expose the delete marker"
);
client
.delete_object()
.bucket(bucket)
.key(key)
.version_id(object_version_id)
.send()
.await
.expect("failed to delete the data version");
client
.delete_object()
.bucket(bucket)
.key(key)
.version_id(delete_marker_version_id)
.send()
.await
.expect("failed to delete the delete marker");
let remaining_versions = client
.list_object_versions()
.bucket(bucket)
.send()
.await
.expect("failed to list versions after cleanup");
assert!(remaining_versions.versions().is_empty());
assert!(remaining_versions.delete_markers().is_empty());
client
.delete_bucket()
.bucket(bucket)
.send()
.await
.expect("Deleting every version must remove xl.meta so the bucket can be deleted normally");
}
#[tokio::test]
#[serial]
async fn test_versioning_auto_enabled_with_object_lock() {
crates/ecstore/Cargo.toml
+87 -5
View File
@@ -33,14 +33,98 @@ workspace = true
[features]
default = []
rio-v2 = ["dep:rustfs-rio-v2"]
hotpath = ["dep:hotpath", "hotpath/hotpath", "rustfs-filemeta/hotpath", "rustfs-rio/hotpath"]
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"hotpath/futures",
"hotpath/async-channel",
"hotpath/parking_lot",
"hotpath/reqwest-0-13",
"rustfs-checksums/hotpath",
"rustfs-common/hotpath",
"rustfs-concurrency/hotpath",
"rustfs-config/hotpath",
"rustfs-credentials/hotpath",
"rustfs-data-usage/hotpath",
"rustfs-filemeta/hotpath",
"rustfs-io-metrics/hotpath",
"rustfs-lifecycle/hotpath",
"rustfs-lock/hotpath",
"rustfs-madmin/hotpath",
"rustfs-object-capacity/hotpath",
"rustfs-policy/hotpath",
"rustfs-protos/hotpath",
"rustfs-replication/hotpath",
"rustfs-rio/hotpath",
"rustfs-rio-v2?/hotpath",
"rustfs-s3-types/hotpath",
"rustfs-signer/hotpath",
"rustfs-storage-api/hotpath",
"rustfs-tls-runtime/hotpath",
"rustfs-utils/hotpath",
"rustfs-crypto/hotpath",
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-checksums/hotpath-alloc",
"rustfs-common/hotpath-alloc",
"rustfs-concurrency/hotpath-alloc",
"rustfs-config/hotpath-alloc",
"rustfs-credentials/hotpath-alloc",
"rustfs-data-usage/hotpath-alloc",
"rustfs-filemeta/hotpath-alloc",
"rustfs-io-metrics/hotpath-alloc",
"rustfs-lifecycle/hotpath-alloc",
"rustfs-lock/hotpath-alloc",
"rustfs-madmin/hotpath-alloc",
"rustfs-object-capacity/hotpath-alloc",
"rustfs-policy/hotpath-alloc",
"rustfs-protos/hotpath-alloc",
"rustfs-replication/hotpath-alloc",
"rustfs-rio/hotpath-alloc",
"rustfs-rio-v2?/hotpath-alloc",
"rustfs-s3-types/hotpath-alloc",
"rustfs-signer/hotpath-alloc",
"rustfs-storage-api/hotpath-alloc",
"rustfs-tls-runtime/hotpath-alloc",
"rustfs-utils/hotpath-alloc",
"rustfs-crypto/hotpath-alloc",
]
hotpath-cpu = [
"hotpath",
"hotpath/hotpath-cpu",
"rustfs-checksums/hotpath-cpu",
"rustfs-common/hotpath-cpu",
"rustfs-concurrency/hotpath-cpu",
"rustfs-config/hotpath-cpu",
"rustfs-credentials/hotpath-cpu",
"rustfs-data-usage/hotpath-cpu",
"rustfs-filemeta/hotpath-cpu",
"rustfs-io-metrics/hotpath-cpu",
"rustfs-lifecycle/hotpath-cpu",
"rustfs-lock/hotpath-cpu",
"rustfs-madmin/hotpath-cpu",
"rustfs-object-capacity/hotpath-cpu",
"rustfs-policy/hotpath-cpu",
"rustfs-protos/hotpath-cpu",
"rustfs-replication/hotpath-cpu",
"rustfs-rio/hotpath-cpu",
"rustfs-rio-v2?/hotpath-cpu",
"rustfs-s3-types/hotpath-cpu",
"rustfs-signer/hotpath-cpu",
"rustfs-storage-api/hotpath-cpu",
"rustfs-tls-runtime/hotpath-cpu",
"rustfs-utils/hotpath-cpu",
"rustfs-crypto/hotpath-cpu",
]
# Exposes shared lifecycle/tier test utilities (MockWarmBackend, fault
# injection, xl.meta transition assertions) via `api::tier::test_util`.
# Enable only from `[dev-dependencies]` (rustfs/backlog#1148 ilm-6).
test-util = []
[dependencies]
hotpath = { workspace = true, optional = true }
hotpath.workspace = true
rustfs-filemeta.workspace = true
rustfs-utils = { workspace = true, features = ["full"] }
rustfs-rio.workspace = true
@@ -57,7 +141,6 @@ rustfs-policy.workspace = true
rustfs-protos.workspace = true
rustfs-replication.workspace = true
rustfs-lifecycle.workspace = true
rustfs-kms.workspace = true
rustfs-s3-types = { workspace = true }
rustfs-data-usage.workspace = true
rustfs-object-capacity.workspace = true
@@ -105,6 +188,7 @@ tempfile.workspace = true
hyper = { workspace = true, features = ["http2", "http1", "server"] }
hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] }
hostname.workspace = true
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
rustls-pki-types.workspace = true
tokio = { workspace = true, features = ["io-util", "sync", "signal", "fs", "rt-multi-thread"] }
@@ -124,8 +208,6 @@ libc.workspace = true
rustix = { workspace = true, features = ["process", "fs"] }
rustfs-madmin.workspace = true
reqwest = { workspace = true }
aes-gcm = { workspace = true, features = ["rand_core"] }
chacha20poly1305.workspace = true
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
urlencoding = { workspace = true }
smallvec = { workspace = true, features = ["serde"] }
crates/ecstore/src/api/mod.rs
+13 -10
View File
@@ -267,12 +267,13 @@ pub mod config {
pub mod com {
pub use crate::config::com::{
COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
ServerConfigCorruptError, ServerConfigSnapshot, delete_config, is_server_config_corrupt_error, lookup_configs,
read_config, read_config_no_lock, read_config_with_metadata, read_config_without_migrate,
read_config_without_migrate_no_lock, read_existing_server_config_no_lock, read_server_config_snapshot, save_config,
save_config_no_lock, save_config_with_opts, save_server_config, save_server_config_no_lock,
save_server_config_snapshot, server_config_path, try_migrate_server_config, with_config_object_read_lock,
with_config_object_write_lock, with_server_config_read_lock, with_server_config_write_lock,
ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config,
is_server_config_corrupt_error, lookup_configs, read_config, read_config_no_lock, read_config_with_metadata,
read_config_without_migrate, read_config_without_migrate_no_lock, read_existing_server_config_no_lock,
read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
save_server_config_no_lock, save_server_config_snapshot, save_server_config_snapshot_with_generation,
server_config_path, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock,
with_server_config_read_lock, with_server_config_write_lock,
};
}
@@ -391,10 +392,12 @@ pub mod notification {
pub mod object {
pub use crate::object_api::{
BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource,
GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, StreamConsumer,
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, ObjectEncryptionResolver, ObjectInfo,
ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode,
ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
unregister_object_mutation_hook,
};
pub use crate::store::PreparedGetObjectReader;
}
crates/ecstore/src/bucket/metadata_sys.rs
+93 -6
View File
@@ -579,8 +579,26 @@ pub struct BucketMetadataSys {
/// Serializes metadata-map commits and their derived cache updates for one
/// bucket. Namespace locks, when present, are acquired before this lock.
metadata_publish_locks: Arc<MetadataPublishLockRegistry>,
/// Deduplicates concurrent lazy loads of one bucket's metadata, so N
/// simultaneous cache misses issue a single disk read instead of N.
///
/// This is the `singleflight` that upstream applies to its own lazy
/// `GetConfig`. Without it the namespace *read* lock the load holds is no
/// help: read locks are shared, so it excludes concurrent config writers
/// but not concurrent readers, and every caller still pays a full
/// erasure-set metadata fanout. A separate registry from
/// `metadata_publish_locks`, reusing the same per-bucket lock machinery.
///
/// Lock order: this lock, then the namespace lock, then the publish lock,
/// then the metadata map. It is only ever taken as the first of those, so
/// it cannot invert against a path that already holds one of the others.
lazy_load_locks: Arc<MetadataPublishLockRegistry>,
#[cfg(test)]
lazy_load_lock_probe: std::sync::atomic::AtomicBool,
/// Counts disk loads taken by the lazy `get_config` path, so a test can
/// prove concurrent misses collapse into one.
#[cfg(test)]
lazy_disk_loads: std::sync::atomic::AtomicUsize,
/// Buckets recently observed to have no persisted metadata. Serving the
/// fabricated default from here (instead of re-reading disk) keeps the
/// per-request cost of repeated lookups for such names bounded — without
@@ -599,8 +617,13 @@ impl BucketMetadataSys {
metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
locks: StdMutex::new(HashMap::new()),
}),
lazy_load_locks: Arc::new(MetadataPublishLockRegistry {
locks: StdMutex::new(HashMap::new()),
}),
#[cfg(test)]
lazy_load_lock_probe: std::sync::atomic::AtomicBool::new(false),
#[cfg(test)]
lazy_disk_loads: std::sync::atomic::AtomicUsize::new(0),
absent_metadata: moka::future::Cache::builder()
.max_capacity(ABSENT_BUCKET_METADATA_MAX_ENTRIES)
.time_to_live(ABSENT_BUCKET_METADATA_TTL)
@@ -615,16 +638,22 @@ impl BucketMetadataSys {
}
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
let mut locks = self
.metadata_publish_locks
.locks
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
}
/// Per-bucket gate for the lazy `get_config` disk load. See
/// [`Self::lazy_load_locks`].
fn lazy_load_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
Self::bucket_lock_in(&self.lazy_load_locks, bucket)
}
fn bucket_lock_in(registry: &Arc<MetadataPublishLockRegistry>, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
let mut locks = registry.locks.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
locks.get(bucket).and_then(Weak::upgrade).unwrap_or_else(|| {
let lock = Arc::new_cyclic(|lock| {
Mutex::new(MetadataPublishLockState {
bucket: bucket.to_string(),
registry: Arc::downgrade(&self.metadata_publish_locks),
registry: Arc::downgrade(registry),
lock: lock.clone(),
})
});
@@ -1079,6 +1108,27 @@ impl BucketMetadataSys {
return Ok((Arc::new(bm), true));
}
// Collapse concurrent misses for this bucket into one disk load.
// Taken before the namespace lock — see `lazy_load_locks` for the
// ordering rule.
let load_lock = self.lazy_load_lock(bucket);
let _load_guard = load_lock.lock_owned().await;
// Re-check both caches: whoever held the gate before us may have
// already answered this exact question, and repeating the fanout
// is the whole cost this gate exists to avoid.
if let Some(bm) = self.metadata_map.read().await.get(bucket).cloned() {
return Ok((bm, true));
}
if self.absent_metadata.get(bucket).await.is_some() {
let mut bm = BucketMetadata::new(bucket);
bm.default_timestamps();
return Ok((Arc::new(bm), true));
}
#[cfg(test)]
self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let lock = self.api.new_ns_lock(bucket, bucket).await?;
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
#[cfg(test)]
@@ -1431,6 +1481,43 @@ mod tests {
use serial_test::serial;
use tokio::time::timeout;
/// Concurrent cache misses for one bucket must collapse into a single disk
/// load.
///
/// The namespace read lock the lazy path already holds does not provide
/// this: read locks are shared, so it excludes concurrent config writers
/// but not concurrent readers. Without the dedup gate every caller pays its
/// own namespace-lock acquisition plus a full erasure-set metadata fanout —
/// and the paths that reach `get_config` are per-request, so the multiplier
/// is request concurrency.
#[tokio::test]
async fn concurrent_lazy_loads_of_one_bucket_issue_a_single_disk_read() {
use std::sync::atomic::Ordering;
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
let sys = Arc::new(BucketMetadataSys::new(ecstore));
// A name with no persisted metadata: every caller misses the map, and
// the absent-cache entry does not exist until the first load records it.
let bucket = "singleflight-bucket";
let waiters = 8;
let results = futures::future::join_all((0..waiters).map(|_| {
let sys = Arc::clone(&sys);
async move { sys.get_config(bucket).await.map(|(bm, _)| bm.name.clone()) }
}))
.await;
for result in results {
assert_eq!(result.expect("every caller must get an answer"), bucket);
}
assert_eq!(
sys.lazy_disk_loads.load(Ordering::Relaxed),
1,
"concurrent misses for one bucket must share a single disk load"
);
}
/// Pins the fail-closed caching contract of the lazy `get_config` path
/// and the refresh no-replace rule: fabricated defaults are returned but
/// never served by the map-only `get()`, persisted metadata is cached on
crates/ecstore/src/client/utils.rs
+1 -15
View File
@@ -46,16 +46,6 @@ lazy_static! {
m.insert("x-amz-replication-status".to_string(), true);
m
};
static ref SSE_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new();
m.insert("x-amz-server-side-encryption".to_string(), true);
m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
m.insert("x-amz-server-side-encryption-context".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
m
};
}
pub fn is_standard_query_value(qs_key: &str) -> bool {
@@ -70,16 +60,12 @@ pub fn is_standard_header(header_key: &str) -> bool {
*SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
}
pub fn is_sse_header(header_key: &str) -> bool {
*SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
}
pub fn is_amz_header(header_key: &str) -> bool {
let key = header_key.to_lowercase();
key.starts_with("x-amz-meta-")
|| key.starts_with("x-amz-grant-")
|| key == "x-amz-acl"
|| is_sse_header(header_key)
|| rustfs_utils::http::is_sse_header(header_key)
|| key.starts_with("x-amz-checksum-")
}
crates/ecstore/src/cluster/rpc/peer_rest_client.rs
+190 -16
View File
@@ -58,7 +58,7 @@ use std::{
collections::HashMap,
io::Cursor,
sync::{
Arc,
Arc, Weak,
atomic::{AtomicBool, Ordering},
},
time::SystemTime,
@@ -350,6 +350,44 @@ impl PeerRestClient {
}
}
fn parse_topology_host(peer_host_port: &str, grid_host: &str) -> Result<XHost> {
let url = url::Url::parse(grid_host).map_err(|_| Error::other("peer grid host is not a valid URL"))?;
if !matches!(url.scheme(), "http" | "https")
|| !url.username().is_empty()
|| url.password().is_some()
|| url.query().is_some()
|| url.fragment().is_some()
|| url.path() != "/"
{
return Err(Error::other("peer grid host has an invalid URL shape"));
}
let url_host = url.host().ok_or_else(|| Error::other("peer grid host is missing a host"))?;
let topology_host = match url.port() {
Some(port) => format!("{url_host}:{port}"),
None => url_host.to_string(),
};
let explicit_port = url.port();
let name = match url_host {
url::Host::Domain(domain) => domain.to_string(),
url::Host::Ipv4(address) => address.to_string(),
url::Host::Ipv6(address) if explicit_port.is_none() => format!("[{address}]"),
url::Host::Ipv6(address) => address.to_string(),
};
let port = url
.port_or_known_default()
.filter(|port| *port > 0)
.ok_or_else(|| Error::other("peer grid host is missing a valid port"))?;
let host = XHost {
name,
port,
is_port_set: explicit_port.is_some(),
};
if topology_host != peer_host_port {
return Err(Error::other("peer topology host does not match its grid URL"));
}
Ok(host)
}
fn build_clients_from_slots(
slots: Vec<(String, Option<String>, bool)>,
) -> (Vec<Option<Self>>, Vec<Option<Self>>, Vec<String>) {
@@ -363,14 +401,14 @@ impl PeerRestClient {
}
let client = match grid_host {
Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
Some(grid_host) => match Self::parse_topology_host(&peer_host_port, &grid_host) {
Ok(host) => {
let mut client = PeerRestClient::new(host, grid_host);
client.topology_member = peer_host_port.clone();
Some(client)
}
Err(err) => {
warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
warn!(peer = %peer_host_port, "peer topology host parse failed while constructing peer client: {err:?}");
None
}
},
@@ -519,9 +557,8 @@ impl PeerRestClient {
}
let grid_host = self.grid_host.clone();
let offline = Arc::clone(&self.offline);
let recovery_running = Arc::clone(&self.recovery_running);
let span = Self::recovery_monitor_span(&grid_host);
let offline = Arc::downgrade(&self.offline);
let recovery_running = Arc::downgrade(&self.recovery_running);
// The offline flag and its recovery are the silent half of
// rustfs/backlog#888: log the monitor's start and its success so an
// "offline then back" episode leaves a trace on the observing node.
@@ -530,13 +567,34 @@ impl PeerRestClient {
grid_host = %self.grid_host,
"peer RPC connection marked offline after a network-like failure; starting background recovery monitor"
);
drop(Self::spawn_recovery_monitor(grid_host, offline, recovery_running));
}
fn spawn_recovery_monitor(
grid_host: String,
offline: Weak<AtomicBool>,
recovery_running: Weak<AtomicBool>,
) -> tokio::task::JoinHandle<()> {
let span = Self::recovery_monitor_span(&grid_host);
super::spawn_background_monitor(span, async move {
let mut delay = get_drive_active_check_interval();
let connect_timeout = get_drive_active_check_timeout();
for attempt in 1..=PEER_REST_RECOVERY_MAX_ATTEMPTS {
if offline.strong_count() == 0 || recovery_running.strong_count() == 0 {
return;
}
tokio::time::sleep(delay).await;
if offline.strong_count() == 0 || recovery_running.strong_count() == 0 {
return;
}
if Self::perform_connectivity_check(&grid_host, connect_timeout).await.is_ok() {
let Some(offline) = offline.upgrade() else {
return;
};
let Some(recovery_running) = recovery_running.upgrade() else {
return;
};
offline.store(false, Ordering::Release);
recovery_running.store(false, Ordering::Release);
info!(
@@ -556,8 +614,10 @@ impl PeerRestClient {
attempts = PEER_REST_RECOVERY_MAX_ATTEMPTS,
"peer recovery monitor reached max attempts; will retry on next request"
);
recovery_running.store(false, Ordering::Release);
});
if let Some(recovery_running) = recovery_running.upgrade() {
recovery_running.store(false, Ordering::Release);
}
})
}
#[cfg(test)]
@@ -1807,9 +1867,13 @@ fn tier_config_reload_status_outcome(status: tonic::Status) -> TierConfigReloadO
mod tests {
use super::*;
use crate::config::com::STORAGE_CLASS_SUB_SYS;
use crate::layout::{disks_layout::DisksLayout, endpoints::SetupType};
use rustfs_config::{ENV_KUBERNETES_SERVICE_HOST, ENV_LOCAL_ENDPOINT_HOST, ENV_STARTUP_TOPOLOGY_WAIT_MODE};
use serde_json::Value;
use serial_test::serial;
use std::io::{self, Write};
use std::sync::{Arc, Mutex};
use temp_env::async_with_vars;
use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt};
#[test]
@@ -1927,30 +1991,115 @@ mod tests {
fn build_clients_from_slots_preserves_missing_remote_topology_slots() {
let slots = vec![
("127.0.0.1:9000".to_string(), None, true),
("127.0.0.1:9001".to_string(), Some("http://127.0.0.1:9001".to_string()), false),
(
"rustfs-1.invalid:9001".to_string(),
Some("http://rustfs-1.invalid:9001".to_string()),
false,
),
("rustfs-2.invalid".to_string(), Some("http://rustfs-2.invalid".to_string()), false),
("127.0.0.1:notaport".to_string(), Some("http://127.0.0.1:notaport".to_string()), false),
("127.0.0.1:9003".to_string(), None, false),
];
let (remote, all, remote_topology_hosts) = PeerRestClient::build_clients_from_slots(slots);
assert_eq!(remote.len(), 3, "local node is excluded but remote slots are not compacted away");
assert_eq!(all.len(), 4, "all slots preserve the sorted cluster topology shape");
assert_eq!(remote.len(), 4, "local node is excluded but remote slots are not compacted away");
assert_eq!(all.len(), 5, "all slots preserve the sorted cluster topology shape");
assert_eq!(
remote_topology_hosts,
vec![
"127.0.0.1:9001".to_string(),
"rustfs-1.invalid:9001".to_string(),
"rustfs-2.invalid".to_string(),
"127.0.0.1:notaport".to_string(),
"127.0.0.1:9003".to_string()
]
);
assert!(remote[0].is_some(), "valid remote peer should get a client");
assert!(remote[1].is_none(), "unparseable remote peer should remain observable as a missing slot");
assert!(remote[2].is_none(), "missing grid host should remain observable as a missing slot");
let unresolved = remote[0]
.as_ref()
.expect("temporarily unresolved remote peer should retain a client");
assert_eq!(unresolved.host.to_string(), "rustfs-1.invalid:9001");
let default_port = remote[1]
.as_ref()
.expect("temporarily unresolved scheme-default remote peer should retain a client");
assert_eq!(default_port.host.to_string(), "rustfs-2.invalid");
assert_eq!(default_port.host.port, 80);
assert!(!default_port.host.is_port_set);
assert!(remote[2].is_none(), "unparseable remote peer should remain observable as a missing slot");
assert!(remote[3].is_none(), "missing grid host should remain observable as a missing slot");
assert!(all[0].is_none(), "local node is represented by the local server_info row");
assert!(all[1].is_some());
assert!(all[2].is_none());
assert!(all[2].is_some());
assert!(all[3].is_none());
assert!(all[4].is_none());
}
#[test]
fn topology_host_parser_preserves_names_and_bracketed_ipv6() {
let domain = PeerRestClient::parse_topology_host("rustfs-1.invalid", "https://rustfs-1.invalid")
.expect("unresolved HTTPS topology host should parse without DNS");
assert_eq!(domain.to_string(), "rustfs-1.invalid");
assert_eq!(domain.port, 443);
assert!(!domain.is_port_set);
let ipv6 = PeerRestClient::parse_topology_host("[2001:db8::1]:9000", "http://[2001:db8::1]:9000")
.expect("bracketed IPv6 topology host should parse without changing its identity");
assert_eq!(ipv6.to_string(), "[2001:db8::1]:9000");
let default_port_ipv6 = PeerRestClient::parse_topology_host("[2001:db8::2]", "http://[2001:db8::2]")
.expect("scheme-default IPv6 topology host should parse without DNS");
assert_eq!(default_port_ipv6.to_string(), "[2001:db8::2]");
assert_eq!(default_port_ipv6.port, 80);
assert!(!default_port_ipv6.is_port_set);
assert!(PeerRestClient::parse_topology_host("peer.invalid:0", "http://peer.invalid:0").is_err());
assert!(PeerRestClient::parse_topology_host("peer-a.invalid:9000", "http://peer-b.invalid:9000").is_err());
assert!(PeerRestClient::parse_topology_host("peer.invalid:9000", "http://peer.invalid:9000/unexpected").is_err());
}
#[tokio::test]
#[serial]
async fn unresolved_default_port_endpoint_topology_retains_all_peer_clients() {
let volumes = (0..4)
.map(|index| format!("http://rustfs-{index}.invalid:80/data{index}"))
.collect::<Vec<_>>();
let layout = DisksLayout::from_volumes(&volumes).expect("distributed default-port topology should parse");
async_with_vars(
[
(ENV_STARTUP_TOPOLOGY_WAIT_MODE, Some("orchestrated")),
(ENV_LOCAL_ENDPOINT_HOST, Some("rustfs-0.invalid")),
(ENV_KUBERNETES_SERVICE_HOST, None),
],
async {
let (server_pools, setup_type) = EndpointServerPools::create_server_endpoints("0.0.0.0:80", &layout)
.await
.expect("explicit local identity should avoid peer DNS during endpoint construction");
assert_eq!(setup_type, SetupType::DistErasure);
let (remote, all, remote_topology_hosts) =
PeerRestClient::build_clients_from_slots(server_pools.peer_grid_host_slots_sorted());
assert_eq!(remote.len(), 3);
assert!(
remote.iter().all(Option::is_some),
"unresolved remote peers must retain reconnectable clients"
);
assert_eq!(all.len(), 4);
assert_eq!(all.iter().filter(|client| client.is_none()).count(), 1);
assert_eq!(remote_topology_hosts.len(), 3);
assert!(
remote_topology_hosts.iter().all(|host| !host.contains(':')),
"scheme-default ports must preserve the legacy topology identity"
);
assert!(
remote
.iter()
.flatten()
.all(|client| client.host.port == 80 && !client.host.is_port_set),
"scheme-default peers must retain the effective dial port"
);
},
)
.await;
}
#[test]
@@ -2740,6 +2889,31 @@ mod tests {
assert!(!client.offline.load(Ordering::Acquire));
}
#[tokio::test(start_paused = true)]
async fn dropped_peer_client_releases_and_stops_its_recovery_monitor() {
let client = test_peer_client();
client.offline.store(true, Ordering::Release);
client.recovery_running.store(true, Ordering::Release);
let offline = Arc::downgrade(&client.offline);
let recovery_running = Arc::downgrade(&client.recovery_running);
let handle = PeerRestClient::spawn_recovery_monitor(client.grid_host.clone(), offline.clone(), recovery_running.clone());
let started = tokio::time::Instant::now();
drop(client);
assert!(offline.upgrade().is_none(), "detached recovery must not retain offline state");
assert!(
recovery_running.upgrade().is_none(),
"detached recovery must not retain its running state"
);
handle.await.expect("recovery monitor should not panic");
assert_eq!(
tokio::time::Instant::now(),
started,
"recovery monitor should stop before advancing to its first delayed probe"
);
}
#[tokio::test]
async fn peer_rest_client_finalize_result_keeps_online_for_app_errors_mentioning_unavailable() {
// Regression: application error text containing "unavailable" (a
crates/ecstore/src/cluster/rpc/remote_disk.rs
+4 -2
View File
@@ -4814,9 +4814,11 @@ mod tests {
async fn test_remote_disk_endpoints_with_different_schemes() {
let test_cases = vec![
("http://server:9000", "server:9000"),
("https://secure-server:443", "secure-server"), // Default HTTPS port is omitted
("http://plain-server:80", "plain-server"),
("http://plain-server", "plain-server"),
("https://secure-server:443", "secure-server"),
("http://192.168.1.100:8080", "192.168.1.100:8080"),
("https://secure-server", "secure-server"), // No port specified
("https://secure-server", "secure-server"),
];
for (url_str, expected_hostname) in test_cases {
crates/ecstore/src/config/com.rs
+642 -53
View File
@@ -53,12 +53,14 @@ use std::sync::LazyLock;
use std::sync::{Arc, RwLock};
use tokio::sync::{OwnedRwLockWriteGuard, RwLock as AsyncRwLock};
use tracing::{debug, error, info, instrument, warn};
use uuid::Uuid;
pub const CONFIG_PREFIX: &str = "config";
const SERVER_CONFIG_OBJECT: &str = "config/config.json";
const CONFIG_TRANSACTION_LOCK_SUFFIX: &str = ".transaction.lock";
// Server-config lock order: SERVER_CONFIG_LOCK -> distributed namespace lock
// for SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order.
// Server-config lock order: SERVER_CONFIG_LOCK -> transaction lock ->
// SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order.
static SERVER_CONFIG_LOCK: LazyLock<Arc<AsyncRwLock<()>>> = LazyLock::new(|| Arc::new(AsyncRwLock::new(())));
fn config_task_join_error(operation: &'static str, error: tokio::task::JoinError) -> Error {
@@ -76,8 +78,11 @@ where
T: Send + 'static,
{
tokio::spawn(async move {
// Lock order: SERVER_CONFIG_LOCK -> namespace write lock.
// Lock order: SERVER_CONFIG_LOCK -> transaction lock -> object lock.
let _local_guard = SERVER_CONFIG_LOCK.write().await;
let transaction_lock = server_config_transaction_lock_path();
let transaction_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?;
let _transaction_guard = transaction_lock.get_write_lock(get_lock_acquire_timeout()).await?;
let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?;
Ok(operation().await)
@@ -96,8 +101,11 @@ where
T: Send + 'static,
{
tokio::spawn(async move {
// Lock order: SERVER_CONFIG_LOCK -> namespace read lock.
// Lock order: SERVER_CONFIG_LOCK -> transaction lock -> object lock.
let _local_guard = SERVER_CONFIG_LOCK.read().await;
let transaction_lock = server_config_transaction_lock_path();
let transaction_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?;
let _transaction_guard = transaction_lock.get_read_lock(get_lock_acquire_timeout()).await?;
let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?;
Ok(operation().await)
@@ -567,6 +575,21 @@ where
}
pub async fn save_config_with_opts<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
save_config_with_opts_and_metadata(api, file, data, opts).await.map(|_| ())
}
async fn save_config_with_opts_and_metadata<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<ObjectInfo>
where
S: ObjectIO<
Error = Error,
@@ -579,11 +602,13 @@ where
>,
{
let mut put_data = PutObjReader::from_vec(data);
if let Err(err) = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
error!("save_config_with_opts: err: {:?}, file: {}", err, file);
return Err(err);
match api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
Ok(object_info) => Ok(object_info),
Err(err) => {
error!("save_config_with_opts: err: {:?}, file: {}", err, file);
Err(err)
}
}
Ok(())
}
fn new_server_config() -> Config {
@@ -594,8 +619,12 @@ async fn new_and_save_server_config<S>(api: Arc<S>) -> Result<Config>
where
S: EcstoreObjectIO + StorageAdminApi + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
{
let snapshot = read_server_config_snapshot(api.clone()).await?;
if snapshot.object_exists() {
return Ok(snapshot.config.clone());
}
let cfg = new_server_config();
save_server_config(api, &cfg).await?;
save_server_config_snapshot(api, &cfg, &snapshot).await?;
Ok(cfg)
}
@@ -617,6 +646,10 @@ pub fn server_config_path() -> String {
SERVER_CONFIG_OBJECT.to_string()
}
fn server_config_transaction_lock_path() -> String {
format!("{}{CONFIG_TRANSACTION_LOCK_SUFFIX}", server_config_path())
}
fn storage_class_kvs_mut(cfg: &mut Config) -> &mut KVS {
let sub_cfg = cfg.0.entry(STORAGE_CLASS_SUB_SYS.to_string()).or_insert_with(|| {
let mut section = HashMap::new();
@@ -819,6 +852,9 @@ fn apply_external_scalar_config_map(
let Some(config_value) = root.get(descriptor.subsystem_key) else {
return Ok(false);
};
if descriptor.subsystem_key == HEAL_SUB_SYS && config_value.is_null() {
return Ok(false);
}
let overrides = decode_scalar_config_value(config_value, descriptor)?;
if overrides.is_empty() {
@@ -1463,21 +1499,142 @@ fn build_audit_object(cfg: &Config) -> Map<String, Value> {
build_target_object(cfg, &audit_target_descriptors())
}
fn sync_rendered_target_instance(existing: Value, rendered: Option<&Value>, valid_keys: &[&str]) -> Option<Value> {
match existing {
Value::Object(mut instance) => {
for key in valid_keys {
instance.remove(*key);
}
if let Some(Value::Object(rendered)) = rendered {
instance.extend(rendered.clone());
}
(!instance.is_empty()).then_some(Value::Object(instance))
}
Value::Array(entries) => {
let mut pending = rendered
.and_then(Value::as_object)
.map(|rendered| {
rendered
.iter()
.filter_map(|(key, value)| parse_target_scalar_value(key, value).map(|value| (key.clone(), value)))
.collect::<HashMap<_, _>>()
})
.unwrap_or_default();
let mut updated = Vec::with_capacity(entries.len().saturating_add(pending.len()));
for entry in entries {
let Some(entry_obj) = entry.as_object() else {
updated.push(entry);
continue;
};
let Some(key) = entry_obj.get("key").and_then(Value::as_str) else {
updated.push(entry);
continue;
};
if !valid_keys.contains(&key) {
updated.push(entry);
continue;
}
let Some(value) = pending.remove(key) else {
continue;
};
let mut entry_obj = entry_obj.clone();
entry_obj.insert("value".to_string(), Value::String(value));
updated.push(Value::Object(entry_obj));
}
updated.extend(rendered_scalar_config_kvs_entries(&pending));
(!updated.is_empty()).then_some(Value::Array(updated))
}
value if rendered.is_none() => Some(value),
_ => rendered.cloned(),
}
}
fn sync_rendered_target_object(
target_obj: &mut Map<String, Value>,
rendered_target: &Map<String, Value>,
descriptors: &[TargetConfigDescriptor],
) {
for descriptor in descriptors {
match rendered_target.get(descriptor.external_key) {
Some(Value::Object(v)) => {
target_obj.insert(descriptor.external_key.to_string(), Value::Object(v.clone()));
target_obj.remove(descriptor.subsystem_key);
let existing = target_obj.remove(descriptor.external_key);
let alias = target_obj.remove(descriptor.subsystem_key);
let mut section = existing
.or(alias)
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
let rendered = rendered_target.get(descriptor.external_key).and_then(Value::as_object);
if is_target_instance_shorthand(&section, descriptor.valid_keys) {
let has_named_instances = rendered.is_some_and(|instances| instances.keys().any(|name| name != "default"));
if !has_named_instances {
if let Some(section) = sync_rendered_target_instance(
Value::Object(section),
rendered.and_then(|instances| instances.get("default")),
descriptor.valid_keys,
) {
target_obj.insert(descriptor.external_key.to_string(), section);
}
continue;
}
_ => {
target_obj.remove(descriptor.external_key);
target_obj.remove(descriptor.subsystem_key);
let mut nested = Map::new();
if let Some(default) = sync_rendered_target_instance(
Value::Object(section),
rendered.and_then(|instances| instances.get("default")),
descriptor.valid_keys,
) {
nested.insert("default".to_string(), default);
}
if let Some(rendered) = rendered {
for (instance_name, instance) in rendered {
if instance_name != "default" {
nested.insert(instance_name.clone(), instance.clone());
}
}
}
if !nested.is_empty() {
target_obj.insert(descriptor.external_key.to_string(), Value::Object(nested));
}
continue;
}
if let Some(default_alias) = section.remove(DEFAULT_DELIMITER) {
if let Some(default) = section.get_mut("default") {
if let Some(alias) = sync_rendered_target_instance(default_alias, None, descriptor.valid_keys) {
match (default, alias) {
(Value::Object(default), Value::Object(alias)) => {
for (key, value) in alias {
default.entry(key).or_insert(value);
}
}
(Value::Array(default), Value::Array(alias)) => default.extend(alias),
_ => {}
}
}
} else {
section.insert("default".to_string(), default_alias);
}
}
let mut merged = Map::new();
for (instance_name, instance) in section {
if let Some(instance) = sync_rendered_target_instance(
instance,
rendered.and_then(|instances| instances.get(&instance_name)),
descriptor.valid_keys,
) {
merged.insert(instance_name, instance);
}
}
if let Some(rendered) = rendered {
for (instance_name, instance) in rendered {
if !merged.contains_key(instance_name) {
merged.insert(instance_name.clone(), instance.clone());
}
}
}
if !merged.is_empty() {
target_obj.insert(descriptor.external_key.to_string(), Value::Object(merged));
}
}
}
@@ -1496,6 +1653,14 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8
Some(Value::Object(v)) => v,
_ => Map::new(),
};
for key in [
storageclass::CLASS_STANDARD,
storageclass::CLASS_RRS,
storageclass::OPTIMIZE,
storageclass::INLINE_BLOCK,
] {
sc_obj.remove(key);
}
for (k, v) in build_storageclass_object(cfg) {
sc_obj.insert(k, v);
}
@@ -1503,7 +1668,10 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8
root.remove("storage_class");
for descriptor in [scanner_config_descriptor(), heal_config_descriptor()] {
let existing = root.remove(descriptor.subsystem_key);
let mut existing = root.remove(descriptor.subsystem_key);
if descriptor.subsystem_key == HEAL_SUB_SYS && existing.as_ref().is_some_and(Value::is_null) {
existing = None;
}
let rendered = build_scalar_config_object(cfg, descriptor);
if let Some(config_value) = sync_rendered_scalar_config_value(existing, &rendered, descriptor)? {
root.insert(descriptor.subsystem_key.to_string(), config_value);
@@ -1560,6 +1728,7 @@ fn is_standard_object_server_config(data: &[u8]) -> bool {
matches!(root.get("version"), Some(Value::String(v)) if !v.trim().is_empty())
&& matches!(root.get("storageclass"), Some(Value::Object(_)))
&& !root.contains_key("storage_class")
&& !matches!(root.get(HEAL_SUB_SYS), Some(Value::Null))
}
fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool {
@@ -1593,7 +1762,7 @@ where
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
> + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
{
if let Some(decrypt) = &decrypt_fn {
register_server_config_decrypt_fn(decrypt.clone());
@@ -1601,14 +1770,7 @@ where
let config_file = server_config_path();
match api
.get_object_info(
RUSTFS_META_BUCKET,
&config_file,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.get_object_info(RUSTFS_META_BUCKET, &config_file, &ObjectOptions::default())
.await
{
Ok(_) => {
@@ -1624,7 +1786,6 @@ where
let opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
@@ -1677,7 +1838,33 @@ where
}
};
match save_config(api, &config_file, normalized).await {
let snapshot = match read_server_config_snapshot(api.clone()).await {
Ok(snapshot) => snapshot,
Err(err) => {
warn!("recheck target server config failed, skip migration: {:?}", err);
return;
}
};
if snapshot.object_exists() {
debug!("server config was created while legacy migration was preparing, skip migration");
return;
}
match save_config_with_opts(
api,
&config_file,
normalized,
&ObjectOptions {
max_parity: true,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
}),
..Default::default()
},
)
.await
{
Ok(()) => {
info!("Migrated compatible server config from legacy metadata bucket");
}
@@ -1769,8 +1956,13 @@ where
{
let config_file = server_config_path();
// Try to read the configuration file
match read_config_no_lock(api.clone(), &config_file).await {
// Try to read the configuration file.
let data = if namespace_lock_held {
read_config_no_lock(api.clone(), &config_file).await
} else {
read_config(api.clone(), &config_file).await
};
match data {
Ok(data) => read_server_config(api, &data, namespace_lock_held).await,
Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", namespace_lock_held).await,
Err(err) => handle_config_read_error(err, &config_file),
@@ -1787,7 +1979,12 @@ where
warn!("Received empty configuration data, try to reread from '{}'", config_file);
// Try to read the configuration again
match read_config_no_lock(api.clone(), &config_file).await {
let data = if namespace_lock_held {
read_config_no_lock(api.clone(), &config_file).await
} else {
read_config(api.clone(), &config_file).await
};
match data {
Ok(cfg_data) => {
let cfg = decode_persisted_server_config(&cfg_data)?;
return Ok(cfg.merge());
@@ -2036,11 +2233,16 @@ pub struct ServerConfigSnapshot {
raw: Option<Vec<u8>>,
seed: Option<Vec<u8>>,
etag: Option<String>,
generation: Option<Uuid>,
_local_guard: OwnedRwLockWriteGuard<()>,
_guard: rustfs_lock::NamespaceLockGuard,
}
impl ServerConfigSnapshot {
pub fn object_exists(&self) -> bool {
self.raw.is_some()
}
pub fn ensure_lock_held(&self) -> Result<()> {
if self._guard.is_lock_lost() {
return Err(Error::other("server config transaction lock was lost"));
@@ -2051,12 +2253,34 @@ impl ServerConfigSnapshot {
pub fn is_lock_lost(&self) -> bool {
self._guard.is_lock_lost()
}
pub fn generation(&self) -> Option<Uuid> {
self.generation
}
}
/// Read a server config transaction snapshot while holding the same local and
/// distributed write locks used by every other server-config writer. Internal
/// reads and the later conditional write use no-lock object I/O; the guards
/// remain live until the snapshot is dropped.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ServerConfigSaveResult {
persisted: bool,
generation: Option<Uuid>,
}
impl ServerConfigSaveResult {
pub fn persisted(&self) -> bool {
self.persisted
}
pub fn generation(&self) -> Option<Uuid> {
self.generation
}
}
/// Read a server config transaction snapshot while holding a dedicated
/// transaction lock. The config object's normal namespace lock remains
/// available to fence reads and the conditional write at commit time.
/// The transaction guard remains live until the snapshot is dropped,
/// serializing persistence and history ordering across admin nodes. Runtime
/// state is reloaded from the durable object after this guard is released.
pub async fn read_server_config_snapshot<S>(api: Arc<S>) -> Result<ServerConfigSnapshot>
where
S: ObjectIO<
@@ -2071,12 +2295,10 @@ where
{
let config_file = server_config_path();
let local_guard = SERVER_CONFIG_LOCK.clone().write_owned().await;
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &config_file).await?;
let transaction_lock = server_config_transaction_lock_path();
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?;
let guard = lock.get_write_lock(get_lock_acquire_timeout()).await?;
let read_options = ObjectOptions {
no_lock: true,
..Default::default()
};
let read_options = ObjectOptions::default();
match read_config_with_metadata_inner(api, &config_file, &read_options, true).await {
Ok((raw, object_info)) => {
let (config, seed) = decode_persisted_server_config_with_seed(&raw)?;
@@ -2085,6 +2307,7 @@ where
raw: Some(raw),
seed: Some(seed),
etag: object_info.etag,
generation: object_info.data_dir.filter(|generation| !generation.is_nil()),
_local_guard: local_guard,
_guard: guard,
})
@@ -2094,6 +2317,7 @@ where
raw: None,
seed: None,
etag: None,
generation: None,
_local_guard: local_guard,
_guard: guard,
}),
@@ -2108,6 +2332,27 @@ where
/// lock, so a concurrent update or transaction lease loss cannot commit an
/// unfenced overwrite.
pub async fn save_server_config_snapshot<S>(api: Arc<S>, cfg: &Config, snapshot: &ServerConfigSnapshot) -> Result<bool>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
{
save_server_config_snapshot_with_generation(api, cfg, snapshot)
.await
.map(|result| result.persisted())
}
pub async fn save_server_config_snapshot_with_generation<S>(
api: Arc<S>,
cfg: &Config,
snapshot: &ServerConfigSnapshot,
) -> Result<ServerConfigSaveResult>
where
S: ObjectIO<
Error = Error,
@@ -2126,13 +2371,19 @@ where
&& configs_semantically_equal(&snapshot.config, cfg)
{
debug!("server config unchanged and already in standard object shape, skip write");
return Ok(false);
return Ok(ServerConfigSaveResult {
persisted: false,
generation: snapshot.generation(),
});
}
let data = encode_server_config_blob(cfg, snapshot.seed.as_deref())?;
if snapshot.raw.as_deref().is_some_and(|current| current == data.as_slice()) {
debug!("server config bytes unchanged after encode, skip write");
return Ok(false);
return Ok(ServerConfigSaveResult {
persisted: false,
generation: snapshot.generation(),
});
}
let http_preconditions = if snapshot.raw.is_some() {
@@ -2152,19 +2403,22 @@ where
}
};
save_config_with_opts(
snapshot.ensure_lock_held()?;
let object_info = save_config_with_opts_and_metadata(
api,
&config_file,
data,
&ObjectOptions {
max_parity: true,
no_lock: true,
http_preconditions: Some(http_preconditions),
..Default::default()
},
)
.await?;
Ok(true)
Ok(ServerConfigSaveResult {
persisted: true,
generation: object_info.data_dir.filter(|generation| !generation.is_nil()),
})
}
/// Saves the server config while an upper layer holds the namespace write
@@ -2301,8 +2555,9 @@ mod tests {
use super::{
SERVER_CONFIG_LOCK, ServerConfigSnapshot, apply_dynamic_config_for_sub_sys_with, config_task_join_error,
configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config,
lookup_configs, read_config, read_config_preserve_empty, read_config_with_metadata, read_config_without_migrate,
read_server_config_snapshot, save_server_config, save_server_config_snapshot, server_config_path, storage_class_kvs_mut,
lookup_configs, new_and_save_server_config, read_config, read_config_preserve_empty, read_config_with_metadata,
read_config_without_migrate, read_server_config_snapshot, save_server_config, save_server_config_snapshot,
save_server_config_snapshot_with_generation, server_config_transaction_lock_path, storage_class_kvs_mut,
};
use crate::config::{audit, heal, notify, oidc, scanner};
use crate::disk::endpoint::Endpoint;
@@ -2311,7 +2566,9 @@ mod tests {
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::runtime::sources as runtime_sources;
use crate::set_disk::SetDisks;
use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, range::HTTPRangeSpec};
use crate::storage_api_contracts::{
admin::StorageAdminApi, namespace::NamespaceLocking as _, object::HTTPPreconditions, range::HTTPRangeSpec,
};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{
@@ -3104,6 +3361,85 @@ mod tests {
);
}
#[test]
fn root_heal_null_decodes_as_no_override_and_is_canonicalized_on_save() {
let seed = br#"{
"version":"33",
"storageclass":{"standard":"","rrs":""},
"heal":null,
"future_root":{"mode":"keep"},
"openid":{"default":{
"config_url":"https://issuer.example/.well-known/openid-configuration",
"client_id":"console",
"client_secret":"oidc-secret",
"future_provider_control":"keep"
}},
"notify":{"webhook":{"primary":{
"enable":true,
"endpoint":"https://notify.example/hook",
"auth_token":"notify-secret",
"future_notify_control":"keep"
}}},
"logger":{"webhook":{"primary":{
"enable":true,
"endpoint":"https://audit.example/hook",
"auth_token":"audit-secret",
"future_audit_control":"keep"
}}}
}"#;
let cfg = decode_server_config_blob(seed).expect("root heal null should mean no persisted override");
assert!(cfg.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER).is_none());
assert!(!is_standard_object_server_config(seed));
let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("legacy seed should canonicalize on an authorized save");
let value: Value = serde_json::from_slice(&encoded).expect("canonical config should be valid JSON");
assert!(value.get(HEAL_SUB_SYS).is_none());
assert_eq!(value["future_root"]["mode"].as_str(), Some("keep"));
assert_eq!(value["openid"]["default"]["client_secret"].as_str(), Some("oidc-secret"));
assert_eq!(value["openid"]["default"]["future_provider_control"].as_str(), Some("keep"));
assert_eq!(value["notify"]["webhook"]["primary"]["auth_token"].as_str(), Some("notify-secret"));
assert_eq!(value["notify"]["webhook"]["primary"]["future_notify_control"].as_str(), Some("keep"));
assert_eq!(value["logger"]["webhook"]["primary"]["auth_token"].as_str(), Some("audit-secret"));
assert_eq!(value["logger"]["webhook"]["primary"]["future_audit_control"].as_str(), Some("keep"));
assert!(is_standard_object_server_config(&encoded));
}
#[test]
fn invalid_scalar_and_nested_null_config_shapes_remain_rejected() {
let invalid_sections = [
r#""scanner":null"#,
r#""heal":"""#,
r#""heal":false"#,
r#""heal":0"#,
r#""heal":{"default":null}"#,
r#""heal":{"_":null}"#,
r#""heal":{"bitrot_cycle":null}"#,
r#""heal":[{"key":"bitrot_cycle","value":null}]"#,
];
for section in invalid_sections {
let input = format!(r#"{{"version":"33","storageclass":{{"standard":"","rrs":""}},{section}}}"#);
let err = decode_server_config_blob(input.as_bytes()).expect_err("invalid scalar shape must remain rejected");
assert!(
err.to_string().contains("expected"),
"invalid section {section} returned an unrelated error: {err}"
);
}
}
#[test]
fn valid_heal_object_and_kvs_array_shapes_remain_accepted() {
let empty_object = br#"{"version":"33","storageclass":{"standard":"","rrs":""},"heal":{}}"#;
let cfg = decode_server_config_blob(empty_object).expect("empty heal object should decode as no override");
assert!(cfg.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER).is_none());
let kvs_array =
br#"{"version":"33","storageclass":{"standard":"","rrs":""},"heal":[{"key":"bitrot_cycle","value":"off"}]}"#;
let cfg = decode_server_config_blob(kvs_array).expect("heal KVS array should decode");
assert!(cfg.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER).is_some());
}
#[test]
fn scanner_update_preserves_unknown_root_and_oidc_provider_fields() {
let seed = br#"{
@@ -3131,6 +3467,171 @@ mod tests {
assert_eq!(value["openid"]["default"]["client_id"].as_str(), Some("console"));
}
#[test]
fn storageclass_reset_removes_stale_inline_block_from_seed() {
let seed = br#"{
"version":"33",
"storageclass":{
"standard":"EC:2",
"rrs":"EC:1",
"optimize":"availability",
"inline_block":"64KiB",
"future_storage_control":"keep"
}
}"#;
let encoded = encode_server_config_blob(&Config::new(), Some(seed)).expect("storageclass reset should encode");
let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json");
let storageclass = value["storageclass"].as_object().expect("storageclass object");
assert!(storageclass.get(crate::config::storageclass::INLINE_BLOCK).is_none());
assert_eq!(storageclass["future_storage_control"].as_str(), Some("keep"));
}
#[test]
fn target_update_preserves_unknown_fields_without_restoring_removed_instances() {
let seed = br#"{
"version":"33",
"storageclass":{"standard":"","rrs":""},
"notify":{"webhook":{
"primary":{
"enable":true,
"endpoint":"https://notify.example/old",
"auth_token":"notify-secret",
"future_control":"keep"
},
"removed":{"enable":true,"endpoint":"https://notify.example/removed"},
"retained":{"enable":true,"endpoint":"https://notify.example/retained","future_control":"keep"},
"enable":{"enable":true,"endpoint":"https://notify.example/named-enable","future_control":"keep"}
}}
}"#;
let mut cfg = decode_server_config_blob(seed).expect("target seed should decode");
let webhook = cfg
.0
.get_mut(NOTIFY_WEBHOOK_SUB_SYS)
.expect("notify webhook subsystem should exist");
webhook
.get_mut("primary")
.expect("primary target should exist")
.insert(rustfs_config::WEBHOOK_ENDPOINT.to_string(), "https://notify.example/new".to_string());
webhook.remove("removed");
webhook.remove("retained");
let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("target update should encode");
let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json");
let webhook = value["notify"]["webhook"].as_object().expect("webhook section");
assert_eq!(webhook["primary"]["endpoint"].as_str(), Some("https://notify.example/new"));
assert_eq!(webhook["primary"]["auth_token"].as_str(), Some("notify-secret"));
assert_eq!(webhook["primary"]["future_control"].as_str(), Some("keep"));
assert!(webhook.get("removed").is_none());
assert_eq!(webhook["retained"]["future_control"].as_str(), Some("keep"));
assert!(webhook["retained"].get("enable").is_none());
assert!(webhook["retained"].get("endpoint").is_none());
assert_eq!(webhook["enable"]["endpoint"].as_str(), Some("https://notify.example/named-enable"));
assert_eq!(webhook["enable"]["future_control"].as_str(), Some("keep"));
}
#[test]
fn shorthand_target_update_preserves_shape_and_unknown_nested_fields() {
let seed = br#"{
"version":"33",
"storageclass":{"standard":"","rrs":""},
"notify":{"webhook":{
"enable":true,
"endpoint":"https://notify.example/old",
"future_control":{"endpoint":"leave-untouched","mode":"keep"}
}}
}"#;
let mut cfg = decode_server_config_blob(seed).expect("shorthand target should decode");
cfg.0
.get_mut(NOTIFY_WEBHOOK_SUB_SYS)
.and_then(|targets| targets.get_mut(DEFAULT_DELIMITER))
.expect("default webhook target should exist")
.insert(rustfs_config::WEBHOOK_ENDPOINT.to_string(), "https://notify.example/new".to_string());
let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("shorthand target update should encode");
let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json");
let webhook = value["notify"]["webhook"].as_object().expect("webhook shorthand object");
assert_eq!(webhook["endpoint"].as_str(), Some("https://notify.example/new"));
assert!(webhook.get("default").is_none());
assert_eq!(webhook["future_control"]["endpoint"].as_str(), Some("leave-untouched"));
assert_eq!(webhook["future_control"]["mode"].as_str(), Some("keep"));
let decoded = decode_server_config_blob(&encoded).expect("updated shorthand target should remain decodable");
assert_eq!(
decoded
.get_value(NOTIFY_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER)
.expect("updated default webhook target")
.get(rustfs_config::WEBHOOK_ENDPOINT),
"https://notify.example/new"
);
}
#[test]
fn target_kvs_update_preserves_unknown_entries_and_attributes() {
let seed = br#"{
"version":"33",
"storageclass":{"standard":"","rrs":""},
"notify":{"webhook":{"primary":[
{"key":"enable","value":"on","hidden_if_empty":false},
{"key":"endpoint","value":"https://notify.example/old","future_attribute":"keep-endpoint"},
{"key":"future_control","value":"keep","future_attribute":"keep-control"}
]}}
}"#;
let mut cfg = decode_server_config_blob(seed).expect("target KVS seed should decode");
cfg.0
.get_mut(NOTIFY_WEBHOOK_SUB_SYS)
.and_then(|targets| targets.get_mut("primary"))
.expect("primary webhook target should exist")
.insert(rustfs_config::WEBHOOK_ENDPOINT.to_string(), "https://notify.example/new".to_string());
let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("target KVS update should encode");
let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json");
let entries = value["notify"]["webhook"]["primary"]
.as_array()
.expect("target KVS shape should be preserved");
let endpoint = entries
.iter()
.find(|entry| entry["key"].as_str() == Some(rustfs_config::WEBHOOK_ENDPOINT))
.expect("endpoint entry should remain");
let future = entries
.iter()
.find(|entry| entry["key"].as_str() == Some("future_control"))
.expect("unknown target entry should remain");
assert_eq!(endpoint["value"].as_str(), Some("https://notify.example/new"));
assert_eq!(endpoint["future_attribute"].as_str(), Some("keep-endpoint"));
assert_eq!(future["value"].as_str(), Some("keep"));
assert_eq!(future["future_attribute"].as_str(), Some("keep-control"));
}
#[test]
fn target_default_alias_is_canonicalized_without_losing_unknown_fields() {
let seed = br#"{
"version":"33",
"storageclass":{"standard":"","rrs":""},
"notify":{"webhook":{
"_":{"enable":false,"endpoint":"https://notify.example/alias","future_alias":"keep"},
"default":{"enable":true,"endpoint":"https://notify.example/default","future_default":"keep"}
}}
}"#;
let cfg = decode_server_config_blob(seed).expect("dual default aliases should decode");
let expected_endpoint = cfg
.get_value(NOTIFY_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER)
.expect("default webhook target should exist")
.get(rustfs_config::WEBHOOK_ENDPOINT);
let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("default alias should canonicalize");
let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json");
let webhook = value["notify"]["webhook"].as_object().expect("webhook section");
assert!(webhook.get(DEFAULT_DELIMITER).is_none());
assert_eq!(webhook["default"]["endpoint"].as_str(), Some(expected_endpoint.as_str()));
assert_eq!(webhook["default"]["future_alias"].as_str(), Some("keep"));
assert_eq!(webhook["default"]["future_default"].as_str(), Some("keep"));
}
#[test]
fn test_scanner_config_changes_are_semantically_significant() {
let baseline = Config::new();
@@ -4124,6 +4625,7 @@ mod tests {
/// What reads of the config object currently return.
enum RecoveryReadState {
Missing,
Blob(Vec<u8>),
QuorumError,
}
@@ -4136,6 +4638,7 @@ mod tests {
heal_calls: AtomicUsize,
write_calls: AtomicUsize,
last_put_no_lock: AtomicBool,
last_put_preconditions: Mutex<Option<HTTPPreconditions>>,
revision: AtomicUsize,
drive_counts: Vec<usize>,
lock_manager: Arc<rustfs_lock::GlobalLockManager>,
@@ -4150,6 +4653,7 @@ mod tests {
heal_calls: AtomicUsize::new(0),
write_calls: AtomicUsize::new(0),
last_put_no_lock: AtomicBool::new(false),
last_put_preconditions: Mutex::new(None),
revision: AtomicUsize::new(1),
drive_counts: vec![2],
lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()),
@@ -4206,6 +4710,7 @@ mod tests {
_opts: &ObjectOptions,
) -> Result<GetObjectReader> {
let data = match &*self.state.lock().expect("state lock poisoned") {
RecoveryReadState::Missing => return Err(Error::ConfigNotFound),
RecoveryReadState::Blob(data) => data.clone(),
RecoveryReadState::QuorumError => return Err(Error::ErasureReadQuorum),
};
@@ -4213,6 +4718,9 @@ mod tests {
size: data.len() as i64,
actual_size: data.len() as i64,
etag: Some(format!("config-{}", self.revision.load(Ordering::SeqCst))),
data_dir: Some(uuid::Uuid::from_u128(
u128::try_from(self.revision.load(Ordering::SeqCst)).expect("test revision should fit in u128"),
)),
..Default::default()
};
Ok(GetObjectReader {
@@ -4231,15 +4739,19 @@ mod tests {
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
let current_etag = format!("config-{}", self.revision.load(Ordering::SeqCst));
let object_exists = matches!(&*self.state.lock().expect("state lock poisoned"), RecoveryReadState::Blob(_));
if let Some(preconditions) = &opts.http_preconditions
&& (preconditions.if_match_value().is_some_and(|etag| etag != current_etag)
|| preconditions.if_none_match_value() == Some("*"))
&& (preconditions
.if_match_value()
.is_some_and(|etag| !object_exists || etag != current_etag)
|| (object_exists && preconditions.if_none_match_value() == Some("*")))
{
return Err(Error::PreconditionFailed);
}
let mut body = Vec::new();
data.stream.read_to_end(&mut body).await?;
self.last_put_no_lock.store(opts.no_lock, Ordering::SeqCst);
*self.last_put_preconditions.lock().expect("preconditions lock poisoned") = opts.http_preconditions.clone();
self.write_calls.fetch_add(1, Ordering::SeqCst);
*self.state.lock().expect("state lock poisoned") = RecoveryReadState::Blob(body.clone());
let revision = self.revision.fetch_add(1, Ordering::SeqCst) + 1;
@@ -4247,6 +4759,7 @@ mod tests {
size: i64::try_from(body.len()).expect("test config should fit in i64"),
actual_size: i64::try_from(body.len()).expect("test config should fit in i64"),
etag: Some(format!("config-{revision}")),
data_dir: Some(uuid::Uuid::from_u128(u128::try_from(revision).expect("test revision should fit in u128"))),
..Default::default()
})
}
@@ -4284,10 +4797,18 @@ mod tests {
.expect("scanner-only config change should be persisted");
assert_eq!(store.write_calls.load(Ordering::SeqCst), 1);
assert!(store.last_put_no_lock.load(Ordering::SeqCst));
assert!(!store.last_put_no_lock.load(Ordering::SeqCst));
let preconditions = store
.last_put_preconditions
.lock()
.expect("preconditions lock poisoned")
.clone()
.expect("existing config update must be conditional");
assert_eq!(preconditions.if_match_value(), Some("config-1"));
assert_eq!(preconditions.if_none_match_value(), None);
assert_eq!(
store.lock_resources.lock().expect("lock resources mutex poisoned").as_slice(),
&[server_config_path()]
&[server_config_transaction_lock_path()]
);
let decoded = read_config_without_migrate(store)
.await
@@ -4301,6 +4822,73 @@ mod tests {
);
}
#[tokio::test]
async fn server_config_snapshot_save_returns_committed_generation() {
let baseline = encode_server_config_blob(&Config::new(), None).expect("baseline config should encode");
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(baseline), None));
let snapshot = read_server_config_snapshot(store.clone())
.await
.expect("server config snapshot");
let result = save_server_config_snapshot_with_generation(store, &config_with_scanner_cycle("61"), &snapshot)
.await
.expect("conditional config save");
assert!(result.persisted());
assert_eq!(result.generation(), Some(uuid::Uuid::from_u128(2)));
}
#[tokio::test]
async fn missing_server_config_is_created_with_if_none_match() {
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None));
let cfg = config_with_scanner_cycle("61");
save_server_config(store.clone(), &cfg)
.await
.expect("missing config should be created conditionally");
assert_eq!(store.write_calls.load(Ordering::SeqCst), 1);
assert!(!store.last_put_no_lock.load(Ordering::SeqCst));
let preconditions = store
.last_put_preconditions
.lock()
.expect("preconditions lock poisoned")
.clone()
.expect("missing config create must be conditional");
assert_eq!(preconditions.if_match_value(), None);
assert_eq!(preconditions.if_none_match_value(), Some("*"));
let persisted = read_config_without_migrate(store)
.await
.expect("created config should reload");
assert_eq!(
persisted
.get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER)
.expect("persisted scanner config")
.get(SCANNER_CYCLE),
"61"
);
}
#[tokio::test]
async fn missing_config_initialization_recheck_preserves_concurrent_config() {
let existing = config_with_scanner_cycle("73");
let baseline = encode_server_config_blob(&existing, None).expect("existing config should encode");
let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(baseline), None));
let observed = new_and_save_server_config(store.clone())
.await
.expect("initialization recheck should return the config created by another writer");
assert_eq!(store.write_calls.load(Ordering::SeqCst), 0);
assert_eq!(
observed
.get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER)
.expect("concurrent scanner config")
.get(SCANNER_CYCLE),
"73"
);
}
#[tokio::test]
async fn stale_server_config_snapshot_cannot_overwrite_newer_update() {
let baseline = encode_server_config_blob(&Config::new(), None).expect("baseline config should encode");
@@ -4357,7 +4945,7 @@ mod tests {
let lock = rustfs_lock::NamespaceLock::new("server-config-lease-loss".to_string(), client.clone());
let guard = lock
.lock_guard(
rustfs_lock::ObjectKey::new(crate::disk::RUSTFS_META_BUCKET, server_config_path()),
rustfs_lock::ObjectKey::new(crate::disk::RUSTFS_META_BUCKET, server_config_transaction_lock_path()),
"server-config-lease-loss",
std::time::Duration::from_secs(1),
std::time::Duration::from_millis(120),
@@ -4372,6 +4960,7 @@ mod tests {
raw: Some(baseline.clone()),
seed: None,
etag: Some("config-0".to_string()),
generation: Some(uuid::Uuid::from_u128(1)),
_local_guard: local_guard,
_guard: guard,
};
crates/ecstore/src/core/sets.rs
+199 -20
View File
@@ -37,7 +37,9 @@ use crate::{
runtime::instance::{InstanceContext, bootstrap_ctx},
runtime::sources as runtime_sources,
set_disk::{PreparedGetObjectMetadata, SetDisks},
store::init_format::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
store::init_format::{
check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum,
},
};
use futures::{
future::join_all,
@@ -947,7 +949,7 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
#[tracing::instrument(skip(self))]
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
let (disks, _) = init_storage_disks_with_errors(
let (disks, init_errs) = init_storage_disks_with_errors(
&self.endpoints.endpoints,
&DiskOption {
cleanup: false,
@@ -955,15 +957,36 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
},
)
.await;
let (formats, errs) = load_format_erasure_all(&disks, true).await;
let (formats, mut errs) = load_format_erasure_all(&disks, true).await;
for (err, init_err) in errs.iter_mut().zip(init_errs) {
if init_err.is_some() {
*err = init_err;
}
}
if errs.iter().any(|err| {
matches!(
err,
Some(DiskError::InconsistentDisk | DiskError::CorruptedFormat | DiskError::CorruptedBackend)
)
}) {
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
}
if let Err(err) = check_format_erasure_values(&formats, self.set_drive_count) {
info!("failed to check formats erasure values: {}", err);
return Ok((HealResultItem::default(), Some(err)));
}
let ref_format = match get_format_erasure_in_quorum(&formats) {
Ok(format) => format,
let (ref_format, quorum_members) = match select_format_erasure_in_quorum(&formats, 0) {
Ok((format, members)) if format.shared_identity() == self.format.shared_identity() => (format, members),
Ok(_) => return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))),
Err(err) => return Ok((HealResultItem::default(), Some(err))),
};
if formats
.iter()
.zip(quorum_members)
.any(|(format, member)| format.is_some() && !member)
{
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
}
let mut res = HealResultItem {
heal_item_type: HealItemType::Metadata.to_string(),
detail: "disk-format".to_string(),
@@ -985,11 +1008,6 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
return Ok((res, Some(StorageError::NoHealRequired)));
}
// if !self.format.eq(&ref_format) {
// info!("format ({:?}) not eq ref_format ({:?})", self.format, ref_format);
// return Ok((res, Some(Error::new(DiskError::CorruptedFormat))));
// }
let (new_format_sets, _) = new_heal_format_sets(&ref_format, self.set_count, self.set_drive_count, &formats, &errs);
if !dry_run {
let mut tmp_new_formats = vec![None; self.set_count * self.set_drive_count];
@@ -1298,7 +1316,7 @@ mod tests {
assert_eq!(result, (Some(3), Some(1), Some(0)));
}
async fn multipart_listing_test_sets() -> (Vec<tempfile::TempDir>, Arc<Sets>) {
async fn two_set_test_sets() -> (Vec<tempfile::TempDir>, Arc<Sets>) {
let format = FormatV3::new(2, 2);
let mut temp_dirs = Vec::new();
let mut all_endpoints = Vec::new();
@@ -1339,8 +1357,8 @@ mod tests {
Arc::new(RwLock::new(disks)),
2,
1,
0,
set_index,
0,
endpoints,
format.clone(),
vec![Arc::new(LocalClient::new()), Arc::new(LocalClient::new())],
@@ -1373,11 +1391,114 @@ mod tests {
(temp_dirs, sets)
}
#[tokio::test]
async fn set_format_heal_accepts_quorum_from_a_nonzero_set() {
let (_temp_dirs, sets) = two_set_test_sets().await;
let (result, err) = sets.disk_set[1]
.heal_format(false)
.await
.expect("the second erasure set should load its own format quorum");
assert!(matches!(err, Some(StorageError::NoHealRequired)), "unexpected heal result: {err:?}");
assert_eq!(result.disk_count, 2);
assert_eq!(result.set_count, 1);
}
#[tokio::test]
async fn format_heal_rejects_foreign_majorities_at_set_and_pool_scopes() {
let (_temp_dirs, _canonical_format, sets) = setup_heal_format_sets(2, true).await;
let set_disks = set_level_heal_view(&sets).await;
let (_, set_err) = set_disks
.heal_format(false)
.await
.expect("set format heal should report a typed mismatch");
assert!(
matches!(set_err, Some(StorageError::CorruptedFormat)),
"foreign set majority must not replace the cached format: {set_err:?}"
);
let (_, pool_err) = sets
.heal_format(false)
.await
.expect("pool format heal should report a typed mismatch");
assert!(
matches!(pool_err, Some(StorageError::CorruptedFormat)),
"foreign pool majority must not replace the cached format: {pool_err:?}"
);
}
#[tokio::test]
async fn pool_format_heal_rejects_a_wrong_slot_minority() {
let (_temp_dirs, canonical_format, sets) = setup_heal_format_sets(3, false).await;
let mut poisoned_format = canonical_format.clone();
poisoned_format.erasure.this = canonical_format.erasure.sets[0][0];
replace_heal_test_format(&sets, 2, &poisoned_format).await;
let probe_err = new_disk(
&sets.endpoints.endpoints.as_ref()[2],
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect_err("a wrong-slot local format must fail disk initialization");
assert_eq!(probe_err, DiskError::InconsistentDisk);
let (_, pool_err) = sets
.heal_format(false)
.await
.expect("pool format heal should report a typed slot mismatch");
assert!(
matches!(pool_err, Some(StorageError::CorruptedFormat)),
"a wrong-slot minority must not be reported as no-heal-required: {pool_err:?}"
);
assert_eq!(
read_heal_test_format(&sets, 2).await,
poisoned_format,
"format heal must not overwrite a wrong-slot disk"
);
}
#[tokio::test]
async fn format_heal_rejects_a_foreign_minority_at_set_and_pool_scopes() {
let (_temp_dirs, canonical_format, sets) = setup_heal_format_sets(3, false).await;
let mut poisoned_format = canonical_format.clone();
poisoned_format.id = Uuid::new_v4();
poisoned_format.erasure.this = poisoned_format.erasure.sets[0][2];
replace_heal_test_format(&sets, 2, &poisoned_format).await;
let set_disks = set_level_heal_view(&sets).await;
let (_, set_err) = set_disks
.heal_format(false)
.await
.expect("set format heal should report a typed identity mismatch");
assert!(
matches!(set_err, Some(StorageError::CorruptedFormat)),
"a foreign minority must not be reported as no-heal-required: {set_err:?}"
);
let (_, pool_err) = sets
.heal_format(false)
.await
.expect("pool format heal should report a typed identity mismatch");
assert!(
matches!(pool_err, Some(StorageError::CorruptedFormat)),
"a foreign minority must not be reported as no-heal-required: {pool_err:?}"
);
assert_eq!(
read_heal_test_format(&sets, 2).await,
poisoned_format,
"format heal must not overwrite a foreign disk"
);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn list_multipart_uploads_merges_all_sets_without_pagination_loss() {
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::Erasure).await;
let (_temp_dirs, sets) = multipart_listing_test_sets().await;
let (_temp_dirs, sets) = two_set_test_sets().await;
let bucket = format!("multipart-list-{}", Uuid::new_v4().simple());
sets.make_bucket(&bucket, &MakeBucketOptions::default())
.await
@@ -1616,11 +1737,15 @@ mod tests {
// formatting the first `num_formatted` of them against a shared reference
// format and leaving the rest unformatted. Returns the live TempDir handles
// (must be kept alive), the reference format, and the assembled `Sets`.
// `disk_set` is intentionally empty: these tests only drive `heal_format`
// with `dry_run == true`, which never touches `disk_set`.
async fn setup_heal_format_sets(num_formatted: usize) -> (Vec<tempfile::TempDir>, FormatV3, Sets) {
// `disk_set` is intentionally empty: these tests only exercise paths that
// return before pool-level healing delegates into a set.
async fn setup_heal_format_sets(num_formatted: usize, foreign_identity: bool) -> (Vec<tempfile::TempDir>, FormatV3, Sets) {
const SET_DRIVE_COUNT: usize = 3;
let ref_format = FormatV3::new(1, SET_DRIVE_COUNT);
let mut stored_format = ref_format.clone();
if foreign_identity {
stored_format.id = Uuid::new_v4();
}
let mut dirs = Vec::with_capacity(SET_DRIVE_COUNT);
let mut endpoints = Vec::with_capacity(SET_DRIVE_COUNT);
@@ -1645,8 +1770,8 @@ mod tests {
)
.await
.expect("disk should be created");
let mut disk_format = ref_format.clone();
disk_format.erasure.this = ref_format.erasure.sets[0][i];
let mut disk_format = stored_format.clone();
disk_format.erasure.this = stored_format.erasure.sets[0][i];
save_format_file(&Some(disk), &Some(disk_format))
.await
.expect("format should be saved");
@@ -1677,6 +1802,60 @@ mod tests {
(dirs, ref_format, sets)
}
async fn set_level_heal_view(sets: &Sets) -> Arc<SetDisks> {
let endpoints = sets.endpoints.endpoints.as_ref().clone();
let mut disks = Vec::with_capacity(endpoints.len());
for endpoint in &endpoints {
disks.push(Some(
new_disk(
endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("fresh set-level disk handle should open"),
));
}
SetDisks::new(
"test-owner".to_string(),
Arc::new(RwLock::new(disks)),
endpoints.len(),
1,
0,
0,
endpoints,
sets.format.clone(),
Vec::new(),
)
.await
}
async fn replace_heal_test_format(sets: &Sets, disk_index: usize, format: &FormatV3) {
let disk = new_disk(
&sets.endpoints.endpoints.as_ref()[disk_index],
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("heal test disk should open");
save_format_file(&Some(disk.clone()), &Some(format.clone()))
.await
.expect("poisoned test format should be written");
}
async fn read_heal_test_format(sets: &Sets, disk_index: usize) -> FormatV3 {
let path = std::path::Path::new(&sets.endpoints.endpoints.as_ref()[disk_index].get_file_path())
.join(crate::disk::RUSTFS_META_BUCKET)
.join(crate::disk::FORMAT_CONFIG_FILE);
let data = tokio::fs::read(path).await.expect("test format should be readable");
FormatV3::try_from(data.as_slice()).expect("test format should parse")
}
// Regression for #956 (NoHealRequired path): with every disk already
// formatted, `heal_format` reports exactly one drive record per disk
// (N = set_count * set_drive_count), each carrying a real endpoint. Before
@@ -1685,7 +1864,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn heal_format_no_heal_required_reports_one_record_per_disk() {
let (_dirs, _ref_format, sets) = setup_heal_format_sets(3).await;
let (_dirs, _ref_format, sets) = setup_heal_format_sets(3, false).await;
let (res, err) = sets.heal_format(true).await.expect("heal_format should succeed");
// All disks formatted -> NoHealRequired early return, still returns `res`.
@@ -1715,7 +1894,7 @@ mod tests {
#[serial]
async fn heal_format_heal_path_reports_one_record_per_disk_aligned() {
// Disks 0 and 1 formatted (quorum), disk 2 unformatted.
let (_dirs, _ref_format, sets) = setup_heal_format_sets(2).await;
let (_dirs, _ref_format, sets) = setup_heal_format_sets(2, false).await;
let (res, err) = sets.heal_format(true).await.expect("heal_format should succeed");
// Unformatted disk present -> heal path, not NoHealRequired.
crates/ecstore/src/disk/disk_store.rs
+4
View File
@@ -1049,6 +1049,10 @@ impl LocalDiskWrapper {
Ok(())
}
pub(crate) async fn set_disk_id_state(&self, id: Option<Uuid>) {
*self.disk_id.write().await = id;
}
/// Get the current disk ID
pub async fn get_current_disk_id(&self) -> Option<Uuid> {
*self.disk_id.read().await
crates/ecstore/src/disk/local.rs
+2 -2
View File
@@ -5078,7 +5078,7 @@ impl LocalDisk {
Ok((buf, mtime))
}
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
async fn read_metadata_with_dmtime(&self, file_path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
check_path_length(file_path.as_ref().to_string_lossy().as_ref())?;
@@ -5121,7 +5121,7 @@ impl LocalDisk {
Ok((data, modtime))
}
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
// TODO: timeout support
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
crates/ecstore/src/disk/mod.rs
+78
View File
@@ -132,6 +132,18 @@ pub enum Disk {
Remote(Box<RemoteDisk>),
}
impl Disk {
pub(crate) async fn set_disk_id_state(&self, id: Option<Uuid>) -> Result<()> {
match self {
Disk::Local(local_disk) => {
local_disk.set_disk_id_state(id).await;
Ok(())
}
Disk::Remote(remote_disk) => remote_disk.set_disk_id(id).await,
}
}
}
#[async_trait::async_trait]
impl DiskAPI for Disk {
fn to_string(&self) -> String {
@@ -1552,6 +1564,72 @@ mod tests {
let _ = fs::remove_dir_all(&test_dir).await;
}
#[tokio::test]
#[serial_test::serial]
async fn local_disk_id_state_does_not_publish_to_the_process_registry() {
let local_dir = tempfile::tempdir().expect("local disk tempdir should be created");
let mut endpoint =
Endpoint::try_from(local_dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let local_disk = LocalDisk::new(&endpoint, false).await.expect("local disk should initialize");
let disk = Disk::Local(Box::new(LocalDiskWrapper::new(Arc::new(local_disk), false)));
let disk_id = Uuid::new_v4();
disk.set_disk_id_state(Some(disk_id))
.await
.expect("local wrapper state should accept a disk ID");
let Disk::Local(local_disk) = &disk else {
panic!("test disk should remain local");
};
assert_eq!(local_disk.get_current_disk_id().await, Some(disk_id));
assert!(
!crate::runtime::global::current_ctx()
.local_disk_id_map()
.read()
.await
.contains_key(&disk_id),
"state-only startup publication must not update the process disk-ID registry"
);
disk.set_disk_id_state(None)
.await
.expect("local wrapper state should clear a disk ID");
assert_eq!(local_disk.get_current_disk_id().await, None);
}
#[tokio::test]
async fn remote_disk_id_state_delegates_some_and_none() {
let mut endpoint = Endpoint::try_from("http://remote-server:9000/data").expect("remote endpoint should parse");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let remote_disk = RemoteDisk::new(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
Arc::new(crate::cluster::rpc::TcpHttpInternodeDataTransport),
)
.await
.expect("remote disk should initialize");
let disk = Disk::Remote(Box::new(remote_disk));
let disk_id = Uuid::new_v4();
disk.set_disk_id_state(Some(disk_id))
.await
.expect("remote state should accept a disk ID");
assert_eq!(disk.get_disk_id().await.expect("remote disk ID should be readable"), Some(disk_id));
disk.set_disk_id_state(None)
.await
.expect("remote state should clear a disk ID");
assert_eq!(disk.get_disk_id().await.expect("remote disk ID should be readable"), None);
}
#[tokio::test]
async fn reset_health_for_store_init_retry_delegates_to_disk_variants() {
let local_dir = tempfile::tempdir().unwrap();
crates/ecstore/src/erasure/coding/bitrot.rs
+3 -3
View File
@@ -103,7 +103,7 @@ where
/// or `out` is larger than one shard. On error `out`'s contents are
/// unspecified but never contain bytes that failed the hash check — the copy
/// happens only after verification.
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub async fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
let want = out.len();
self.begin_read(want)?;
@@ -303,7 +303,7 @@ where
/// Write a (hash+data) block. Returns the number of data bytes written.
/// Returns an error if called after a short write or if data exceeds shard_size.
#[cfg_attr(feature = "hotpath", hotpath::measure(label = "BitrotWriter::write"))]
#[hotpath::measure(label = "BitrotWriter::write")]
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if buf.is_empty() {
return Ok(0);
@@ -455,7 +455,7 @@ pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorith
/// stores those as whole-file bitrot with no interleaved hash, so the size guard
/// on the next line would reject a genuinely healthy part. Reading legacy V1
/// whole-file-bitrot objects would need a separate verification path.
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
mut r: R,
want_size: usize,
crates/ecstore/src/erasure/coding/decode.rs
+2 -2
View File
@@ -691,7 +691,7 @@ impl<R> ParallelReader<R>
where
R: crate::erasure::coding::ShardSource,
{
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
// On the reconstruction-verifying GET path, read every live shard reader
// in lockstep so all readers advance one block per stripe and stay
@@ -1505,7 +1505,7 @@ where
}
impl Erasure {
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub async fn decode<W, R>(
&self,
writer: &mut W,
crates/ecstore/src/erasure/coding/encode.rs
+4 -4
View File
@@ -504,7 +504,7 @@ impl Erasure {
Ok((reader, total))
}
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub async fn encode<R>(
self: Arc<Self>,
reader: R,
@@ -670,7 +670,7 @@ impl Erasure {
Ok((reader, total))
}
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub async fn encode_batched<R>(
self: Arc<Self>,
mut reader: R,
@@ -798,7 +798,7 @@ impl Erasure {
/// Fast path for small inline objects: skip tokio::spawn + mpsc channel.
/// Reads all data, encodes directly, writes shards sequentially.
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub async fn encode_inline_small<R>(
self: Arc<Self>,
reader: R,
@@ -813,7 +813,7 @@ impl Erasure {
/// Fast path for single-block non-inline objects: avoids the producer/consumer
/// pipeline in `encode()` while keeping the same writer/quorum/shutdown semantics.
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub async fn encode_single_block_non_inline<R>(
self: Arc<Self>,
reader: R,
crates/ecstore/src/erasure/coding/erasure.rs
+5 -5
View File
@@ -640,7 +640,7 @@ impl Erasure {
/// # Returns
/// A vector of encoded shards as `Bytes`.
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
@@ -688,7 +688,7 @@ impl Erasure {
/// Encode owned data, avoiding a copy when the caller already has a heap buffer.
/// Falls back to copying into a new buffer if zero-copy conversion fails.
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub fn encode_data_owned(&self, data: Vec<u8>) -> io::Result<Vec<Bytes>> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
@@ -752,7 +752,7 @@ impl Erasure {
/// block), the `resize(need_total_size)` below stays within capacity for every
/// `data_len <= block_size` — both shard-size formulas are monotone in
/// `data_len` — so this function never reallocates the buffer.
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub fn encode_data_bytes_mut(&self, mut data_buffer: BytesMut, data_len: usize) -> io::Result<Vec<Bytes>> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
@@ -805,7 +805,7 @@ impl Erasure {
///
/// # Returns
/// Ok if reconstruction succeeds, error otherwise.
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
if self.parity_shards > 0 {
if self.uses_legacy {
@@ -825,7 +825,7 @@ impl Erasure {
}
/// Decode and reconstruct missing data shards, then regenerate parity shards.
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub fn decode_data_and_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
if self.parity_shards > 0 {
if self.uses_legacy {
crates/ecstore/src/layout/disks_layout.rs
+12 -1
View File
@@ -203,7 +203,7 @@ fn get_all_sets<T: AsRef<str>>(set_drive_count: usize, is_ellipses: bool, args:
for args in set_args.iter() {
for arg in args {
if unique_args.contains(arg) {
return Err(Error::other(format!("Input args {arg} has duplicate ellipses")));
return Err(Error::other("input arguments contain a duplicate endpoint after ellipsis expansion"));
}
unique_args.insert(arg);
}
@@ -924,4 +924,15 @@ mod test {
}
}
}
#[test]
fn layout_errors_do_not_echo_url_credentials() {
for volumes in [
vec!["http://:duplicate-secret@server/path", "http://:duplicate-secret@server/path"],
vec!["http://:ellipsis...secret@server/path"],
] {
let err = DisksLayout::from_volumes(&volumes).unwrap_err();
assert!(!err.to_string().contains("secret"), "layout error leaked endpoint credentials: {err}");
}
}
}
crates/ecstore/src/layout/endpoint.rs
+19 -2
View File
@@ -88,6 +88,7 @@ impl TryFrom<&str> for Endpoint {
// - All field should be empty except Host and Path.
if !((url.scheme() == "http" || url.scheme() == "https")
&& url.username().is_empty()
&& url.password().is_none()
&& url.fragment().is_none()
&& url.query().is_none())
{
@@ -366,6 +367,12 @@ mod test {
expected_type: None,
expected_err: Some(Error::other("invalid URL endpoint format")),
},
TestCase {
arg: "http://:topsecret@server/path",
expected_endpoint: None,
expected_type: None,
expected_err: Some(Error::other("invalid URL endpoint format")),
},
TestCase {
arg: "http://:/path",
expected_endpoint: None,
@@ -505,8 +512,18 @@ mod test {
let endpoint = Endpoint::try_from("http://example.com:9000/path").unwrap();
assert_eq!(endpoint.host_port(), "example.com:9000");
let endpoint_no_port = Endpoint::try_from("https://example.com/path").unwrap();
assert_eq!(endpoint_no_port.host_port(), "example.com");
for endpoint in [
Endpoint::try_from("http://example.com/path").unwrap(),
Endpoint::try_from("http://example.com:80/path").unwrap(),
] {
assert_eq!(endpoint.host_port(), "example.com");
}
for endpoint in [
Endpoint::try_from("https://example.com/path").unwrap(),
Endpoint::try_from("https://example.com:443/path").unwrap(),
] {
assert_eq!(endpoint.host_port(), "example.com");
}
let file_endpoint = Endpoint::try_from("/tmp/data").unwrap();
assert_eq!(file_endpoint.host_port(), "");
crates/ecstore/src/layout/endpoints.rs
+1129 -92
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/layout/format.rs
+52 -47
View File
@@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize};
use serde_json::Error as JsonError;
use uuid::Uuid;
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
pub enum FormatMetaVersion {
#[serde(rename = "1")]
V1,
@@ -27,7 +27,7 @@ pub enum FormatMetaVersion {
Unknown,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
pub enum FormatBackend {
#[serde(rename = "xl")]
Erasure,
@@ -64,7 +64,7 @@ pub struct FormatErasureV3 {
pub distribution_algo: DistributionAlgoVersion,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
pub enum FormatErasureVersion {
#[serde(rename = "1")]
V1,
@@ -77,7 +77,7 @@ pub enum FormatErasureVersion {
Unknown,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
pub enum DistributionAlgoVersion {
#[serde(rename = "CRCMOD")]
V1,
@@ -121,6 +121,15 @@ pub struct FormatV3 {
pub disk_info: Option<DiskInfo>,
}
pub(crate) type SharedFormatIdentity<'a> = (
&'a FormatMetaVersion,
&'a FormatBackend,
&'a Uuid,
&'a FormatErasureVersion,
&'a [Vec<Uuid>],
&'a DistributionAlgoVersion,
);
impl TryFrom<&[u8]> for FormatV3 {
type Error = JsonError;
@@ -198,52 +207,24 @@ impl FormatV3 {
}
pub fn check_other(&self, other: &FormatV3) -> Result<()> {
let mut tmp = other.clone();
let this = tmp.erasure.this;
tmp.erasure.this = Uuid::nil();
if self.erasure.sets.len() != other.erasure.sets.len() {
return Err(Error::other(format!(
"Expected number of sets {}, got {}",
self.erasure.sets.len(),
other.erasure.sets.len()
)));
if self.shared_identity() != other.shared_identity() {
return Err(Error::other("storage formats do not match"));
}
for i in 0..self.erasure.sets.len() {
if self.erasure.sets[i].len() != other.erasure.sets[i].len() {
return Err(Error::other(format!(
"Each set should be of same size, expected {}, got {}",
self.erasure.sets[i].len(),
other.erasure.sets[i].len()
)));
}
self.find_disk_index_by_disk_id(other.erasure.this).map(|_| ())
}
for j in 0..self.erasure.sets[i].len() {
if self.erasure.sets[i][j] != other.erasure.sets[i][j] {
return Err(Error::other(format!(
"UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)",
i,
j,
self.erasure.sets[i][j].to_string(),
other.erasure.sets[i][j].to_string(),
)));
}
}
}
for i in 0..tmp.erasure.sets.len() {
for j in 0..tmp.erasure.sets[i].len() {
if this == tmp.erasure.sets[i][j] {
return Ok(());
}
}
}
Err(Error::other(format!(
"DriveID {:?} not found in any drive sets {:?}",
this, other.erasure.sets
)))
/// Fields that must agree across every disk in one erasure format,
/// excluding the disk-specific `this` UUID and runtime-only `disk_info`.
pub(crate) fn shared_identity(&self) -> SharedFormatIdentity<'_> {
(
&self.version,
&self.format,
&self.id,
&self.erasure.version,
&self.erasure.sets,
&self.erasure.distribution_algo,
)
}
}
@@ -437,6 +418,30 @@ mod test {
assert!(result.is_ok());
}
#[test]
fn test_check_other_rejects_shared_identity_mismatches() {
type FormatMutation = (&'static str, fn(&mut FormatV3));
let format = FormatV3::new(1, 2);
let mutations: [FormatMutation; 5] = [
("meta version", |other| other.version = FormatMetaVersion::Unknown),
("backend", |other| other.format = FormatBackend::ErasureSingle),
("deployment id", |other| other.id = Uuid::new_v4()),
("erasure version", |other| other.erasure.version = FormatErasureVersion::V2),
("distribution algorithm", |other| {
other.erasure.distribution_algo = DistributionAlgoVersion::V2
}),
];
for (field, mutate) in mutations {
let mut other = format.clone();
other.erasure.this = format.erasure.sets[0][0];
mutate(&mut other);
assert!(format.check_other(&other).is_err(), "{field} mismatch must be rejected");
}
}
#[test]
fn test_check_other_different_set_count() {
let format1 = FormatV3::new(2, 4);
crates/ecstore/src/lib.rs
+3 -1
View File
@@ -12,8 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#![recursion_limit = "256"]
/// Scope-based hotpath measurement for `#[async_trait]` methods, where
/// `#[cfg_attr(feature = "hotpath", hotpath::measure)]` would only time the boxed-future construction.
/// `#[hotpath::measure]` would only time the boxed-future construction.
/// The guard records wall time from this statement until the enclosing
/// (desugared) async block completes, including early returns via `?`.
#[cfg(feature = "hotpath")]
crates/ecstore/src/object_api/encryption.rs
+80
View File
@@ -0,0 +1,80 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use async_trait::async_trait;
use http::{HeaderMap, HeaderValue};
use std::collections::HashMap;
use std::error::Error;
use std::fmt::{Display, Formatter};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReadEncryptionMode {
Direct { base_nonce: [u8; 12] },
Object,
}
pub struct ReadEncryptionMaterial {
pub key_bytes: [u8; 32],
pub mode: ReadEncryptionMode,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncryptionResolutionErrorKind {
InvalidRequest,
InvalidMetadata,
ServiceUnavailable,
DecryptionFailed,
}
#[derive(Debug)]
pub struct EncryptionResolutionError {
kind: EncryptionResolutionErrorKind,
message: String,
}
impl EncryptionResolutionError {
pub fn new(kind: EncryptionResolutionErrorKind, message: impl Into<String>) -> Self {
Self {
kind,
message: message.into(),
}
}
pub fn kind(&self) -> EncryptionResolutionErrorKind {
self.kind
}
}
impl Display for EncryptionResolutionError {
fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.message)
}
}
impl Error for EncryptionResolutionError {}
pub struct ReadEncryptionRequest<'a> {
pub bucket: &'a str,
pub object: &'a str,
pub metadata: &'a HashMap<String, String>,
pub headers: &'a HeaderMap<HeaderValue>,
}
#[async_trait]
pub trait ObjectEncryptionResolver: Send + Sync {
async fn resolve_read_material(
&self,
request: ReadEncryptionRequest<'_>,
) -> Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError>;
}
crates/ecstore/src/object_api/mod.rs
+5
View File
@@ -84,6 +84,7 @@ pub(crate) fn legacy_encrypted_range_seek_enabled() -> bool {
}
mod body_cache_hook;
mod encryption;
mod hook_slot;
mod object_mutation_hook;
mod readers;
@@ -98,6 +99,10 @@ pub use body_cache_hook::{
pub(crate) use body_cache_hook::{
get_object_body_cache_hook, get_object_body_cache_hook_suppressed, without_get_object_body_cache_hook,
};
pub use encryption::{
EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial,
ReadEncryptionMode, ReadEncryptionRequest,
};
pub(crate) use object_mutation_hook::notify_object_mutation;
pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook};
pub use readers::*;
crates/ecstore/src/object_api/readers.rs
+137 -1509
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/object_api/types.rs
+17 -46
View File
@@ -273,29 +273,9 @@ impl ObjectInfo {
}
pub fn is_encrypted(&self) -> bool {
// Corresponding to the logic in rustfs/src/sse.rs/encryption_material_to_metadata function
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
self.user_defined.keys().any(|key| {
let lower = key.to_ascii_lowercase();
lower.starts_with("x-minio-encryption-")
|| lower.starts_with("x-minio-internal-server-side-encryption-")
|| matches!(
lower.as_str(),
"x-minio-internal-encrypted-multipart"
| "x-rustfs-encryption-key"
| "x-rustfs-encryption-algorithm"
| "x-rustfs-encryption-iv"
| "x-rustfs-encryption-key-id"
| "x-rustfs-encryption-context"
| "x-rustfs-encryption-tag"
| "x-amz-server-side-encryption-aws-kms-key-id"
| SSEC_ALGORITHM_HEADER
| SSEC_KEY_HEADER
| SSEC_KEY_MD5_HEADER
| "x-amz-server-side-encryption"
)
})
self.user_defined
.keys()
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
}
/// Maximum inline size for non-versioned objects (128 KiB).
@@ -339,26 +319,7 @@ impl ObjectInfo {
}
pub fn encryption_original_size(&self) -> std::io::Result<Option<i64>> {
let actual_size = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE);
if let Some(size_str) = self
.user_defined
.get("x-rustfs-encryption-original-size")
.map(String::as_str)
.or_else(|| {
self.user_defined
.get("x-amz-server-side-encryption-customer-original-size")
.map(String::as_str)
})
.or(actual_size.as_deref())
&& !size_str.is_empty()
{
let size = size_str
.parse::<i64>()
.map_err(|e| std::io::Error::other(format!("Failed to parse encryption original size: {e}")))?;
return Ok(Some(size));
}
Ok(None)
rustfs_utils::http::get_object_encryption_original_size(&self.user_defined)
}
pub fn decrypted_size(&self) -> std::io::Result<i64> {
@@ -388,9 +349,6 @@ impl ObjectInfo {
return Ok(actual_size);
}
// Check if object is encrypted
// Managed SSE stores original size in x-rustfs-encryption-original-size metadata
// SSE-C stores original size in x-amz-server-side-encryption-customer-original-size
if let Some(size) = self.encryption_original_size()? {
return Ok(size);
}
@@ -881,6 +839,19 @@ mod tests {
assert!(!object.is_inline_fast_path_eligible(), "transitioned objects must fall back");
}
#[test]
fn minio_internal_encryption_metadata_is_not_treated_as_plaintext() {
let object = ObjectInfo {
user_defined: Arc::new(HashMap::from([(
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key".to_string(),
"sealed".to_string(),
)])),
..Default::default()
};
assert!(object.is_encrypted());
}
#[test]
fn versions_after_marker_handles_null_version_marker() {
let first_version = Uuid::parse_str("11111111-2222-3333-4444-555555555555").unwrap();
crates/ecstore/src/runtime/instance.rs
+17
View File
@@ -46,6 +46,7 @@ use crate::bucket::metadata_sys::BucketMetadataSys;
use crate::bucket::replication::{DynReplicationPool, ReplicationStats};
use crate::disk::DiskStore;
use crate::layout::endpoints::{EndpointServerPools, SetupType};
use crate::object_api::ObjectEncryptionResolver;
use crate::services::event_notification::EventNotifier;
use crate::services::tier::tier::TierConfigMgr;
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
@@ -159,6 +160,8 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
#[cfg(test)]
@@ -197,6 +200,7 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
#[cfg(test)]
@@ -209,6 +213,19 @@ impl InstanceContext {
self.lock_manager.clone()
}
/// Install the application-owned object-encryption resolver once.
pub fn set_object_encryption_resolver(
&self,
resolver: Arc<dyn ObjectEncryptionResolver>,
) -> Result<(), Arc<dyn ObjectEncryptionResolver>> {
self.object_encryption_resolver.set(resolver)
}
/// Return the configured object-encryption resolver, if startup installed one.
pub fn object_encryption_resolver(&self) -> Option<&dyn ObjectEncryptionResolver> {
self.object_encryption_resolver.get().map(Arc::as_ref)
}
/// Set this instance's S3 region.
///
/// Write-once: panics on a second write, preserving the startup fail-fast
crates/ecstore/src/runtime/sources.rs
+132 -12
View File
@@ -27,7 +27,7 @@ use crate::{
bucket::replication::{DynReplicationPool, ReplicationStats},
config::{get_global_storage_class, get_global_storage_class_snapshot, set_global_storage_class, storageclass},
disk::{DiskAPI, DiskOption, DiskStore, new_disk},
error::Result,
error::{Error, Result},
layout::endpoints::{EndpointServerPools, SetupType},
runtime::global::{
GLOBAL_BOOT_TIME, GLOBAL_LIFECYCLE_SYS, GLOBAL_LOCAL_NODE_NAME_FALLBACK, GLOBAL_ROOT_DISK_THRESHOLD,
@@ -46,7 +46,6 @@ use crate::{
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service};
use rustfs_lock::client::LockClient;
use s3s::dto::BucketLifecycleConfiguration;
use s3s::region::Region;
@@ -105,10 +104,6 @@ pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_
global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error);
}
pub(crate) async fn object_encryption_service() -> Option<Arc<ObjectEncryptionService>> {
get_global_encryption_service().await
}
pub fn object_store_handle() -> Option<Arc<ECStore>> {
resolve_object_store_handle()
}
@@ -417,14 +412,14 @@ pub(crate) async fn clear_local_disk_id_map_for_test() {
local_disk_id_map_handle().write().await.clear();
}
pub(crate) async fn record_local_disk_id(instance_ctx: &Arc<InstanceContext>, disk_id: Uuid, endpoint: String) {
instance_ctx.local_disk_id_map().write().await.insert(disk_id, endpoint);
}
pub(crate) async fn replace_local_disk_id(previous: Option<Uuid>, current: Option<Uuid>, endpoint: String) {
let id_map = local_disk_id_map_handle();
let mut disk_id_map = id_map.write().await;
if let Some(previous_id) = previous {
if let Some(previous_id) = previous
&& disk_id_map
.get(&previous_id)
.is_some_and(|registered_endpoint| registered_endpoint == &endpoint)
{
disk_id_map.remove(&previous_id);
}
if let Some(current_id) = current {
@@ -432,6 +427,53 @@ pub(crate) async fn replace_local_disk_id(previous: Option<Uuid>, current: Optio
}
}
pub(crate) async fn reconcile_local_disk_ids(
instance_ctx: &InstanceContext,
pool_endpoints: &[String],
selected: &[(Uuid, String)],
) {
let pool_endpoints = pool_endpoints.iter().map(String::as_str).collect::<HashSet<_>>();
let disk_id_map = instance_ctx.local_disk_id_map();
let mut disk_ids = disk_id_map.write().await;
disk_ids.retain(|_, registered_endpoint| !pool_endpoints.contains(registered_endpoint.as_str()));
disk_ids.extend(selected.iter().cloned());
}
pub(crate) async fn quarantine_local_disks(instance_ctx: &InstanceContext, endpoints: &[Endpoint]) -> Result<()> {
let slots = endpoints
.iter()
.map(|endpoint| {
Ok((
usize::try_from(endpoint.pool_idx).map_err(|_| Error::CorruptedFormat)?,
usize::try_from(endpoint.set_idx).map_err(|_| Error::CorruptedFormat)?,
usize::try_from(endpoint.disk_idx).map_err(|_| Error::CorruptedFormat)?,
))
})
.collect::<Result<Vec<_>>>()?;
let local_disk_map = instance_ctx.local_disk_map();
let mut local_disks = local_disk_map.write().await;
for endpoint in endpoints {
local_disks.insert(endpoint.to_string(), None);
}
drop(local_disks);
let set_drives = instance_ctx.local_disk_set_drives();
let mut local_set_drives = set_drives.write().await;
if local_set_drives.is_empty() {
return Ok(());
}
for (pool_idx, set_idx, disk_idx) in slots {
let disk = local_set_drives
.get_mut(pool_idx)
.and_then(|sets| sets.get_mut(set_idx))
.and_then(|disks| disks.get_mut(disk_idx))
.ok_or(Error::CorruptedFormat)?;
*disk = None;
}
Ok(())
}
pub(crate) async fn record_local_disks(instance_ctx: &Arc<InstanceContext>, disks: Vec<DiskStore>) {
let map = instance_ctx.local_disk_map();
let mut global_local_disk_map = map.write().await;
@@ -558,10 +600,14 @@ pub(crate) async fn init_tier_config_mgr(store: Arc<ECStore>) -> Result<()> {
#[cfg(test)]
mod tests {
use super::{LockRegistry, local_node_name, set_local_node_name};
use super::{
LockRegistry, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name, reconcile_local_disk_ids,
replace_local_disk_id, set_local_node_name,
};
use crate::disk::endpoint::Endpoint;
use rustfs_lock::{LocalClient, LockClient};
use std::{collections::HashMap, sync::Arc};
use uuid::Uuid;
fn url_endpoint(raw: &str) -> Endpoint {
Endpoint {
@@ -607,4 +653,78 @@ mod tests {
assert_eq!(observed, next);
}
#[tokio::test]
#[serial_test::serial]
async fn clearing_a_stale_disk_id_does_not_remove_another_endpoint() {
clear_local_disk_id_map_for_test().await;
let disk_id = Uuid::new_v4();
replace_local_disk_id(None, Some(disk_id), "endpoint-a".to_string()).await;
replace_local_disk_id(Some(disk_id), None, "endpoint-b".to_string()).await;
assert_eq!(local_disk_path_by_id(&disk_id).await, Some("endpoint-a".to_string()));
clear_local_disk_id_map_for_test().await;
}
#[tokio::test]
#[serial_test::serial]
async fn reconciling_pool_disk_ids_preserves_other_endpoints() {
let instance_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let process_ctx = crate::runtime::global::current_ctx();
let bootstrap_ctx = crate::runtime::instance::bootstrap_ctx();
let retained_id = Uuid::new_v4();
let removed_id = Uuid::new_v4();
let selected_id = Uuid::new_v4();
let process_sentinel = Uuid::new_v4();
let bootstrap_sentinel = Uuid::new_v4();
instance_ctx.local_disk_id_map().write().await.extend([
(retained_id, "endpoint-a".to_string()),
(removed_id, "endpoint-b".to_string()),
]);
process_ctx
.local_disk_id_map()
.write()
.await
.insert(process_sentinel, "endpoint-b".to_string());
bootstrap_ctx
.local_disk_id_map()
.write()
.await
.insert(bootstrap_sentinel, "endpoint-b".to_string());
reconcile_local_disk_ids(
&instance_ctx,
&["endpoint-b".to_string(), "endpoint-c".to_string()],
&[(selected_id, "endpoint-c".to_string())],
)
.await;
let disk_ids = instance_ctx.local_disk_id_map();
let disk_ids = disk_ids.read().await;
assert_eq!(disk_ids.get(&retained_id).map(String::as_str), Some("endpoint-a"));
assert_eq!(disk_ids.get(&removed_id), None);
assert_eq!(disk_ids.get(&selected_id).map(String::as_str), Some("endpoint-c"));
drop(disk_ids);
assert_eq!(
process_ctx
.local_disk_id_map()
.read()
.await
.get(&process_sentinel)
.map(String::as_str),
Some("endpoint-b")
);
assert_eq!(
bootstrap_ctx
.local_disk_id_map()
.read()
.await
.get(&bootstrap_sentinel)
.map(String::as_str),
Some("endpoint-b")
);
process_ctx.local_disk_id_map().write().await.remove(&process_sentinel);
bootstrap_ctx.local_disk_id_map().write().await.remove(&bootstrap_sentinel);
}
}
crates/ecstore/src/services/notification_sys.rs
+4 -9
View File
@@ -2044,15 +2044,10 @@ fn synthesized_disks(host: &str, endpoints: &EndpointServerPools, state: ItemSta
/// Whether `peer_host` refers to the same node as an endpoint whose
/// `host_port()` is `ep_host_port`.
///
/// `PeerRestClient::host` is an `XHost`, which resolves names to an address on
/// construction (`hosts_sorted` -> `XHost::try_from` -> `to_socket_addrs`), so
/// `peer_host` is the resolved `IP:port`. An endpoint's `host_port()`, however,
/// is `url.host():port` — still the raw `hostname:port` on hostname-based
/// deployments. A plain string compare therefore misses on hostname clusters,
/// leaving the synthesized/degraded drive list empty and `unknownDisks` at 0
/// (rustfs/rustfs#4607 follow-up). Compare directly first (fast path / IP
/// deployments), then canonicalize the endpoint side through the same `XHost`
/// resolution and compare again.
/// Current topology clients preserve the endpoint `hostname:port`, so the
/// direct comparison is the normal path. The resolution fallback keeps
/// compatibility with older or manually constructed clients whose `XHost`
/// contains a resolved `IP:port` (rustfs/rustfs#4607 follow-up).
fn endpoint_host_matches(peer_host: &str, ep_host_port: &str) -> bool {
if peer_host == ep_host_port {
return true;
crates/ecstore/src/services/tier/tier.rs
+71 -3
View File
@@ -72,6 +72,7 @@ use crate::{
cluster::rpc::peer_rest_client::{PeerRestClient, PeerTierMutationState},
config::com::{CONFIG_PREFIX, read_config, read_config_with_metadata},
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
layout::endpoints::EndpointServerPools,
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
runtime::sources as runtime_sources,
set_disk::get_lock_acquire_timeout,
@@ -904,14 +905,17 @@ async fn remote_tier_mutation_peers() -> io::Result<Vec<Arc<dyn TierMutationPeer
let Some(endpoints) = runtime_sources::endpoint_pools() else {
return Err(tier_mutation_replay_error("cluster endpoint topology is not initialized"));
};
let remote_host_count = endpoints.hosts_sorted().iter().flatten().count();
let (peers, _) = PeerRestClient::new_clients(endpoints).await;
remote_tier_mutation_peers_from_topology(endpoints).await
}
async fn remote_tier_mutation_peers_from_topology(endpoints: EndpointServerPools) -> io::Result<Vec<Arc<dyn TierMutationPeer>>> {
let (peers, _, remote_topology_hosts) = PeerRestClient::new_clients_with_topology(endpoints).await;
let peers = peers
.into_iter()
.flatten()
.map(|peer| Arc::new(peer) as Arc<dyn TierMutationPeer>)
.collect::<Vec<_>>();
ensure_complete_tier_mutation_commit_peer_set(peers.len(), remote_host_count)?;
ensure_complete_tier_mutation_commit_peer_set(peers.len(), remote_topology_hosts.len())?;
Ok(peers)
}
@@ -4307,6 +4311,34 @@ fn tier_config_not_initialized_error(operation: &str) -> std::io::Error {
#[cfg(test)]
mod tests {
use super::*;
use crate::layout::{
endpoint::Endpoint,
endpoints::{Endpoints, PoolEndpoints, SetupType},
};
struct SetupTypeGuard {
previous: SetupType,
}
impl SetupTypeGuard {
async fn switch_to(next: SetupType) -> Self {
let previous = runtime_sources::current_setup_type().await;
runtime_sources::set_setup_type(next).await;
Self { previous }
}
}
impl Drop for SetupTypeGuard {
fn drop(&mut self) {
let previous = self.previous.clone();
let handle = tokio::runtime::Handle::current();
tokio::task::block_in_place(|| {
handle.block_on(async move {
runtime_sources::set_setup_type(previous).await;
});
});
}
}
fn build_s3_tier(name: &str) -> TierConfig {
TierConfig {
@@ -6354,6 +6386,42 @@ mod tests {
assert!(err.to_string().contains("without peer commit clients"), "{err}");
}
#[tokio::test(flavor = "multi_thread")]
#[serial_test::serial]
async fn tier_mutation_peer_composition_preserves_unresolved_topology_slots() {
let mut endpoints = Vec::new();
for disk_index in 0..4 {
let mut endpoint = Endpoint::try_from(format!("http://rustfs-{disk_index}.invalid:9000/data{disk_index}").as_str())
.expect("unresolved topology endpoint should parse without DNS");
endpoint.is_local = disk_index == 0;
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
endpoints.push(endpoint);
}
let topology = EndpointServerPools::from(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: "unresolved-tier-mutation-topology".to_string(),
platform: "test".to_string(),
}]);
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let peers = remote_tier_mutation_peers_from_topology(topology)
.await
.expect("every unresolved remote topology slot should retain a tier mutation client");
assert_eq!(
peers.iter().map(|peer| peer.peer_label()).collect::<Vec<_>>(),
vec![
"http://rustfs-1.invalid:9000".to_string(),
"http://rustfs-2.invalid:9000".to_string(),
"http://rustfs-3.invalid:9000".to_string(),
]
);
}
#[tokio::test]
async fn coordinator_fanout_prepare_failure_aborts_prepared_peers_without_cas() {
let manager = TierConfigMgr::new();
crates/ecstore/src/set_disk/metadata.rs
+1 -3
View File
@@ -505,9 +505,7 @@ impl SetDisks {
}
fn file_info_has_encryption_metadata(meta: &FileInfo) -> bool {
meta.metadata
.keys()
.any(|name| http::is_encryption_metadata_key(name) || http::is_sse_header(name))
meta.metadata.keys().any(|name| http::is_object_encryption_marker(name))
}
fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool {
crates/ecstore/src/set_disk/mod.rs
+255 -26
View File
@@ -110,7 +110,10 @@ use crate::{
object_api::{GetObjectReader, ObjectInfo, PutObjReader},
// event::name::EventName,
services::event_notification::{EventArgs, send_event},
store::init_format::{get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all, save_format_file},
store::init_format::{
formats_match_reference_slots, get_format_erasure_in_quorum, load_format_erasure, load_format_erasure_all,
save_format_file,
},
};
use bytes::Bytes;
use bytesize::ByteSize;
@@ -144,15 +147,17 @@ use rustfs_object_capacity::capacity_scope::{
CapacityScope, CapacityScopeDisk, current_dirty_generation, record_capacity_scope, record_global_dirty_scope,
};
use rustfs_s3_types::EventName;
#[cfg(test)]
use rustfs_utils::http::SSEC_ALGORITHM_HEADER;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
use rustfs_utils::http::headers::{
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
};
use rustfs_utils::http::{
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE,
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, contains_key_str,
get_header_map, get_str, insert_str, is_encryption_metadata_key, remove_header_map,
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC,
SUFFIX_RESTORE_OPERATION_ID, contains_key_str, get_header_map, get_str, insert_str, is_object_encryption_marker,
remove_header_map,
};
use rustfs_utils::{
HashAlgorithm,
@@ -667,10 +672,7 @@ pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap<String, S
}
fn should_persist_encryption_original_size(metadata: &HashMap<String, String>) -> bool {
metadata.keys().any(|key| is_encryption_metadata_key(key))
|| metadata.contains_key(SSEC_ALGORITHM_HEADER)
|| metadata.contains_key(SSEC_KEY_HEADER)
|| metadata.contains_key(SSEC_KEY_MD5_HEADER)
metadata.keys().any(|key| is_object_encryption_marker(key))
}
/// Per-set memoized capacity dirty scope.
@@ -4712,6 +4714,7 @@ mod tests {
use crate::layout::endpoints::SetupType;
use crate::object_api::BLOCK_SIZE_V2;
use crate::object_api::ObjectInfo;
use crate::set_disk::core::io_primitives::rename_fanout_barrier;
use crate::storage_api_contracts::{
heal::HealOperations as _, lifecycle::TransitionedObject, list::ListOperations as _, multipart::CompletePart,
namespace::NamespaceLocking as _, object::ObjectIO as _, object::ObjectOperations as _,
@@ -5849,23 +5852,27 @@ mod tests {
crate::disk::DataDirDeleteStatus::Deleted
);
release_slow_candidate.notify_one();
for _ in 0..10 {
tokio::task::yield_now().await;
}
assert_eq!(
disk2
.delete_data_dir(
bucket,
data_dir,
DeleteOptions {
recursive: true,
..Default::default()
},
)
.await
.expect("a token acquired after the deadline must be released"),
crate::disk::DataDirDeleteStatus::Deleted
);
timeout(Duration::from_secs(1), async {
loop {
match disk2
.delete_data_dir(
bucket,
data_dir,
DeleteOptions {
recursive: true,
..Default::default()
},
)
.await
.expect("a token acquired after the deadline must be released")
{
crate::disk::DataDirDeleteStatus::Deleted => return,
crate::disk::DataDirDeleteStatus::Deferred => tokio::time::sleep(Duration::from_millis(1)).await,
}
}
})
.await
.expect("late snapshot lease cleanup must finish within the bounded wait");
}
#[tokio::test(start_paused = true)]
@@ -9559,6 +9566,15 @@ mod tests {
make_local_bucket_test_set_disks_with_drive_count(2).await
}
fn assert_exclusive_object_lock_held(set_disks: &SetDisks, bucket: &str, object: &str) {
let lock = set_disks
.local_lock_manager_for_test()
.get_lock_info(&ObjectKey::new(bucket, object))
.expect("object lock should be visible while rename is paused");
assert!(matches!(lock.mode, rustfs_lock::LockMode::Exclusive));
assert_eq!(lock.owner.as_ref(), set_disks.locker_owner.as_str());
}
async fn make_local_bucket_test_set_disks_with_drive_count(drive_count: usize) -> Arc<SetDisks> {
let format = FormatV3::new(1, drive_count);
let mut endpoints = Vec::new();
@@ -9593,7 +9609,9 @@ mod tests {
disks.push(Some(disk));
}
let set_disks = SetDisks::new(
let instance_ctx = Arc::new(InstanceContext::new());
instance_ctx.update_erasure_type(SetupType::Erasure).await;
let set_disks = SetDisks::new_with_instance_ctx(
"test-owner".to_string(),
Arc::new(RwLock::new(disks)),
drive_count,
@@ -9603,6 +9621,7 @@ mod tests {
endpoints,
format,
Vec::new(),
instance_ctx,
)
.await;
set_disks.set_test_storage_class_config(
@@ -10256,6 +10275,216 @@ mod tests {
));
}
#[tokio::test]
async fn conditional_replace_holds_object_lock_through_rename() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-conditional-replace-fence";
let object = "config/conditional-replace.json";
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut initial_reader = PutObjReader::from_vec(b"initial config".to_vec());
let initial = set_disks
.put_object(
bucket,
object,
&mut initial_reader,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("initial config should be written");
let initial_etag = initial.etag.expect("initial config should have an ETag");
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer_store = set_disks.clone();
let expected_etag = initial_etag.clone();
let writer = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(b"replacement config".to_vec());
writer_store
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
preserve_etag: Some("replacement-etag".to_string()),
http_preconditions: Some(HTTPPreconditions {
if_match: Some(expected_etag),
..Default::default()
}),
..Default::default()
},
)
.await
});
tokio::time::timeout(std::time::Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("conditional replace should reach the rename barrier");
assert_exclusive_object_lock_held(&set_disks, bucket, object);
barrier.release();
writer
.await
.expect("conditional writer task should finish")
.expect("matching conditional replace should commit");
assert!(
set_disks
.local_lock_manager_for_test()
.get_lock_info(&ObjectKey::new(bucket, object))
.is_none(),
"conditional replace should release the object lock after commit"
);
let contender = set_disks
.new_ns_lock(bucket, object)
.await
.expect("contender namespace lock should be created");
let contender_guard = contender
.get_write_lock(std::time::Duration::from_secs(30))
.await
.expect("contender should acquire after conditional replace commits");
drop(contender_guard);
let mut stale_reader = PutObjReader::from_vec(b"stale config".to_vec());
let err = set_disks
.put_object(
bucket,
object,
&mut stale_reader,
&ObjectOptions {
http_preconditions: Some(HTTPPreconditions {
if_match: Some(initial_etag),
..Default::default()
}),
..Default::default()
},
)
.await
.expect_err("the old ETag must fail after the fenced replacement commits");
assert_eq!(err, StorageError::PreconditionFailed);
}
#[tokio::test]
async fn repeated_body_write_keeps_etag_but_changes_data_dir_generation() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-write-generation";
let object = "config/write-generation.json";
let body = b"identical config body".to_vec();
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut first_reader = PutObjReader::from_vec(body.clone());
let first = set_disks
.put_object(
bucket,
object,
&mut first_reader,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("first config body should be written");
let mut second_reader = PutObjReader::from_vec(body);
let second = set_disks
.put_object(
bucket,
object,
&mut second_reader,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("identical config body should be rewritten");
assert_eq!(first.etag, second.etag, "content ETag should expose the ABA collision");
assert_ne!(first.data_dir, second.data_dir, "each committed body write needs a unique generation");
assert!(first.data_dir.is_some() && second.data_dir.is_some());
}
#[tokio::test]
async fn conditional_create_holds_object_lock_through_rename() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-conditional-create-fence";
let object = "config/conditional-create.json";
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer_store = set_disks.clone();
let writer = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(b"created config".to_vec());
writer_store
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
}),
..Default::default()
},
)
.await
});
tokio::time::timeout(std::time::Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("conditional create should reach the rename barrier");
assert_exclusive_object_lock_held(&set_disks, bucket, object);
barrier.release();
writer
.await
.expect("conditional writer task should finish")
.expect("first conditional create should commit");
assert!(
set_disks
.local_lock_manager_for_test()
.get_lock_info(&ObjectKey::new(bucket, object))
.is_none(),
"conditional create should release the object lock after commit"
);
let contender = set_disks
.new_ns_lock(bucket, object)
.await
.expect("contender namespace lock should be created");
let contender_guard = contender
.get_write_lock(std::time::Duration::from_secs(30))
.await
.expect("contender should acquire after conditional create commits");
drop(contender_guard);
let mut duplicate_reader = PutObjReader::from_vec(b"duplicate config".to_vec());
let err = set_disks
.put_object(
bucket,
object,
&mut duplicate_reader,
&ObjectOptions {
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
}),
..Default::default()
},
)
.await
.expect_err("a second create-only write must not replace the committed config");
assert_eq!(err, StorageError::PreconditionFailed);
}
#[tokio::test]
async fn set_level_if_none_match_fails_closed_without_read_quorum() {
let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
crates/ecstore/src/set_disk/ops/heal.rs
+63 -4
View File
@@ -1373,11 +1373,24 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
let disks = self.disks.read().await.clone();
let (formats, errs) = load_format_erasure_all(&disks, true).await;
let ref_format = match get_format_erasure_in_quorum(&formats) {
Ok(format) => format,
if errs.iter().any(|err| {
matches!(
err,
Some(DiskError::InconsistentDisk | DiskError::CorruptedFormat | DiskError::CorruptedBackend)
)
}) {
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
}
let slot_offset = self
.set_index
.checked_mul(self.set_drive_count)
.ok_or_else(|| Error::other("erasure set slot offset overflow"))?;
let ref_format = match get_format_erasure_in_quorum(&formats, slot_offset) {
Ok(format) if format.shared_identity() == self.format.shared_identity() => format,
Ok(_) => return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat))),
Err(err) => {
let can_use_cached_layout = count_errs(&errs, &DiskError::UnformattedDisk) > 0
&& formats.iter().flatten().all(|format| self.format.check_other(format).is_ok())
&& formats_match_reference_slots(&formats, &self.format, slot_offset)
&& errs
.iter()
.all(|err| err.is_none() || matches!(err, Some(DiskError::UnformattedDisk)));
@@ -1388,6 +1401,9 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
}
}
};
if !formats_match_reference_slots(&formats, &ref_format, slot_offset) {
return Ok((HealResultItem::default(), Some(StorageError::CorruptedFormat)));
}
let endpoints = crate::layout::endpoints::Endpoints::from(self.set_endpoints.clone());
let before_drives = crate::layout::set_heal::formats_to_drives_info(&endpoints, &formats, &errs);
@@ -1543,11 +1559,16 @@ mod heal_result_report_tests {
use crate::disk::error::DiskError;
use crate::disk::format::FormatV3;
use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, new_disk};
use crate::error::Error;
use crate::object_api::{ObjectOptions, PutObjReader};
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage_api_contracts::heal::HealOperations as _;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use crate::{config::storageclass, store::init_format::save_format_file};
use crate::{
config::storageclass,
store::init_format::{load_format_erasure, save_format_file},
};
use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode};
use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE};
use std::sync::Arc;
@@ -1863,6 +1884,44 @@ mod heal_result_report_tests {
}
}
#[tokio::test]
async fn format_heal_cached_layout_rejects_a_disk_from_another_slot() {
let mut _temp_dirs = Vec::new();
let mut endpoints = Vec::new();
let mut disks = Vec::new();
for disk_index in 0..3 {
let (temp_dir, mut endpoint, disk) = real_disk().await;
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
_temp_dirs.push(temp_dir);
endpoints.push(endpoint);
disks.push(Some(disk));
}
let set = set_disks_with(disks.clone(), endpoints, 1).await;
let mut wrong_slot = set.format.clone();
wrong_slot.erasure.this = set.format.erasure.sets[0][1];
save_format_file(&disks[0], &Some(wrong_slot))
.await
.expect("wrong-slot format fixture should be saved");
let mut correct_slot = set.format.clone();
correct_slot.erasure.this = set.format.erasure.sets[0][2];
save_format_file(&disks[2], &Some(correct_slot))
.await
.expect("correct format fixture should be saved");
let (_, heal_err) = set
.heal_format(false)
.await
.expect("format heal should report the quorum failure in its result");
assert!(matches!(heal_err, Some(Error::CorruptedFormat)));
let unformatted = load_format_erasure(disks[1].as_ref().expect("second disk should be online"), true)
.await
.expect_err("a rejected fallback must not format the missing slot");
assert_eq!(unformatted, DiskError::UnformattedDisk);
}
// Regression for #955: an offline disk must contribute exactly one drive
// record. Before the fix the offline branch fell through and pushed a second
// (Corrupt) record for the same disk, so `before/after.drives` grew to
crates/ecstore/src/set_disk/ops/locking.rs
+114 -9
View File
@@ -343,20 +343,23 @@ impl SetDisks {
}
};
// The drive's format may place it in a different erasure set than this
// one. Claiming a misplaced drive into `self.disks` would let two sets
// manage the same drive and degrade together, so reject it here
// (backlog#799 B19).
if set_idx != self.set_index {
// Claiming a misplaced drive into `self.disks` would let two slots or
// sets manage the same drive and degrade together (backlog#799 B19).
if set_idx != self.set_index || self.set_endpoints.get(disk_idx) != Some(ep) {
warn!(
"renew_disk: drive {:?} belongs to set {} but is being renewed on set {}; skipping",
ep, set_idx, self.set_index
endpoint = %ep,
format_set_index = set_idx,
format_disk_index = disk_idx,
endpoint_pool_index = ep.pool_idx,
endpoint_set_index = ep.set_idx,
endpoint_disk_index = ep.disk_idx,
expected_pool_index = self.pool_index,
expected_set_index = self.set_index,
"renew_disk rejected a drive whose endpoint and format do not identify the same topology slot"
);
return;
}
// Check that the endpoint matches
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
new_disk.enable_health_check();
@@ -715,6 +718,108 @@ mod tests {
drop(temp_dirs);
}
#[tokio::test]
async fn renew_disk_rejects_a_format_from_another_slot_or_cluster() {
let disk_count = 3;
let format = FormatV3::new(1, disk_count);
let mut temp_dirs = Vec::with_capacity(disk_count);
let mut endpoints = Vec::with_capacity(disk_count);
let mut fixture_disks = Vec::with_capacity(disk_count);
for disk_idx in 0..disk_count {
let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
temp_dirs.push(temp_dir);
endpoints.push(endpoint);
fixture_disks.push(disk);
}
let set_disks = SetDisks::new(
"test-owner".to_string(),
Arc::new(RwLock::new(vec![Some(fixture_disks[0].clone()), None, None])),
disk_count,
disk_count / 2,
0,
0,
endpoints.clone(),
format.clone(),
Vec::new(),
)
.await;
let mut other_cluster_format = format.clone();
other_cluster_format.id = Uuid::new_v4();
other_cluster_format.erasure.this = format.erasure.sets[0][2];
save_format_file(&Some(fixture_disks[2].clone()), &Some(other_cluster_format))
.await
.expect("other-cluster format should be written for the rejection test");
set_disks.renew_disk(&endpoints[2]).await;
let disks = set_disks.get_disks_internal().await;
assert_eq!(
disks[0]
.as_ref()
.expect("the canonical first slot must remain attached")
.endpoint(),
endpoints[0]
);
assert!(
disks[2].is_none(),
"a disk from another deployment must remain detached even when its slot UUID matches"
);
let mut correct_format = format.clone();
correct_format.erasure.this = format.erasure.sets[0][2];
let replacement_disk = new_disk(
&endpoints[2],
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("third endpoint should reopen after other-cluster rejection");
save_format_file(&Some(replacement_disk), &Some(correct_format))
.await
.expect("correct slot format should be restored");
set_disks.renew_disk(&endpoints[2]).await;
let disks = set_disks.get_disks_internal().await;
assert_eq!(
disks[0]
.as_ref()
.expect("the canonical first slot must remain attached")
.endpoint(),
endpoints[0]
);
assert_eq!(disks[2].as_ref().expect("the restored third slot should attach").endpoint(), endpoints[2]);
let third_disk = disks[2].clone();
let mut wrong_slot_format = format.clone();
wrong_slot_format.erasure.this = format.erasure.sets[0][0];
save_format_file(&third_disk, &Some(wrong_slot_format))
.await
.expect("wrong-slot format should be written for the rejection test");
set_disks.disks.write().await[2] = None;
let mut misplaced_endpoint = endpoints[2].clone();
misplaced_endpoint.set_disk_index(0);
set_disks.renew_disk(&misplaced_endpoint).await;
let disks = set_disks.get_disks_internal().await;
assert_eq!(
disks[0]
.as_ref()
.expect("the canonical first slot must remain attached")
.endpoint(),
endpoints[0]
);
assert!(disks[2].is_none(), "a disk claiming another endpoint's slot must remain detached");
drop(temp_dirs);
}
// SetDisks split P0 (#816): the borrow handle must mirror the core state and
// the List operation family must run identically through it.
#[tokio::test]
crates/ecstore/src/set_disk/ops/object.rs
+71 -2
View File
@@ -42,6 +42,7 @@ use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppress
use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
use crate::store::ECStore;
use futures::FutureExt as _;
use http::HeaderValue;
use std::future::Future;
fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Erasure> {
@@ -49,6 +50,17 @@ fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Er
.map_err(Error::from)
}
async fn get_object_reader_with_context(
ctx: &InstanceContext,
reader: Box<dyn AsyncRead + Unpin + Send + Sync>,
range: Option<HTTPRangeSpec>,
object_info: &ObjectInfo,
opts: &ObjectOptions,
headers: &HeaderMap<HeaderValue>,
) -> Result<(GetObjectReader, usize, i64)> {
GetObjectReader::new_with_resolver(reader, range, object_info, opts, headers, ctx.object_encryption_resolver()).await
}
/// Length of the full plaintext body when — and only when — this read's output
/// is exactly the object's complete plaintext, so the app-layer body cache may
/// serve it in place of the erasure read.
@@ -713,7 +725,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
size_bucket,
);
record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket);
let (mut reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?;
let (mut reader, _offset, _length) =
get_object_reader_with_context(&self.ctx, stream, range, &object_info, opts, &h).await?;
// Carry the hook probe result so the app layer skips its
// now-redundant lookup on the streaming miss path (ODC-16).
reader.body_source = body_source;
@@ -745,7 +758,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
let (rd, wd) = tokio::io::duplex(duplex_buffer_size);
debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer");
let (mut reader, offset, length) = GetObjectReader::new(Box::new(rd), range, &object_info, opts, &h).await?;
let (mut reader, offset, length) =
get_object_reader_with_context(&self.ctx, Box::new(rd), range, &object_info, opts, &h).await?;
// Carry the hook probe result so the app layer skips its now-redundant
// lookup on the streaming miss path (ODC-16).
reader.body_source = body_source;
@@ -4536,6 +4550,61 @@ mod erasure_construction_tests {
}
}
#[cfg(test)]
mod object_encryption_resolver_wiring_tests {
use super::*;
use crate::object_api::{EncryptionResolutionError, ObjectEncryptionResolver, ReadEncryptionMaterial, ReadEncryptionRequest};
use std::io::Cursor;
use std::sync::atomic::{AtomicUsize, Ordering};
struct CountingResolver {
calls: AtomicUsize,
}
#[async_trait::async_trait]
impl ObjectEncryptionResolver for CountingResolver {
async fn resolve_read_material(
&self,
_request: ReadEncryptionRequest<'_>,
) -> std::result::Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
self.calls.fetch_add(1, Ordering::Relaxed);
Ok(None)
}
}
#[tokio::test]
async fn get_object_reader_forwards_instance_resolver() {
let resolver = Arc::new(CountingResolver {
calls: AtomicUsize::new(0),
});
let ctx = InstanceContext::new();
assert!(
ctx.set_object_encryption_resolver(resolver.clone()).is_ok(),
"fresh context should accept resolver"
);
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: 1,
user_defined: Arc::new(HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())])),
..Default::default()
};
let result = get_object_reader_with_context(
&ctx,
Box::new(Cursor::new(Vec::<u8>::new())),
None,
&object_info,
&ObjectOptions::default(),
&HeaderMap::new(),
)
.await;
assert!(result.is_err(), "resolver returning no material must fail closed");
assert_eq!(resolver.calls.load(Ordering::Relaxed), 1);
}
}
#[cfg(test)]
pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
//! Shared hermetic `SetDisks` construction for the ops tests below: the
crates/ecstore/src/set_disk/read.rs
+7 -7
View File
@@ -199,7 +199,7 @@ impl SetDisks {
);
}
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub async fn read_version_optimized(
&self,
bucket: &str,
@@ -238,7 +238,7 @@ impl SetDisks {
}
#[tracing::instrument(level = "debug", skip(self))]
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub(super) async fn get_object_fileinfo(
&self,
bucket: &str,
@@ -410,7 +410,7 @@ impl SetDisks {
Ok((fi, parts_metadata, op_online_disks))
}
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub(super) async fn get_object_info_and_quorum(
&self,
bucket: &str,
@@ -605,7 +605,7 @@ impl SetDisks {
}
#[allow(clippy::too_many_arguments)]
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub(super) async fn get_object_with_fileinfo<W>(
// &self,
bucket: &str,
@@ -1140,7 +1140,7 @@ impl SetDisks {
}
#[allow(clippy::too_many_arguments)]
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub(super) async fn get_object_decode_reader_with_fileinfo(
bucket: &str,
object: &str,
@@ -1296,7 +1296,7 @@ impl SetDisks {
}
#[allow(clippy::too_many_arguments)]
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
async fn build_codec_streaming_part_reader(
bucket: &str,
object: &str,
@@ -1469,7 +1469,7 @@ fn multipart_part_checksum_algo(fi: &FileInfo, part_number: usize) -> HashAlgori
/// `get_object_with_fileinfo` (backlog#870) so both report the same
/// stage-duration semantics.
#[allow(clippy::too_many_arguments)]
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
async fn setup_multipart_part_readers(
files: &[FileInfo],
disks: &[Option<DiskStore>],
crates/ecstore/src/store/bucket.rs
+4 -3
View File
@@ -310,12 +310,13 @@ impl ECStore {
meta.set_created(opts.created_at);
if opts.lock_enabled {
meta.object_lock_config_xml = crate::bucket::utils::serialize::<ObjectLockConfiguration>(&enableObjcetLockConfig)?;
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
meta.object_lock_config_xml =
crate::bucket::utils::serialize::<ObjectLockConfiguration>(&ENABLED_OBJECT_LOCK_CONFIG)?;
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
}
if opts.versioning_enabled {
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&enableVersioningConfig)?;
meta.versioning_config_xml = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)?;
}
await_bucket_namespace_operation(
crates/ecstore/src/store/heal.rs
+137 -2
View File
@@ -30,6 +30,7 @@ impl ECStore {
};
let mut count_no_heal = 0;
let mut first_error = None;
for pool in self.pools.iter() {
let (mut result, err) = pool.heal_format(dry_run).await?;
if let Some(err) = err {
@@ -37,8 +38,8 @@ impl ECStore {
StorageError::NoHealRequired => {
count_no_heal += 1;
}
_ => {
continue;
err => {
first_error.get_or_insert(err);
}
}
}
@@ -47,6 +48,9 @@ impl ECStore {
r.before.drives.append(&mut result.before.drives);
r.after.drives.append(&mut result.after.drives);
}
if let Some(err) = first_error {
return Ok((r, Some(err)));
}
if count_no_heal == self.pools.len() {
info!(
event = EVENT_HEAL_FORMAT_COMPLETED,
@@ -165,3 +169,134 @@ impl ECStore {
Err(StorageError::NotImplemented)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::disk::{DiskOption, format::FormatV3, new_disk};
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
use crate::store::init_format::{load_format_erasure, save_format_file};
#[tokio::test]
async fn handle_heal_format_continues_after_a_pool_error() {
let canonical_format = FormatV3::new(1, 3);
let mut foreign_format = canonical_format.clone();
foreign_format.id = Uuid::new_v4();
let mut temp_dirs = Vec::new();
let mut endpoints = Vec::new();
let mut disks = Vec::new();
for disk_index in 0..3 {
let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
.expect("temporary endpoint should parse");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("temporary disk should open");
let mut disk_format = foreign_format.clone();
disk_format.erasure.this = foreign_format.erasure.sets[0][disk_index];
save_format_file(&Some(disk.clone()), &Some(disk_format))
.await
.expect("foreign format should be written");
temp_dirs.push(temp_dir);
endpoints.push(endpoint);
disks.push(Some(disk));
}
let pool_endpoints = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 3,
endpoints: Endpoints::from(endpoints),
cmd_line: "foreign-format-majority-test".to_string(),
platform: "test".to_string(),
};
let pool = Sets::new(disks, &pool_endpoints, &canonical_format, 0, 1)
.await
.expect("test pool should build around the cached canonical format");
let mut recoverable_format = FormatV3::new(1, 3);
recoverable_format.id = canonical_format.id;
let mut recoverable_temp_dirs = Vec::new();
let mut recoverable_endpoints = Vec::new();
let mut recoverable_disks = Vec::new();
let mut unformatted_disk = None;
for disk_index in 0..3 {
let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
.expect("temporary endpoint should parse");
endpoint.set_pool_index(1);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("temporary disk should open");
if disk_index < 2 {
let mut disk_format = recoverable_format.clone();
disk_format.erasure.this = recoverable_format.erasure.sets[0][disk_index];
save_format_file(&Some(disk.clone()), &Some(disk_format))
.await
.expect("recoverable format should be written");
} else {
unformatted_disk = Some(disk.clone());
}
recoverable_temp_dirs.push(temp_dir);
recoverable_endpoints.push(endpoint);
recoverable_disks.push(Some(disk));
}
let recoverable_pool_endpoints = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 3,
endpoints: Endpoints::from(recoverable_endpoints),
cmd_line: "recoverable-format-test".to_string(),
platform: "test".to_string(),
};
let recoverable_pool = Sets::new(recoverable_disks, &recoverable_pool_endpoints, &recoverable_format, 1, 1)
.await
.expect("recoverable test pool should build");
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone(), recoverable_pool_endpoints.clone()]);
let store = ECStore {
id: canonical_format.id,
disk_map: HashMap::new(),
pools: vec![pool, recoverable_pool],
peer_sys: S3PeerSys::new(&endpoint_pools),
pool_meta: RwLock::new(PoolMeta::default()),
rebalance_meta: RwLock::new(None),
decommission_cancelers: RwLock::new(Vec::new()),
start_gate: Mutex::new(()),
pool_meta_save_gate: Mutex::new(()),
ctx: crate::runtime::instance::bootstrap_ctx(),
};
let (result, err) = store
.handle_heal_format(false)
.await
.expect("format heal should return the typed pool error");
assert!(
matches!(err, Some(StorageError::CorruptedFormat)),
"foreign format majority must not be downgraded to a successful heal: {err:?}"
);
assert_eq!(result.disk_count, 3, "the recoverable pool should still be inspected");
let healed = load_format_erasure(&unformatted_disk.expect("the unformatted disk handle should be retained"), true)
.await
.expect("the later pool should be healed despite the first pool error");
assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
}
}
crates/ecstore/src/store/init.rs
+36 -4
View File
@@ -101,6 +101,10 @@ fn should_retry_local_decommission_resume(err: &Error, attempt: usize) -> bool {
matches!(err, Error::ConfigNotFound) && attempt < LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES
}
fn should_retry_format_load(err: &Error) -> bool {
!matches!(err, Error::CorruptedFormat)
}
fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_meta_loaded: bool) -> bool {
rebalance_meta_loaded && !decommission_running
}
@@ -294,7 +298,7 @@ impl ECStore {
// periodic monitoring until format loading succeeds. Startup RPC
// failures can still spawn recovery probes for peers that come up
// after this node.
let (disks, errs) = init_format::init_disks(
let (mut disks, errs) = init_format::init_disks(
&pool_eps.endpoints,
&DiskOption {
cleanup: true,
@@ -309,9 +313,10 @@ impl ECStore {
let mut times = 0;
let mut interval = 1;
loop {
match init_format::connect_load_init_formats(
match init_format::connect_load_init_formats_with_instance_ctx(
&instance_ctx,
pool_first_is_local,
&disks,
&mut disks,
pool_eps.set_count,
pool_eps.drives_per_set,
deployment_id,
@@ -319,6 +324,7 @@ impl ECStore {
.await
{
Ok(fm) => break Ok(fm),
Err(e) if !should_retry_format_load(&e) => break Err(e),
// Wrap the final error if we are giving up
Err(e) if times >= 10 => {
break Err(Error::other(format!("store init failed to load formats after {times} retries: {e}")));
@@ -551,7 +557,7 @@ mod tests {
LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES, load_pool_meta_for_startup, pool_first_endpoint_is_local,
pool_meta_has_active_decommission, preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with,
resolve_store_init_stage_result, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay,
should_retry_format_load, should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay,
};
#[cfg(feature = "test-util")]
use crate::{
@@ -773,6 +779,13 @@ mod tests {
assert!(!should_retry_local_decommission_resume(&StorageError::SlowDown, 0));
}
#[test]
fn test_should_retry_format_load_rejects_permanent_corruption() {
assert!(!should_retry_format_load(&StorageError::CorruptedFormat));
assert!(should_retry_format_load(&StorageError::ErasureReadQuorum));
assert!(should_retry_format_load(&StorageError::FirstDiskWait));
}
#[test]
fn test_should_auto_start_rebalance_after_init_allows_loaded_rebalance_without_decommission() {
assert!(should_auto_start_rebalance_after_init(false, true));
@@ -1247,6 +1260,16 @@ mod tests {
let registered: Vec<String> = instance_ctx.local_disk_map().read().await.keys().cloned().collect();
assert_eq!(registered.len(), 4, "the passed context must register all four local disks");
let registered_disk_ids = instance_ctx.local_disk_id_map();
let registered_disk_ids = registered_disk_ids.read().await;
assert_eq!(registered_disk_ids.len(), 4, "the passed context must publish all four disk IDs");
for endpoint in registered_disk_ids.values() {
assert!(
registered.contains(endpoint),
"every disk ID in the passed context must resolve to one of its registered endpoints"
);
}
drop(registered_disk_ids);
let bootstrap = crate::runtime::instance::bootstrap_ctx();
assert_ne!(
bootstrap.deployment_id(),
@@ -1261,6 +1284,15 @@ mod tests {
"the bootstrap context must not absorb the fresh store's disks"
);
}
drop(bootstrap_map);
let bootstrap_disk_ids = bootstrap.local_disk_id_map();
let bootstrap_disk_ids = bootstrap_disk_ids.read().await;
for endpoint in bootstrap_disk_ids.values() {
assert!(
!registered.contains(endpoint),
"the bootstrap context must not absorb the fresh store's disk IDs"
);
}
}
#[tokio::test]
crates/ecstore/src/store/init_format.rs
+1254 -148
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/store/mod.rs
+4 -4
View File
@@ -428,11 +428,11 @@ impl ECStore {
}
lazy_static! {
static ref enableObjcetLockConfig: ObjectLockConfiguration = ObjectLockConfiguration {
static ref ENABLED_OBJECT_LOCK_CONFIG: ObjectLockConfiguration = ObjectLockConfiguration {
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
..Default::default()
};
static ref enableVersioningConfig: VersioningConfiguration = VersioningConfiguration {
static ref ENABLED_VERSIONING_CONFIG: VersioningConfiguration = VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
};
@@ -989,7 +989,7 @@ mod tests {
init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
let (disks, errs) = init_disks(
let (mut disks, errs) = init_disks(
&endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
&DiskOption {
cleanup: true,
@@ -999,7 +999,7 @@ mod tests {
.await;
assert!(errs.iter().all(|err| err.is_none()), "disk init should succeed: {errs:?}");
connect_load_init_formats(true, &disks, 1, 4, None)
connect_load_init_formats(true, &mut disks, 1, 4, None)
.await
.expect("initialize format metadata");
crates/ecstore/src/store/multipart.rs
+1 -1
View File
@@ -238,7 +238,7 @@ impl ECStore {
}
#[instrument(skip(self, data))]
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub(super) async fn handle_put_object_part(
&self,
bucket: &str,
crates/ecstore/src/store/object.rs
+2 -2
View File
@@ -815,7 +815,7 @@ impl ECStore {
}
#[instrument(level = "debug", skip(self))]
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub(super) async fn handle_get_object_reader(
&self,
bucket: &str,
@@ -849,7 +849,7 @@ impl ECStore {
}
#[instrument(level = "debug", skip(self, data))]
#[cfg_attr(feature = "hotpath", hotpath::measure)]
#[hotpath::measure]
pub(super) async fn handle_put_object(
&self,
bucket: &str,
crates/ecstore/src/store/peer.rs
+73 -2
View File
@@ -28,8 +28,26 @@ async fn remember_local_disk_id(disk: &DiskStore) -> Option<Uuid> {
async fn remember_local_disk_id_with_instance_ctx(instance_ctx: &Arc<InstanceContext>, disk: &DiskStore) -> Option<Uuid> {
let disk_id = disk.get_disk_id().await.ok().flatten()?;
runtime_sources::record_local_disk_id(instance_ctx, disk_id, disk.endpoint().to_string()).await;
Some(disk_id)
record_local_disk_id_if_active(instance_ctx, disk, disk_id)
.await
.then_some(disk_id)
}
async fn record_local_disk_id_if_active(instance_ctx: &Arc<InstanceContext>, disk: &DiskStore, disk_id: Uuid) -> bool {
let endpoint = disk.endpoint().to_string();
let local_disk_map = instance_ctx.local_disk_map();
let local_disks = local_disk_map.read().await;
let Some(active_disk) = local_disks.get(&endpoint).and_then(Option::as_ref) else {
return false;
};
if !Arc::ptr_eq(active_disk, disk) {
return false;
}
// Lock order is local_disk_map -> local_disk_id_map so quarantine is the
// linearization point for rejecting an in-flight stale disk snapshot.
instance_ctx.local_disk_id_map().write().await.insert(disk_id, endpoint);
true
}
pub async fn find_local_disk(disk_path: &str) -> Option<DiskStore> {
@@ -228,6 +246,7 @@ pub async fn get_disk_infos(disks: &[Option<DiskStore>]) -> Vec<Option<DiskInfo>
#[cfg(test)]
mod tests {
use super::*;
use crate::disk::new_disk;
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
fn single_local_disk_pools(dir: &std::path::Path) -> EndpointServerPools {
@@ -314,4 +333,56 @@ mod tests {
);
}
}
#[tokio::test]
async fn stale_local_disk_snapshot_cannot_repopulate_the_id_registry() {
let temp_dir = tempfile::tempdir().expect("create temp disk dir");
let endpoint_pools = single_local_disk_pools(temp_dir.path());
let instance_ctx = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools)
.await
.expect("local disk should be registered");
let disk = instance_ctx
.local_disk_map()
.read()
.await
.values()
.find_map(|disk| disk.clone())
.expect("registered local disk");
let endpoint = disk.endpoint().to_string();
let disk_id = Uuid::new_v4();
let local_disk_map = instance_ctx.local_disk_map();
let mut quarantine = local_disk_map.write().await;
let replacement = new_disk(
&disk.endpoint(),
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("replacement disk should initialize");
assert!(!Arc::ptr_eq(&disk, &replacement));
let task_ctx = instance_ctx.clone();
let task_disk = disk.clone();
let remember = tokio::spawn(async move { record_local_disk_id_if_active(&task_ctx, &task_disk, disk_id).await });
tokio::task::yield_now().await;
quarantine.insert(endpoint.clone(), Some(replacement.clone()));
drop(quarantine);
assert!(!remember.await.expect("stale lookup task should complete"));
assert!(!instance_ctx.local_disk_id_map().read().await.contains_key(&disk_id));
let active = instance_ctx
.local_disk_map()
.read()
.await
.get(&endpoint)
.cloned()
.flatten()
.expect("replacement disk should remain registered");
assert!(Arc::ptr_eq(&active, &replacement));
assert!(record_local_disk_id_if_active(&instance_ctx, &replacement, disk_id).await);
assert_eq!(instance_ctx.local_disk_id_map().read().await.get(&disk_id), Some(&endpoint));
}
}
crates/ecstore/tests/README.md
+2 -2
View File
@@ -2,7 +2,7 @@
## MinIO-generated encrypted fixtures
`minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
`rustfs/src/storage/minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
`.\rustfs\scripts\minio_fixture_lab\lab.py`.
It currently covers multipart fixtures for:
@@ -20,5 +20,5 @@ Example:
```powershell
$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key'
$env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '<base64-32-byte-local-minio-kms-key>'
cargo +1.97.1 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
cargo +1.97.1 test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored
```
crates/ecstore/tests/minio_generated_read_test.rs
-299
View File
@@ -1,299 +0,0 @@
#![cfg(feature = "rio-v2")]
use std::fs;
use std::io::Cursor;
use std::path::{Path, PathBuf};
mod storage_api;
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use storage_api::minio_generated_read::{
DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
};
use temp_env::async_with_vars;
use tokio::io::{AsyncReadExt, AsyncWrite};
#[derive(Debug, Deserialize)]
struct ManifestRecord {
bucket: String,
object: String,
backend_files: Vec<String>,
}
#[derive(Default)]
struct VecAsyncWriter {
bytes: Vec<u8>,
}
impl AsyncWrite for VecAsyncWriter {
fn poll_write(
mut self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
buf: &[u8],
) -> std::task::Poll<std::io::Result<usize>> {
self.bytes.extend_from_slice(buf);
std::task::Poll::Ready(Ok(buf.len()))
}
fn poll_flush(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Ready(Ok(()))
}
fn poll_shutdown(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Ready(Ok(()))
}
}
fn fixture_root() -> PathBuf {
std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/minio-generated"))
}
fn case_dir(case_id: &str) -> PathBuf {
fixture_root().join("cases").join(case_id)
}
fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> T {
let text = fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()));
serde_json::from_str(&text).unwrap_or_else(|err| panic!("parse {}: {err}", path.display()))
}
fn require_fixture_case(case_id: &str) -> PathBuf {
let path = case_dir(case_id);
assert!(
path.is_dir(),
"fixture case missing: {}. Run scripts/minio_fixture_lab/lab.py capture-matrix first.",
path.display()
);
path
}
fn read_plaintext_sha256(case_dir: &Path) -> String {
fs::read_to_string(case_dir.join("plaintext.sha256"))
.unwrap_or_else(|err| panic!("read plaintext.sha256 under {}: {err}", case_dir.display()))
.trim()
.to_string()
}
fn minio_static_kms_key_b64() -> String {
std::env::var("RUSTFS_MINIO_STATIC_KMS_KEY_B64")
.unwrap_or_else(|_| panic!("RUSTFS_MINIO_STATIC_KMS_KEY_B64 must point to the 32-byte static MinIO KMS key"))
}
fn object_xl_meta_path(case_dir: &Path, manifest: &ManifestRecord) -> PathBuf {
let expected = format!("disk1/{}/{}/xl.meta", manifest.bucket, manifest.object);
let relative = manifest
.backend_files
.iter()
.find(|entry| entry.as_str() == expected)
.unwrap_or_else(|| panic!("object xl.meta missing from manifest backend_files: {expected}"));
case_dir.join("backend").join(relative)
}
fn load_file_info(case_dir: &Path, manifest: &ManifestRecord) -> FileInfo {
let xl_meta_path = object_xl_meta_path(case_dir, manifest);
let xl_meta = fs::read(&xl_meta_path).unwrap_or_else(|err| panic!("read {}: {err}", xl_meta_path.display()));
get_file_info(
&xl_meta,
&manifest.bucket,
&manifest.object,
"",
FileInfoOpts {
data: true,
include_free_versions: true,
},
)
.unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta_path.display()))
}
fn load_object_info(file_info: &FileInfo, manifest: &ManifestRecord) -> ObjectInfo {
ObjectInfo::from_file_info(file_info, &manifest.bucket, &manifest.object, false)
}
fn sha256_hex(bytes: &[u8]) -> String {
hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
}
async fn load_fixture_reader_input(case_id: &str) -> (ObjectInfo, Vec<u8>, String) {
let case_dir = require_fixture_case(case_id);
let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json"));
let expected_sha256 = read_plaintext_sha256(&case_dir);
let file_info = load_file_info(&case_dir, &manifest);
let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await;
let object_info = load_object_info(&file_info, &manifest);
(object_info, encrypted, expected_sha256)
}
async fn read_fixture_plaintext(encrypted: Vec<u8>, object_info: ObjectInfo, kms_key_b64: String) -> Result<Vec<u8>, String> {
let object_size = object_info.size;
async_with_vars(
[
("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)),
("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
],
async move {
let (mut reader, offset, length) = GetObjectReader::new(
Box::new(Cursor::new(encrypted)),
None,
&object_info,
&ObjectOptions::default(),
&http::HeaderMap::new(),
)
.await
.map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?;
if offset != 0 || length != object_size {
return Err(format!("unexpected fixture range offset={offset} length={length} size={object_size}"));
}
let mut plaintext = Vec::new();
reader
.read_to_end(&mut plaintext)
.await
.map_err(|err| format!("read plaintext from MinIO raw fixture: {err}"))?;
Ok(plaintext)
},
)
.await
}
async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, file_info: &FileInfo) -> Vec<u8> {
let mut disks = Vec::with_capacity(file_info.erasure.distribution.len());
for disk_number in 1..=file_info.erasure.distribution.len() {
let disk_root = case_dir.join("backend").join(format!("disk{disk_number}"));
let disk_root_str = disk_root
.to_str()
.unwrap_or_else(|| panic!("non-utf8 disk root {}", disk_root.display()));
let mut endpoint = Endpoint::try_from(disk_root_str).expect("fixture disk endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_number - 1);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.unwrap_or_else(|err| panic!("open fixture disk {disk_number}: {err}"));
disks.push(disk);
}
let mut disk_order = vec![None; disks.len()];
for (idx, disk) in disks.iter().enumerate() {
let block_index = file_info.erasure.distribution[idx];
disk_order[block_index - 1] = Some(disk);
}
let data_dir = file_info
.data_dir
.as_ref()
.unwrap_or_else(|| panic!("fixture {} is missing data_dir", manifest.object));
let mut encrypted = Vec::new();
for part in &file_info.parts {
let checksum_info = file_info.erasure.get_checksum_info(part.number);
let path = format!("{}/{}/part.{}", manifest.object, data_dir, part.number);
let shard_read_len = file_info.erasure.shard_file_size(part.size as i64);
let mut readers = Vec::with_capacity(disks.len());
for (idx, disk) in disk_order.iter().enumerate() {
let reader = create_bitrot_reader(
None,
*disk,
&manifest.bucket,
&path,
0,
shard_read_len as usize,
file_info.erasure.shard_size(),
checksum_info.algorithm.clone(),
false,
false,
)
.await
.unwrap_or_else(|err| panic!("create bitrot reader for disk{} {path}: {err:?}", idx + 1));
readers.push(reader);
}
let erasure = Erasure::new(
file_info.erasure.data_blocks,
file_info.erasure.parity_blocks,
file_info.erasure.block_size,
);
let mut writer = VecAsyncWriter::default();
let (written, err) = erasure.decode(&mut writer, readers, 0, part.size, part.size).await;
if let Some(err) = err {
panic!("decode erasure shards for {path}: {err}");
}
assert_eq!(written, part.size, "decoded part size should match xl.meta part size");
encrypted.extend_from_slice(&writer.bytes);
}
for disk in disks {
disk.close().await.expect("close fixture disk");
}
encrypted
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_s3_multipart_fixture() {
assert_fixture_round_trip("sse-s3-multipart-8m", 8 * 1024 * 1024).await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_kms_multipart_fixture() {
assert_fixture_round_trip("sse-kms-multipart-8m", 8 * 1024 * 1024).await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() {
let (object_info, encrypted, _) = load_fixture_reader_input("sse-s3-multipart-8m").await;
let wrong_key_b64 = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=".to_string();
let result = read_fixture_plaintext(encrypted, object_info, wrong_key_b64).await;
assert!(result.is_err(), "wrong KMS key must fail closed");
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext() {
let (object_info, mut encrypted, expected_sha256) = load_fixture_reader_input("sse-s3-multipart-8m").await;
encrypted.truncate(encrypted.len() / 2);
let result = read_fixture_plaintext(encrypted, object_info, minio_static_kms_key_b64()).await;
if let Ok(plaintext) = result {
assert_ne!(
sha256_hex(&plaintext),
expected_sha256,
"truncated ciphertext must not restore the original plaintext"
);
}
}
async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
let (object_info, encrypted, expected_sha256) = load_fixture_reader_input(case_id).await;
// `ObjectInfo.size` is the on-disk size. For SSE objects that is the
// DARE-encrypted size (plaintext + 32 bytes per 64 KiB block), which is
// deliberately larger than the logical object size. The size a client sees
// (and what MinIO records via `x-*-internal-actual-size`) comes from
// `decrypted_size()`/`get_actual_size()`, so assert against that — the raw
// `size` field would never equal the plaintext length for encrypted objects.
let decrypted_size = object_info.decrypted_size().expect("decrypted size from MinIO metadata");
let kms_key_b64 = minio_static_kms_key_b64();
let plaintext = read_fixture_plaintext(encrypted, object_info, kms_key_b64)
.await
.expect("fixture must restore with the configured KMS key");
assert_eq!(decrypted_size, expected_size);
assert_eq!(plaintext.len(), expected_size as usize);
assert_eq!(sha256_hex(&plaintext), expected_sha256);
}
crates/extension-schema/Cargo.toml
+7
View File
@@ -27,7 +27,14 @@ categories = ["web-programming", "development-tools"]
[lib]
doctest = false
[features]
default = []
hotpath = ["hotpath/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
[dependencies]
hotpath.workspace = true
serde = { workspace = true, features = ["derive"] }
thiserror.workspace = true
crates/filemeta/Cargo.toml
+4 -2
View File
@@ -27,10 +27,12 @@ documentation = "https://docs.rs/rustfs-filemeta/latest/rustfs_filemeta/"
[features]
default = []
hotpath = ["dep:hotpath", "hotpath/hotpath"]
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-utils/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-utils/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-utils/hotpath-cpu"]
[dependencies]
hotpath = { workspace = true, optional = true }
hotpath.workspace = true
crc-fast = { workspace = true }
rmp.workspace = true
rmp-serde.workspace = true
crates/filemeta/src/filemeta/codec.rs
+3 -3
View File
@@ -19,7 +19,7 @@ impl FileMeta {
!matches!(Self::check_xl2_v1(buf), Err(_e))
}
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "FileMeta"))]
#[hotpath::measure(impl_type = "FileMeta")]
pub fn load(buf: &[u8]) -> Result<FileMeta> {
let mut xl = FileMeta::default();
xl.unmarshal_msg(buf)?;
@@ -112,7 +112,7 @@ impl FileMeta {
Ok((bin_len, remaining))
}
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "FileMeta"))]
#[hotpath::measure(impl_type = "FileMeta")]
pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result<u64> {
let i = buf.len() as u64;
@@ -326,7 +326,7 @@ impl FileMeta {
}
}
#[cfg_attr(feature = "hotpath", hotpath::measure(impl_type = "FileMeta"))]
#[hotpath::measure(impl_type = "FileMeta")]
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut wr = Vec::new();
crates/heal/Cargo.toml
+41
View File
@@ -29,7 +29,48 @@ categories = ["web-programming", "development-tools", "filesystem"]
[lints]
workspace = true
[features]
default = []
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"hotpath/futures",
"rustfs-common/hotpath",
"rustfs-concurrency/hotpath",
"rustfs-config/hotpath",
"rustfs-ecstore/hotpath",
"rustfs-madmin/hotpath",
"rustfs-storage-api/hotpath",
"rustfs-utils/hotpath",
"rustfs-test-utils/hotpath",
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-common/hotpath-alloc",
"rustfs-concurrency/hotpath-alloc",
"rustfs-config/hotpath-alloc",
"rustfs-ecstore/hotpath-alloc",
"rustfs-madmin/hotpath-alloc",
"rustfs-storage-api/hotpath-alloc",
"rustfs-utils/hotpath-alloc",
"rustfs-test-utils/hotpath-alloc",
]
hotpath-cpu = [
"hotpath",
"hotpath/hotpath-cpu",
"rustfs-common/hotpath-cpu",
"rustfs-concurrency/hotpath-cpu",
"rustfs-config/hotpath-cpu",
"rustfs-ecstore/hotpath-cpu",
"rustfs-madmin/hotpath-cpu",
"rustfs-storage-api/hotpath-cpu",
"rustfs-utils/hotpath-cpu",
"rustfs-test-utils/hotpath-cpu",
]
[dependencies]
hotpath.workspace = true
rustfs-config = { workspace = true }
rustfs-concurrency = { workspace = true }
rustfs-ecstore = { workspace = true }
crates/heal/src/heal/erasure_healer.rs
+1
View File
@@ -159,6 +159,7 @@ impl ErasureSetHealer {
/// execute erasure set heal with resume
#[tracing::instrument(skip(self, buckets), fields(set_disk_id = %set_disk_id, bucket_count = buckets.len()))]
#[hotpath::measure]
pub async fn heal_erasure_set(&self, buckets: &[String], set_disk_id: &str) -> Result<()> {
debug!(
target: "rustfs::heal::erasure_healer",
crates/heal/src/heal/task.rs
+3
View File
@@ -584,6 +584,7 @@ impl HealTask {
}
#[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))]
#[hotpath::measure]
pub async fn execute(&self) -> Result<()> {
// update status and timestamps atomically to avoid race conditions
let now = SystemTime::now();
@@ -759,6 +760,7 @@ impl HealTask {
// specific heal implementation method
#[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))]
#[hotpath::measure]
async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
debug!(
target: "rustfs::heal::task",
@@ -1404,6 +1406,7 @@ impl HealTask {
self.heal_bucket_objects(bucket, prefix).await
}
#[hotpath::measure]
async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> {
let mut continuation_token: Option<String> = None;
let mut scanned = 0u64;
crates/iam/Cargo.toml
+48
View File
@@ -28,7 +28,55 @@ documentation = "https://docs.rs/rustfs-iam/latest/rustfs_iam/"
[lints]
workspace = true
[features]
default = []
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"hotpath/futures",
"hotpath/reqwest-0-13",
"rustfs-config/hotpath",
"rustfs-credentials/hotpath",
"rustfs-crypto/hotpath",
"rustfs-ecstore/hotpath",
"rustfs-io-metrics/hotpath",
"rustfs-madmin/hotpath",
"rustfs-policy/hotpath",
"rustfs-storage-api/hotpath",
"rustfs-utils/hotpath",
"rustfs-test-utils/hotpath",
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-config/hotpath-alloc",
"rustfs-credentials/hotpath-alloc",
"rustfs-crypto/hotpath-alloc",
"rustfs-ecstore/hotpath-alloc",
"rustfs-io-metrics/hotpath-alloc",
"rustfs-madmin/hotpath-alloc",
"rustfs-policy/hotpath-alloc",
"rustfs-storage-api/hotpath-alloc",
"rustfs-utils/hotpath-alloc",
"rustfs-test-utils/hotpath-alloc",
]
hotpath-cpu = [
"hotpath",
"hotpath/hotpath-cpu",
"rustfs-config/hotpath-cpu",
"rustfs-credentials/hotpath-cpu",
"rustfs-crypto/hotpath-cpu",
"rustfs-ecstore/hotpath-cpu",
"rustfs-io-metrics/hotpath-cpu",
"rustfs-madmin/hotpath-cpu",
"rustfs-policy/hotpath-cpu",
"rustfs-storage-api/hotpath-cpu",
"rustfs-utils/hotpath-cpu",
"rustfs-test-utils/hotpath-cpu",
]
[dependencies]
hotpath.workspace = true
rustfs-credentials = { workspace = true }
rustfs-config = { workspace = true, features = ["server-config-model"] }
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
crates/iam/src/manager.rs
+27 -3
View File
@@ -556,7 +556,7 @@ where
Ok(now)
}
pub async fn list_polices(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
pub async fn list_policies(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
let mut m = HashMap::new();
self.api.load_policy_docs(&mut m).await?;
@@ -588,6 +588,15 @@ where
Ok(filtered)
}
/// Backward-compatible misspelling retained until the next breaking release.
#[deprecated(
since = "1.0.0",
note = "use list_policies instead; this alias will be removed in the next breaking release"
)]
pub async fn list_polices(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
self.list_policies(bucket_name).await
}
pub async fn merge_policies(&self, name: &str) -> (String, Policy) {
let mut policies = Vec::new();
let mut to_merge = Vec::new();
@@ -2184,7 +2193,7 @@ where
}
}
pub fn get_default_policyes() -> HashMap<String, PolicyDoc> {
pub fn get_default_policies() -> HashMap<String, PolicyDoc> {
let default_policies = &DEFAULT_POLICIES;
default_policies
.iter()
@@ -2201,6 +2210,15 @@ pub fn get_default_policyes() -> HashMap<String, PolicyDoc> {
.collect()
}
/// Backward-compatible misspelling retained until the next breaking release.
#[deprecated(
since = "1.0.0",
note = "use get_default_policies instead; this alias will be removed in the next breaking release"
)]
pub fn get_default_policyes() -> HashMap<String, PolicyDoc> {
get_default_policies()
}
fn set_default_canned_policies(policies: &mut HashMap<String, PolicyDoc>) {
let default_policies = &DEFAULT_POLICIES;
for (k, v) in default_policies.iter() {
@@ -2900,7 +2918,7 @@ mod tests {
#[test]
fn test_get_default_policies() {
let policies = get_default_policyes();
let policies = get_default_policies();
// Should contain some default policies
assert!(!policies.is_empty());
@@ -2913,6 +2931,12 @@ mod tests {
}
}
#[test]
#[allow(deprecated)]
fn deprecated_get_default_policyes_matches_current_api() {
assert_eq!(get_default_policyes().len(), get_default_policies().len());
}
#[test]
fn test_get_token_signing_key() {
// This function returns the global action credential's secret key
crates/iam/src/oidc.rs
+58 -27
View File
@@ -1139,8 +1139,11 @@ impl OidcSys {
let mut policies = Vec::new();
let mut groups = Vec::new();
// Add default role policy if configured
if !config.role_policy.is_empty() {
// Role-policy and claim-based authorization are separate OIDC modes. When a
// role policy is configured, group claims still provide group context but
// must not also become policy names.
let has_role_policy = !config.role_policy.trim().is_empty();
if has_role_policy {
for policy in config.role_policy.split(',') {
let policy = policy.trim();
if !policy.is_empty() {
@@ -1149,21 +1152,20 @@ impl OidcSys {
}
}
// Map groups claim to policies
for group in &claims.groups {
groups.push(group.clone());
let policy_name = if config.claim_prefix.is_empty() {
group.clone()
} else {
format!("{}{}", config.claim_prefix, group)
};
policies.push(policy_name);
if !has_role_policy {
let policy_name = if config.claim_prefix.is_empty() {
group.clone()
} else {
format!("{}{}", config.claim_prefix, group)
};
policies.push(policy_name);
}
}
// Map primary claim (if different from groups)
if config.claim_name != config.groups_claim {
let claim_values = extract_groups_claim(&claims.raw, &config.claim_name);
for val in claim_values {
if !has_role_policy && config.claim_name != config.groups_claim {
for val in extract_groups_claim(&claims.raw, &config.claim_name) {
let policy_name = if config.claim_prefix.is_empty() {
val
} else {
@@ -3201,26 +3203,22 @@ mod tests {
}
#[test]
fn test_map_claims_to_policies_with_provider() {
let mut config = test_config("okta");
config.role_policy = "readwrite".to_string();
config.display_name = "Okta".to_string();
fn role_policy_does_not_map_groups_as_policies() {
let mut config = test_config("authentik");
config.role_policy = "consoleAdmin".to_string();
config.claim_name = "policy".to_string();
let sys = make_test_sys(vec![config]);
let claims = OidcClaims {
sub: "user123".to_string(),
email: "user@example.com".to_string(),
username: "user".to_string(),
groups: vec!["admin".to_string(), "devs".to_string()],
raw: HashMap::new(),
groups: vec!["authentik Admins".to_string(), "users".to_string()],
raw: HashMap::from([("policy".to_string(), serde_json::json!(["readonly"]))]),
..Default::default()
};
let (policies, groups) = sys.map_claims_to_policies("okta", &claims);
assert_eq!(groups, vec!["admin", "devs"]);
assert!(policies.contains(&"readwrite".to_string()));
assert!(policies.contains(&"admin".to_string()));
assert!(policies.contains(&"devs".to_string()));
let (policies, groups) = sys.map_claims_to_policies("authentik", &claims);
assert_eq!(groups, vec!["authentik Admins", "users"]);
assert_eq!(policies, vec!["consoleAdmin"]);
}
#[test]
@@ -3245,6 +3243,39 @@ mod tests {
assert_eq!(policies.len(), 1);
}
#[test]
fn blank_role_policy_uses_claim_mapping() {
let mut config = test_config("keycloak");
config.role_policy = " ".to_string();
let sys = make_test_sys(vec![config]);
let claims = OidcClaims {
groups: vec!["readonly".to_string()],
..Default::default()
};
let (policies, groups) = sys.map_claims_to_policies("keycloak", &claims);
assert_eq!(groups, vec!["readonly"]);
assert_eq!(policies, vec!["readonly"]);
}
#[test]
fn claim_mapping_keeps_groups_with_distinct_primary_claim() {
let mut config = test_config("keycloak");
config.claim_name = "policy".to_string();
let sys = make_test_sys(vec![config]);
let claims = OidcClaims {
groups: vec!["developers".to_string()],
raw: HashMap::from([("policy".to_string(), serde_json::json!(["readonly"]))]),
..Default::default()
};
let (policies, groups) = sys.map_claims_to_policies("keycloak", &claims);
assert_eq!(groups, vec!["developers"]);
assert_eq!(policies, vec!["developers", "readonly"]);
}
#[test]
fn test_list_providers() {
let mut config = test_config("keycloak");
crates/iam/src/store/object.rs
+4 -2
View File
@@ -22,7 +22,7 @@ use crate::{
cache::{Cache, CacheEntity},
error::{is_err_no_such_policy, is_err_no_such_user},
keyring,
manager::{extract_jwt_claims, extract_jwt_claims_allow_missing_exp, get_default_policyes},
manager::{extract_jwt_claims, extract_jwt_claims_allow_missing_exp, get_default_policies},
root_credentials,
};
use futures::future::join_all;
@@ -469,6 +469,7 @@ impl ObjectStore {
});
}
#[hotpath::measure]
async fn list_all_iamconfig_items(&self) -> Result<HashMap<String, Vec<String>>> {
let (tx, mut rx) = mpsc::channel::<StringOrErr>(100);
@@ -508,6 +509,7 @@ impl ObjectStore {
Ok(res)
}
#[hotpath::measure]
async fn load_policy_doc_concurrent(&self, names: &[String], mode: LoadMode) -> Result<Vec<PolicyDoc>> {
let mut futures = Vec::with_capacity(names.len());
@@ -1127,7 +1129,7 @@ impl Store for ObjectStore {
let cache_snapshot = cache.snapshot();
let listed_config_items = self.list_all_iamconfig_items().await?;
let mut policy_docs_cache = CacheEntity::new(get_default_policyes());
let mut policy_docs_cache = CacheEntity::new(get_default_policies());
if let Some(policies_list) = listed_config_items.get(POLICIES_LIST_KEY) {
// Load in fixed-size chunks so each policy is fetched exactly once.
crates/iam/src/sys.rs
+20 -5
View File
@@ -18,7 +18,7 @@ use crate::error::is_err_no_such_temp_account;
use crate::error::{Error, Result};
use crate::federation::OIDC_VIRTUAL_PARENT_CLAIM;
use crate::manager::extract_jwt_claims;
use crate::manager::get_default_policyes;
use crate::manager::get_default_policies;
use crate::manager::{IamCache, IamSyncMetricsSnapshot};
use crate::store::GroupInfo;
use crate::store::MappedPolicy;
@@ -249,7 +249,7 @@ impl<T: Store> IamSys<T> {
}
pub async fn delete_policy(&self, name: &str, notify: bool) -> Result<()> {
for k in get_default_policyes().keys() {
for k in get_default_policies().keys() {
if k == name {
return Err(Error::other("system policy can not be deleted"));
}
@@ -291,8 +291,17 @@ impl<T: Store> IamSys<T> {
self.store.api.load_mapped_policies(user_type, is_group, m).await
}
pub async fn list_policies(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
self.store.list_policies(bucket_name).await
}
/// Backward-compatible misspelling retained until the next breaking release.
#[deprecated(
since = "1.0.0",
note = "use list_policies instead; this alias will be removed in the next breaking release"
)]
pub async fn list_polices(&self, bucket_name: &str) -> Result<HashMap<String, Policy>> {
self.store.list_polices(bucket_name).await
self.list_policies(bucket_name).await
}
pub async fn list_policy_docs(&self, bucket_name: &str) -> Result<HashMap<String, PolicyDoc>> {
@@ -1683,11 +1692,17 @@ mod tests {
use super::*;
use crate::cache::{Cache, CacheEntity};
use crate::error::Error;
use crate::manager::get_default_policyes;
use crate::manager::get_default_policies;
use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
use rustfs_credentials::{Credentials, init_global_action_credentials};
use rustfs_policy::auth::{UserIdentity, get_new_credentials_with_metadata};
use rustfs_policy::policy::Args;
#[test]
#[allow(deprecated)]
fn deprecated_list_polices_api_is_available() {
let _ = IamSys::<StsTestMockStore>::list_polices;
}
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
use rustfs_policy::policy::policy_uses_existing_object_tag_conditions;
use serde_json::Value;
@@ -1925,7 +1940,7 @@ mod tests {
}
async fn load_all(&self, cache: &Cache) -> Result<()> {
let mut policy_docs = get_default_policyes();
let mut policy_docs = get_default_policies();
let custom_claim_policy =
Policy::parse_config(CUSTOM_STS_CLAIM_POLICY_JSON.as_bytes()).expect("custom STS claim policy should parse");
policy_docs.insert(CUSTOM_STS_CLAIM_POLICY.to_string(), PolicyDoc::new(custom_claim_policy));
crates/io-core/Cargo.toml
+7
View File
@@ -27,7 +27,14 @@ categories = ["development-tools", "filesystem"]
[lints]
workspace = true
[features]
default = []
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-io-metrics/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-io-metrics/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-metrics/hotpath-cpu"]
[dependencies]
hotpath.workspace = true
bytes = { workspace = true, features = ["serde"] }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["io-util", "fs", "sync", "rt-multi-thread"] }
crates/io-metrics/Cargo.toml
+27
View File
@@ -28,9 +28,36 @@ categories = ["development-tools", "filesystem"]
name = "metrics_pipeline"
harness = false
[features]
default = []
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"rustfs-common/hotpath",
"rustfs-s3-ops/hotpath",
"rustfs-utils/hotpath",
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-common/hotpath-alloc",
"rustfs-s3-ops/hotpath-alloc",
"rustfs-utils/hotpath-alloc",
]
hotpath-cpu = [
"hotpath",
"hotpath/hotpath-cpu",
"rustfs-common/hotpath-cpu",
"rustfs-s3-ops/hotpath-cpu",
"rustfs-utils/hotpath-cpu",
]
[dependencies]
hotpath.workspace = true
metrics = { workspace = true }
rustfs-common = { workspace = true }
rustfs-s3-ops = { workspace = true }
rustfs-utils = { workspace = true, features = ["ip"] }
num_cpus = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
crates/io-metrics/src/internode_metrics.rs
+199 -54
View File
@@ -15,7 +15,7 @@
use metrics::{counter, gauge};
use std::collections::HashMap;
use std::sync::{
Arc, LazyLock, RwLock,
Arc, LazyLock, OnceLock, RwLock,
atomic::{AtomicU64, Ordering},
};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
@@ -40,6 +40,7 @@ pub const INTERNODE_MSGPACK_CODEC_JSON: &str = "json";
const OPERATION_LABEL: &str = "operation";
const BACKEND_LABEL: &str = "backend";
const SERVER_LABEL: &str = "server";
const CLASSIFICATION_LABEL: &str = "classification";
const STAGE_LABEL: &str = "stage";
const DOMINANT_ERROR_LABEL: &str = "dominant_error";
@@ -77,74 +78,93 @@ pub struct InternodeOperationMetricDescriptor {
pub labels: &'static [&'static str],
}
const OPERATION_BACKEND_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL];
const OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL];
const OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL];
const QUORUM_FAILURE_LABELS: &[&str] = &[STAGE_LABEL, DOMINANT_ERROR_LABEL];
const SERVER_OPERATION_BACKEND_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL];
const SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] =
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL];
const SERVER_OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL];
const SERVER_QUORUM_FAILURE_LABELS: &[&str] = &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL];
pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &[
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_SENT_BYTES_TOTAL,
labels: OPERATION_BACKEND_LABELS,
labels: SERVER_OPERATION_BACKEND_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_RECV_BYTES_TOTAL,
labels: OPERATION_BACKEND_LABELS,
labels: SERVER_OPERATION_BACKEND_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL,
labels: OPERATION_BACKEND_LABELS,
labels: SERVER_OPERATION_BACKEND_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL,
labels: OPERATION_BACKEND_LABELS,
labels: SERVER_OPERATION_BACKEND_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_ERRORS_TOTAL,
labels: OPERATION_BACKEND_LABELS,
labels: SERVER_OPERATION_BACKEND_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_DURATION_MS,
labels: OPERATION_BACKEND_LABELS,
labels: SERVER_OPERATION_BACKEND_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL,
labels: OPERATION_BACKEND_CLASSIFICATION_LABELS,
labels: SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_RETRIES_TOTAL,
labels: OPERATION_BACKEND_CLASSIFICATION_LABELS,
labels: SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL,
labels: OPERATION_BACKEND_CLASSIFICATION_LABELS,
labels: SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL,
labels: OPERATION_BACKEND_HTTP_VERSION_LABELS,
labels: SERVER_OPERATION_BACKEND_HTTP_VERSION_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL,
labels: OPERATION_BACKEND_LABELS,
labels: SERVER_OPERATION_BACKEND_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL,
labels: OPERATION_BACKEND_LABELS,
labels: SERVER_OPERATION_BACKEND_LABELS,
},
InternodeOperationMetricDescriptor {
name: ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
labels: QUORUM_FAILURE_LABELS,
labels: SERVER_QUORUM_FAILURE_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_PAYLOAD_BYTES,
labels: OPERATION_BACKEND_LABELS,
labels: SERVER_OPERATION_BACKEND_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL,
labels: OPERATION_BACKEND_LABELS,
labels: SERVER_OPERATION_BACKEND_LABELS,
},
];
fn current_server_label() -> &'static str {
static STABLE_SERVER_LABEL: OnceLock<String> = OnceLock::new();
static FALLBACK_SERVER_LABEL: LazyLock<String> = LazyLock::new(rustfs_utils::get_local_ip_with_default);
if let Some(server) = STABLE_SERVER_LABEL.get() {
return server.as_str();
}
if let Some(server) = rustfs_common::try_get_global_local_node_name() {
let _ = STABLE_SERVER_LABEL.set(server);
if let Some(server) = STABLE_SERVER_LABEL.get() {
return server.as_str();
}
}
FALLBACK_SERVER_LABEL.as_str()
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct InternodeMetricsSnapshot {
pub sent_bytes_total: u64,
@@ -193,7 +213,7 @@ impl InternodeMetrics {
return;
}
self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed);
counter!("rustfs_system_network_internode_sent_bytes_total").increment(bytes);
counter!("rustfs_system_network_internode_sent_bytes_total", SERVER_LABEL => current_server_label()).increment(bytes);
}
pub fn record_sent_bytes_for_operation(&self, operation: &'static str, bytes: usize) {
@@ -207,7 +227,13 @@ impl InternodeMetrics {
if bytes == 0 {
return;
}
counter!(INTERNODE_OPERATION_SENT_BYTES_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(bytes);
counter!(
INTERNODE_OPERATION_SENT_BYTES_TOTAL,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend
)
.increment(bytes);
}
pub fn record_recv_bytes(&self, bytes: usize) {
@@ -216,7 +242,7 @@ impl InternodeMetrics {
return;
}
self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed);
counter!("rustfs_system_network_internode_recv_bytes_total").increment(bytes);
counter!("rustfs_system_network_internode_recv_bytes_total", SERVER_LABEL => current_server_label()).increment(bytes);
}
pub fn record_recv_bytes_for_operation(&self, operation: &'static str, bytes: usize) {
@@ -230,12 +256,18 @@ impl InternodeMetrics {
if bytes == 0 {
return;
}
counter!(INTERNODE_OPERATION_RECV_BYTES_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(bytes);
counter!(
INTERNODE_OPERATION_RECV_BYTES_TOTAL,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend
)
.increment(bytes);
}
pub fn record_outgoing_request(&self) {
self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed);
counter!("rustfs_system_network_internode_requests_outgoing_total").increment(1);
counter!("rustfs_system_network_internode_requests_outgoing_total", SERVER_LABEL => current_server_label()).increment(1);
}
pub fn record_outgoing_request_for_operation(&self, operation: &'static str) {
@@ -244,13 +276,18 @@ impl InternodeMetrics {
pub fn record_outgoing_request_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
self.record_outgoing_request();
counter!(INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend)
.increment(1);
counter!(
INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend
)
.increment(1);
}
pub fn record_incoming_request(&self) {
self.incoming_requests_total.fetch_add(1, Ordering::Relaxed);
counter!("rustfs_system_network_internode_requests_incoming_total").increment(1);
counter!("rustfs_system_network_internode_requests_incoming_total", SERVER_LABEL => current_server_label()).increment(1);
}
pub fn record_incoming_request_for_operation(&self, operation: &'static str) {
@@ -259,13 +296,18 @@ impl InternodeMetrics {
pub fn record_incoming_request_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
self.record_incoming_request();
counter!(INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend)
.increment(1);
counter!(
INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend
)
.increment(1);
}
pub fn record_error(&self) {
self.errors_total.fetch_add(1, Ordering::Relaxed);
counter!("rustfs_system_network_internode_errors_total").increment(1);
counter!("rustfs_system_network_internode_errors_total", SERVER_LABEL => current_server_label()).increment(1);
}
pub fn record_error_for_operation(&self, operation: &'static str) {
@@ -274,13 +316,24 @@ impl InternodeMetrics {
pub fn record_error_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
self.record_error();
counter!(INTERNODE_OPERATION_ERRORS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(1);
counter!(
INTERNODE_OPERATION_ERRORS_TOTAL,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend
)
.increment(1);
}
pub fn record_duration_for_operation_and_backend(&self, operation: &'static str, backend: &'static str, duration: Duration) {
let duration_ms = duration.as_secs_f64() * 1000.0;
metrics::histogram!(INTERNODE_OPERATION_DURATION_MS, OPERATION_LABEL => operation, BACKEND_LABEL => backend)
.record(duration_ms);
metrics::histogram!(
INTERNODE_OPERATION_DURATION_MS,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend
)
.record(duration_ms);
}
pub fn record_classified_error_for_operation_and_backend(
@@ -291,6 +344,7 @@ impl InternodeMetrics {
) {
counter!(
INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend,
CLASSIFICATION_LABEL => classification
@@ -306,6 +360,7 @@ impl InternodeMetrics {
) {
counter!(
INTERNODE_OPERATION_RETRIES_TOTAL,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend,
CLASSIFICATION_LABEL => classification
@@ -321,6 +376,7 @@ impl InternodeMetrics {
) {
counter!(
INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend,
CLASSIFICATION_LABEL => classification
@@ -337,6 +393,7 @@ impl InternodeMetrics {
self.operation_http_versions_total.fetch_add(1, Ordering::Relaxed);
counter!(
INTERNODE_OPERATION_HTTP_VERSIONS_TOTAL,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend,
HTTP_VERSION_LABEL => http_version
@@ -346,13 +403,24 @@ impl InternodeMetrics {
pub fn record_stall_timeout_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
self.operation_stall_timeouts_total.fetch_add(1, Ordering::Relaxed);
counter!(INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(1);
counter!(
INTERNODE_OPERATION_STALL_TIMEOUTS_TOTAL,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend
)
.increment(1);
}
pub fn record_write_shutdown_error_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
self.operation_write_shutdown_errors_total.fetch_add(1, Ordering::Relaxed);
counter!(INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend)
.increment(1);
counter!(
INTERNODE_OPERATION_WRITE_SHUTDOWN_ERRORS_TOTAL,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend
)
.increment(1);
}
/// Record the payload size (bytes) of a completed internode operation into a histogram
@@ -360,15 +428,26 @@ impl InternodeMetrics {
/// (`ReadAll`/`ReadMultiple`/`WriteAll`) would benefit from being moved off the shared
/// control-plane channel (see docs/grpc-optimization P1).
pub fn record_operation_payload_bytes(&self, operation: &'static str, backend: &'static str, bytes: usize) {
metrics::histogram!(INTERNODE_OPERATION_PAYLOAD_BYTES, OPERATION_LABEL => operation, BACKEND_LABEL => backend)
.record(bytes as f64);
metrics::histogram!(
INTERNODE_OPERATION_PAYLOAD_BYTES,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend
)
.record(bytes as f64);
}
/// Increment the large-payload counter for an operation+backend whose payload exceeded the
/// caller-configured warning threshold. Feeds alerting on large unary RPCs that contend with
/// latency-sensitive control-plane traffic on the shared connection.
pub fn record_large_operation_payload(&self, operation: &'static str, backend: &'static str) {
counter!(INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(1);
counter!(
INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL,
SERVER_LABEL => current_server_label(),
OPERATION_LABEL => operation,
BACKEND_LABEL => backend
)
.increment(1);
}
/// Count a decode that fell back to the JSON compatibility field because the msgpack `_bin`
@@ -377,13 +456,20 @@ impl InternodeMetrics {
/// dropped (grpc-optimization P2). `direction` is [`INTERNODE_MSGPACK_DIRECTION_REQUEST`] or
/// [`INTERNODE_MSGPACK_DIRECTION_RESPONSE`]; `message` is the low-cardinality value name.
pub fn record_msgpack_json_fallback(&self, direction: &'static str, message: &'static str) {
counter!(INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL, DIRECTION_LABEL => direction, MESSAGE_LABEL => message).increment(1);
counter!(
INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL,
SERVER_LABEL => current_server_label(),
DIRECTION_LABEL => direction,
MESSAGE_LABEL => message
)
.increment(1);
}
pub fn record_msgpack_json_decode(&self, direction: &'static str, message: &'static str, codec: &'static str) {
self.msgpack_json_decode_total.fetch_add(1, Ordering::Relaxed);
counter!(
INTERNODE_MSGPACK_JSON_DECODE_TOTAL,
SERVER_LABEL => current_server_label(),
DIRECTION_LABEL => direction,
MESSAGE_LABEL => message,
CODEC_LABEL => codec
@@ -395,6 +481,7 @@ impl InternodeMetrics {
self.msgpack_json_decode_error_total.fetch_add(1, Ordering::Relaxed);
counter!(
INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL,
SERVER_LABEL => current_server_label(),
DIRECTION_LABEL => direction,
MESSAGE_LABEL => message,
CODEC_LABEL => codec
@@ -420,7 +507,7 @@ impl InternodeMetrics {
/// enabled; after the strict flip the legacy fallback path is closed and the counter stays flat.
pub fn record_signature_v1_fallback(&self) {
self.signature_v1_fallback_total.fetch_add(1, Ordering::Relaxed);
counter!(INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL).increment(1);
counter!(INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL, SERVER_LABEL => current_server_label()).increment(1);
}
/// Count a mutating internode disk RPC that was accepted without a signature-bound canonical
@@ -431,14 +518,14 @@ impl InternodeMetrics {
/// mutations are rejected and the counter stays flat.
pub fn record_body_digest_fallback(&self) {
self.body_digest_fallback_total.fetch_add(1, Ordering::Relaxed);
counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1);
counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL, SERVER_LABEL => current_server_label()).increment(1);
}
/// Count an accepted v1/v2 request that does not carry the replay-scoped signature. This is
/// the convergence signal for `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT`.
pub fn record_replay_scope_fallback(&self) {
self.replay_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
counter!(INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL).increment(1);
counter!(INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL, SERVER_LABEL => current_server_label()).increment(1);
}
/// Count a body-bound internode RPC rejected because the replay-protection nonce cache was
@@ -447,12 +534,13 @@ impl InternodeMetrics {
/// mutation rate and writes are being refused — alert on this counter.
pub fn record_replay_cache_overflow(&self) {
self.replay_cache_overflow_total.fetch_add(1, Ordering::Relaxed);
counter!(INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL).increment(1);
counter!(INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL, SERVER_LABEL => current_server_label()).increment(1);
}
pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) {
counter!(
ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
SERVER_LABEL => current_server_label(),
STAGE_LABEL => stage,
DOMINANT_ERROR_LABEL => dominant_error
)
@@ -464,11 +552,12 @@ impl InternodeMetrics {
self.dial_total_time_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed);
let samples = self.dial_samples_total.fetch_add(1, Ordering::Relaxed) + 1;
let total = self.dial_total_time_nanos.load(Ordering::Relaxed);
gauge!("rustfs_system_network_internode_dial_avg_time_nanos").set(total as f64 / samples as f64);
gauge!("rustfs_system_network_internode_dial_avg_time_nanos", SERVER_LABEL => current_server_label())
.set(total as f64 / samples as f64);
if !success {
self.dial_errors_total.fetch_add(1, Ordering::Relaxed);
counter!("rustfs_system_network_internode_dial_errors_total").increment(1);
counter!("rustfs_system_network_internode_dial_errors_total", SERVER_LABEL => current_server_label()).increment(1);
}
let now_ms = SystemTime::now()
@@ -687,6 +776,9 @@ fn cluster_peer_health_keys() -> Vec<String> {
#[cfg(test)]
mod tests {
use super::*;
use metrics::with_local_recorder;
use metrics_util::debugging::DebuggingRecorder;
use std::collections::HashSet;
#[test]
fn snapshot_reports_recorded_values() {
@@ -750,22 +842,22 @@ mod tests {
fn operation_metric_descriptors_include_backend_and_operation_labels() {
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 15);
for metric in &INTERNODE_OPERATION_METRICS[..6] {
assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]);
assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
}
for metric in &INTERNODE_OPERATION_METRICS[6..9] {
assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]);
assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]);
}
assert_eq!(
INTERNODE_OPERATION_METRICS[9].labels,
&[OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL]
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL]
);
for metric in &INTERNODE_OPERATION_METRICS[10..12] {
assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]);
assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
}
assert_eq!(INTERNODE_OPERATION_METRICS[12].labels, &[STAGE_LABEL, DOMINANT_ERROR_LABEL]);
assert_eq!(INTERNODE_OPERATION_METRICS[12].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]);
// Payload histogram + large-payload counter carry operation+backend labels.
assert_eq!(INTERNODE_OPERATION_METRICS[13].labels, &[OPERATION_LABEL, BACKEND_LABEL]);
assert_eq!(INTERNODE_OPERATION_METRICS[14].labels, &[OPERATION_LABEL, BACKEND_LABEL]);
assert_eq!(INTERNODE_OPERATION_METRICS[13].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
assert_eq!(INTERNODE_OPERATION_METRICS[14].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
}
#[test]
@@ -843,6 +935,59 @@ mod tests {
);
}
#[test]
fn direct_internode_metrics_emit_stable_server_label() {
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let metrics = InternodeMetrics::default();
with_local_recorder(&recorder, || {
metrics.record_sent_bytes_for_operation_and_backend(
INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
128,
);
metrics.record_recv_bytes_for_operation_and_backend(
INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
256,
);
metrics.record_dial_result(Duration::from_millis(3), false);
});
let observed: Vec<(String, HashSet<String>, Option<String>)> = snapshotter
.snapshot()
.into_vec()
.into_iter()
.filter(|(composite, _, _, _)| {
matches!(
composite.key().name(),
"rustfs_system_network_internode_sent_bytes_total"
| "rustfs_system_network_internode_recv_bytes_total"
| INTERNODE_OPERATION_SENT_BYTES_TOTAL
| INTERNODE_OPERATION_RECV_BYTES_TOTAL
| "rustfs_system_network_internode_dial_avg_time_nanos"
| "rustfs_system_network_internode_dial_errors_total"
)
})
.map(|(composite, _, _, _)| {
let labels = composite.key().labels();
let keys = labels.clone().map(|label| label.key().to_string()).collect();
let server = labels
.filter(|label| label.key() == SERVER_LABEL)
.map(|label| label.value().to_string())
.next();
(composite.key().name().to_string(), keys, server)
})
.collect();
assert_eq!(observed.len(), 6);
for (name, keys, server) in observed {
assert!(keys.contains(SERVER_LABEL), "{name} must carry the server label");
assert!(server.is_some_and(|value| !value.is_empty()), "{name} server label must not be empty");
}
}
#[test]
fn msgpack_json_fallback_counter_records_without_panicking() {
// Smoke test: the counter accepts both directions and a static message label.
crates/keystone/Cargo.toml
+27
View File
@@ -28,7 +28,34 @@ authors.workspace = true
[lints]
workspace = true
[features]
default = []
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"hotpath/futures",
"hotpath/reqwest-0-13",
"rustfs-credentials/hotpath",
"rustfs-policy/hotpath",
"rustfs-utils/hotpath",
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-credentials/hotpath-alloc",
"rustfs-policy/hotpath-alloc",
"rustfs-utils/hotpath-alloc",
]
hotpath-cpu = [
"hotpath",
"hotpath/hotpath-cpu",
"rustfs-credentials/hotpath-cpu",
"rustfs-policy/hotpath-cpu",
"rustfs-utils/hotpath-cpu",
]
[dependencies]
hotpath.workspace = true
tokio = { workspace = true, features = ["rt", "sync"] }
reqwest = { workspace = true, features = ["json"] }
serde = { workspace = true, features = ["derive"] }
crates/keystone/src/client.rs
+2
View File
@@ -95,6 +95,7 @@ impl KeystoneClient {
}
/// Validate a Keystone token
#[hotpath::measure]
pub async fn validate_token(&self, token: &str) -> Result<KeystoneToken> {
match self.version {
KeystoneVersion::V3 => self.validate_token_v3(token).await,
@@ -238,6 +239,7 @@ impl KeystoneClient {
}
/// Get EC2 credentials for a user
#[hotpath::measure]
pub async fn get_ec2_credentials(&self, user_id: &str, project_id: Option<&str>) -> Result<Vec<EC2Credential>> {
let admin_token = self.get_admin_token().await?;
crates/kms/AGENTS.md
+22
View File
@@ -25,3 +25,25 @@ For local KMS end-to-end tests, keep proxy bypass settings:
NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \
cargo test --package e2e_test test_local_kms_end_to_end -- --nocapture --test-threads=1
```
## Local Key Export for SSE-S3 Migration Tests
Use the read-only `local_kms_key_decrypt` example to export an AES-256 Local
KMS key as the base64 value expected by `RUSTFS_SSE_S3_MASTER_KEY`:
```bash
export RUSTFS_KMS_LOCAL_MASTER_KEY='<local-kms-at-rest-master-key>'
export RUSTFS_SSE_S3_MASTER_KEY="$(
cargo run -q -p rustfs-kms --example local_kms_key_decrypt -- \
/absolute/path/to/<key-id>.key
)"
```
For a `plaintext-dev-only` Local KMS key file,
`RUSTFS_KMS_LOCAL_MASTER_KEY` is not required.
The example writes only the base64-encoded 32-byte key to stdout. Diagnostics
go to stderr. Never paste its output into logs, shell history, issue comments,
or committed configuration. The export path must remain read-only and must
reuse `LocalKmsClient` decoding so current Argon2id and legacy key-file
compatibility stay aligned with the backend.
crates/kms/Cargo.toml
+26
View File
@@ -28,6 +28,7 @@ categories = ["cryptography", "web-programming", "authentication"]
workspace = true
[dependencies]
hotpath.workspace = true
# Core dependencies
async-trait = { workspace = true }
tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thread", "sync", "time"] }
@@ -37,6 +38,8 @@ serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true, features = ["raw_value"] }
tracing = { workspace = true }
thiserror = { workspace = true }
# Operation metrics emitted by the retry policy engine (crate::policy).
metrics = { workspace = true }
# Cryptography
aes-gcm = { workspace = true, features = ["rand_core"] }
@@ -65,11 +68,34 @@ rustfs-security-governance = { workspace = true }
# HTTP client for Vault
reqwest = { workspace = true }
vaultrs = { workspace = true }
# vaultrs surfaces transport-level failures as wrapped rustify errors; the
# operation policy needs the concrete type to classify them for retry decisions.
rustify = { workspace = true }
tokio-util = { workspace = true }
[dev-dependencies]
anyhow = { workspace = true }
# Debugging recorder for asserting emitted metrics in tests.
metrics-util = { version = "0.20", features = ["debugging"] }
insta = { workspace = true, features = ["yaml", "json"] }
tempfile = { workspace = true }
temp-env = { workspace = true }
# "net" backs the scripted loopback Vault used by the policy wiring tests.
tokio = { workspace = true, features = ["net", "test-util"] }
[features]
default = []
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"hotpath/reqwest-0-13",
"rustfs-security-governance/hotpath",
"rustfs-utils/hotpath",
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-security-governance/hotpath-alloc",
"rustfs-utils/hotpath-alloc",
]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-security-governance/hotpath-cpu", "rustfs-utils/hotpath-cpu"]
crates/kms/examples/local_kms_key_decrypt.rs
+112
View File
@@ -0,0 +1,112 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use rustfs_kms::{LocalConfig, backends::local::LocalKmsClient};
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use zeroize::Zeroizing;
const LOCAL_KMS_MASTER_KEY_ENV: &str = "RUSTFS_KMS_LOCAL_MASTER_KEY";
fn usage(program: &str) -> String {
format!(
"Usage: {program} <local-kms-key-file>\n\
Reads {LOCAL_KMS_MASTER_KEY_ENV} when the key file is encrypted.\n\
Writes only the base64-encoded 32-byte key to stdout."
)
}
fn resolve_key_file(path: &Path) -> Result<(PathBuf, String), String> {
let canonical = std::fs::canonicalize(path).map_err(|error| format!("cannot open Local KMS key file: {error}"))?;
if canonical.extension().and_then(|extension| extension.to_str()) != Some("key") {
return Err("Local KMS key file must have a .key extension".to_string());
}
let key_dir = canonical
.parent()
.ok_or_else(|| "Local KMS key file must have a parent directory".to_string())?
.to_path_buf();
let key_id = canonical
.file_stem()
.and_then(|stem| stem.to_str())
.filter(|stem| !stem.is_empty())
.ok_or_else(|| "Local KMS key file name must contain a valid UTF-8 key ID".to_string())?
.to_string();
Ok((key_dir, key_id))
}
async fn run() -> Result<(), String> {
let mut args = std::env::args();
let program = args.next().unwrap_or_else(|| "local_kms_key_decrypt".to_string());
let Some(key_file) = args.next() else {
return Err(usage(&program));
};
if args.next().is_some() {
return Err(usage(&program));
}
let (key_dir, key_id) = resolve_key_file(Path::new(&key_file))?;
let master_key = std::env::var(LOCAL_KMS_MASTER_KEY_ENV).ok().filter(|value| !value.is_empty());
let client = LocalKmsClient::new_for_key_export(LocalConfig {
key_dir,
master_key,
file_permissions: Some(0o600),
})
.await
.map_err(|error| error.to_string())?;
let key_material = client
.decrypt_key_material_for_export(&key_id)
.await
.map_err(|error| error.to_string())?;
let encoded = Zeroizing::new(BASE64_STANDARD.encode(key_material.as_ref()));
let mut stdout = io::stdout().lock();
writeln!(stdout, "{}", encoded.as_str()).map_err(|error| format!("failed to write decrypted key: {error}"))
}
#[tokio::main]
async fn main() {
if let Err(error) = run().await {
let _ = writeln!(io::stderr().lock(), "local_kms_key_decrypt: {error}");
std::process::exit(1);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn resolve_key_file_extracts_directory_and_key_id() {
let directory = tempfile::tempdir().expect("create temporary directory");
let key_file = directory.path().join("migration-key.key");
std::fs::write(&key_file, b"{}").expect("create key file");
let (key_dir, key_id) = resolve_key_file(&key_file).expect("resolve key file");
assert_eq!(key_dir, directory.path().canonicalize().expect("canonical directory"));
assert_eq!(key_id, "migration-key");
}
#[test]
fn resolve_key_file_rejects_non_key_extension() {
let directory = tempfile::tempdir().expect("create temporary directory");
let key_file = directory.path().join("migration-key.json");
std::fs::write(&key_file, b"{}").expect("create key file");
let error = resolve_key_file(&key_file).expect_err("non-key file must be rejected");
assert!(error.contains(".key"));
}
}
crates/kms/src/api_types.rs
+102 -12
View File
@@ -71,7 +71,10 @@ impl fmt::Debug for ConfigureLocalKmsRequest {
}
}
/// Request to configure KMS with Vault KV v2 + Transit backend
/// Request to configure KMS with the Vault KV v2 storage backend.
///
/// This backend stores master key material directly in KV v2; confidentiality relies on
/// Vault ACLs and KV v2 at-rest encryption, with no Transit wrapping involved.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ConfigureVaultKmsRequest {
@@ -82,7 +85,8 @@ pub struct ConfigureVaultKmsRequest {
pub auth_method: VaultAuthMethod,
/// Vault namespace (Vault Enterprise, optional)
pub namespace: Option<String>,
/// Transit engine mount path
/// Deprecated: legacy Transit engine mount path. Still accepted so older clients keep
/// working, but the Vault KV2 backend never uses it.
pub mount_path: Option<String>,
/// KV engine mount path for storing keys
pub kv_mount: Option<String>,
@@ -192,7 +196,7 @@ pub enum ConfigureKmsRequest {
/// Configure with Local backend
#[serde(alias = "local", alias = "Local")]
Local(ConfigureLocalKmsRequest),
/// Configure with Vault KV v2 + Transit backend
/// Configure with the Vault KV v2 storage backend
#[serde(
rename = "VaultKV2",
alias = "Vault",
@@ -231,15 +235,55 @@ pub struct StartKmsRequest {
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
enum StrictVaultAuthMethod {
Token { token: String },
AppRole { role_id: String, secret_id: String },
Token {
token: String,
},
AppRole {
role_id: String,
#[serde(default)]
secret_id: String,
#[serde(default)]
secret_id_file: Option<std::path::PathBuf>,
#[serde(default)]
mount: Option<String>,
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
TokenFile {
path: std::path::PathBuf,
#[serde(default)]
poll_interval_secs: Option<u64>,
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
}
impl From<StrictVaultAuthMethod> for VaultAuthMethod {
fn from(value: StrictVaultAuthMethod) -> Self {
match value {
StrictVaultAuthMethod::Token { token } => Self::Token { token },
StrictVaultAuthMethod::AppRole { role_id, secret_id } => Self::AppRole { role_id, secret_id },
StrictVaultAuthMethod::AppRole {
role_id,
secret_id,
secret_id_file,
mount,
refresh_safety_window_secs,
} => Self::AppRole {
role_id,
secret_id,
secret_id_file,
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()),
refresh_safety_window_secs,
},
StrictVaultAuthMethod::TokenFile {
path,
poll_interval_secs,
refresh_safety_window_secs,
} => Self::TokenFile {
path,
poll_interval_secs,
refresh_safety_window_secs,
},
}
}
}
@@ -333,7 +377,7 @@ pub enum BackendSummary {
/// File permissions (octal)
file_permissions: Option<u32>,
},
/// Vault KV v2 + Transit backend summary
/// Vault KV v2 storage backend summary
#[serde(alias = "vault")]
VaultKv2 {
/// Vault server address
@@ -344,7 +388,8 @@ pub enum BackendSummary {
has_stored_credentials: bool,
/// Namespace (if configured)
namespace: Option<String>,
/// Transit engine mount path
/// Deprecated: legacy Transit mount path. Unused by the backend; kept only so the
/// serialized response shape stays stable for existing consumers.
mount_path: String,
/// KV engine mount path
kv_mount: String,
@@ -398,6 +443,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
auth_method_type: match &vault_config.auth_method {
VaultAuthMethod::Token { .. } => "token".to_string(),
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
},
has_stored_credentials: true,
namespace: vault_config.namespace.clone(),
@@ -411,6 +457,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
auth_method_type: match &vault_config.auth_method {
VaultAuthMethod::Token { .. } => "token".to_string(),
VaultAuthMethod::AppRole { .. } => "approle".to_string(),
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
},
has_stored_credentials: true,
namespace: vault_config.namespace.clone(),
@@ -621,6 +668,52 @@ mod tests {
}
}
#[test]
fn test_deserialize_vault_kv2_configure_request_mount_path_optional_but_accepted() {
// deny_unknown_fields regression guard: mount_path is deprecated but must remain
// accepted so older clients that still send it do not get a 400.
let with_mount_path = serde_json::json!({
"backend_type": "VaultKV2",
"address": "http://127.0.0.1:8200",
"auth_method": { "Token": { "token": "dev-root-token" } },
"mount_path": "transit"
});
let request: ConfigureKmsRequest =
serde_json::from_value(with_mount_path).expect("request with deprecated mount_path should deserialize");
let config = request.to_kms_config();
assert_eq!(config.vault_config().expect("vault-kv2 config").mount_path, "transit");
let without_mount_path = serde_json::json!({
"backend_type": "VaultKV2",
"address": "http://127.0.0.1:8200",
"auth_method": { "Token": { "token": "dev-root-token" } }
});
let request: ConfigureKmsRequest =
serde_json::from_value(without_mount_path).expect("request without mount_path should deserialize");
let config = request.to_kms_config();
assert_eq!(config.vault_config().expect("vault-kv2 config").mount_path, "transit");
}
#[test]
fn test_vault_kv2_status_summary_does_not_mention_transit() {
let config = KmsConfig::vault(
url::Url::parse("https://vault.example.com:8200").expect("vault URL"),
"summary-token".to_string(),
);
let response = KmsStatusResponse {
status: KmsServiceStatus::Running,
backend_type: Some(config.backend.clone()),
healthy: Some(true),
config_summary: Some(KmsConfigSummary::from(&config)),
};
let json = serde_json::to_string(&response).expect("kms status response should serialize");
assert!(
!json.contains("Transit"),
"vault-kv2 status output must not describe the backend as Transit: {json}"
);
}
#[test]
fn test_deserialize_vault_transit_configure_request() {
let cases = ["VaultTransit", "vault-transit", "vault_transit"];
@@ -821,10 +914,7 @@ mod tests {
});
let approle = ConfigureKmsRequest::VaultKv2(ConfigureVaultKmsRequest {
address: "https://vault.example.com:8200".to_string(),
auth_method: VaultAuthMethod::AppRole {
role_id: "configure-role-id".to_string(),
secret_id: "configure-approle-secret-id".to_string(),
},
auth_method: VaultAuthMethod::approle("configure-role-id".to_string(), "configure-approle-secret-id".to_string()),
namespace: None,
mount_path: Some("transit".to_string()),
kv_mount: Some("secret".to_string()),
crates/kms/src/backends/contract_tests.rs
+345
View File
@@ -0,0 +1,345 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Shared key state × operation contract tests for KMS backends.
//!
//! Every stateful backend must satisfy the same lifecycle matrix (see
//! `ensure_key_state_permits`): Enabled permits everything, Disabled permits
//! decryption and lifecycle recovery but rejects new cryptographic use, and
//! PendingDeletion rejects everything except decryption and cancellation.
//! Decryption staying available in Disabled/PendingDeletion is an explicit,
//! tested deviation from AWS KMS: disabling a key must not break reads of
//! objects already encrypted under it.
//!
//! The full matrix runs offline against the Local backend. The Vault KV2 and
//! Vault Transit runs exercise the same helper but need a live Vault dev
//! server, so they are `#[ignore]`d in CI. Static is covered by its own
//! stateless contract below.
use super::KmsBackend;
use super::local::LocalKmsBackend;
use super::static_kms::StaticKmsBackend;
use super::vault::VaultKmsBackend;
use super::vault_transit::VaultTransitKmsBackend;
use crate::config::KmsConfig;
use crate::error::{KmsError, Result};
use crate::manager::KmsManager;
use crate::service::ObjectEncryptionService;
use crate::types::{
CancelKeyDeletionRequest, CreateKeyRequest, DecryptRequest, DeleteKeyRequest, DescribeKeyRequest, EncryptRequest,
GenerateDataKeyRequest, KeySpec, KeyState, KeyUsage, ObjectEncryptionContext,
};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use rand::RngExt as _;
use std::collections::HashMap;
use std::sync::Arc;
fn expect_unsupported<T: std::fmt::Debug>(result: Result<T>) {
match result {
Err(KmsError::UnsupportedCapability { .. }) => {}
other => panic!("expected UnsupportedCapability, got {other:?}"),
}
}
/// Rotation while not Enabled: backends with rotation support must reject it
/// through the state machine; backends without it report the capability gap.
async fn expect_rotate_rejected(backend: &dyn KmsBackend, key_id: &str) {
let result = backend.rotate_key(key_id).await;
if backend.capabilities().rotate {
expect_invalid_key_state(result, "");
} else {
expect_unsupported(result);
}
}
fn expect_invalid_key_state<T: std::fmt::Debug>(result: Result<T>, expected_fragment: &str) {
match result {
Err(KmsError::InvalidOperation { message }) => assert!(
message.contains(expected_fragment),
"expected invalid-key-state message containing {expected_fragment:?}, got {message:?}"
),
other => panic!("expected InvalidOperation (invalid key state), got {other:?}"),
}
}
fn context() -> HashMap<String, String> {
HashMap::from([("bucket".to_string(), "contract".to_string())])
}
fn generate_request(key_id: &str) -> GenerateDataKeyRequest {
GenerateDataKeyRequest {
key_id: key_id.to_string(),
key_spec: KeySpec::Aes256,
encryption_context: context(),
}
}
fn encrypt_request(key_id: &str) -> EncryptRequest {
EncryptRequest {
key_id: key_id.to_string(),
plaintext: b"contract-plaintext".to_vec(),
encryption_context: context(),
grant_tokens: Vec::new(),
}
}
fn decrypt_request(ciphertext: Vec<u8>) -> DecryptRequest {
DecryptRequest {
ciphertext,
encryption_context: context(),
grant_tokens: Vec::new(),
}
}
fn schedule_request(key_id: &str) -> DeleteKeyRequest {
DeleteKeyRequest {
key_id: key_id.to_string(),
pending_window_in_days: Some(7),
force_immediate: None,
}
}
fn cancel_request(key_id: &str) -> CancelKeyDeletionRequest {
CancelKeyDeletionRequest {
key_id: key_id.to_string(),
}
}
fn create_request(key_name: String) -> CreateKeyRequest {
CreateKeyRequest {
key_name: Some(key_name),
key_usage: KeyUsage::EncryptDecrypt,
..Default::default()
}
}
async fn assert_key_state(backend: &dyn KmsBackend, key_id: &str, expected: KeyState) {
let described = backend
.describe_key(DescribeKeyRequest {
key_id: key_id.to_string(),
})
.await
.expect("describe_key must succeed for an existing key");
assert_eq!(described.key_metadata.key_state, expected, "unexpected state for key {key_id}");
}
/// Drives one freshly created (Enabled) key through the full state matrix,
/// entirely through the `KmsBackend` product surface.
async fn assert_state_machine_contract(backend: &dyn KmsBackend, key_id: &str) {
// Enabled: cryptographic use is allowed. Keep an envelope around to prove
// decryption keeps working in later states.
let data_key = backend
.generate_data_key(generate_request(key_id))
.await
.expect("Enabled key must generate data keys");
backend
.encrypt(encrypt_request(key_id))
.await
.expect("Enabled key must encrypt");
// Enabled -> Disabled.
backend.disable_key(key_id).await.expect("disable from Enabled must succeed");
assert_key_state(backend, key_id, KeyState::Disabled).await;
// Disabled: new cryptographic use and rotation are rejected...
expect_invalid_key_state(backend.encrypt(encrypt_request(key_id)).await, "disabled");
expect_invalid_key_state(backend.generate_data_key(generate_request(key_id)).await, "disabled");
expect_rotate_rejected(backend, key_id).await;
// ...but decryption of existing data keeps working (explicit AWS deviation)...
let decrypted = backend
.decrypt(decrypt_request(data_key.ciphertext_blob.clone()))
.await
.expect("decrypt with a disabled key must keep working");
assert_eq!(decrypted.plaintext, data_key.plaintext_key, "decrypt must recover the original data key");
// ...disable stays idempotent, cancel has nothing to cancel, and enable recovers.
backend.disable_key(key_id).await.expect("disable must be idempotent");
expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "not pending deletion");
backend.enable_key(key_id).await.expect("enable from Disabled must succeed");
assert_key_state(backend, key_id, KeyState::Enabled).await;
// Disabled keys may still be scheduled for deletion.
backend
.disable_key(key_id)
.await
.expect("disable before scheduling must succeed");
backend
.delete_key(schedule_request(key_id))
.await
.expect("scheduling deletion of a disabled key must succeed");
assert_key_state(backend, key_id, KeyState::PendingDeletion).await;
// PendingDeletion: everything except decryption and cancellation is rejected.
expect_invalid_key_state(backend.encrypt(encrypt_request(key_id)).await, "pending deletion");
expect_invalid_key_state(backend.generate_data_key(generate_request(key_id)).await, "pending deletion");
expect_invalid_key_state(backend.enable_key(key_id).await, "pending deletion");
expect_invalid_key_state(backend.disable_key(key_id).await, "pending deletion");
expect_rotate_rejected(backend, key_id).await;
expect_invalid_key_state(backend.delete_key(schedule_request(key_id)).await, "pending deletion");
let decrypted = backend
.decrypt(decrypt_request(data_key.ciphertext_blob.clone()))
.await
.expect("decrypt with a pending-deletion key must keep working");
assert_eq!(decrypted.plaintext, data_key.plaintext_key);
// PendingDeletion -> Enabled through cancellation.
backend
.cancel_key_deletion(cancel_request(key_id))
.await
.expect("cancel from PendingDeletion must succeed");
assert_key_state(backend, key_id, KeyState::Enabled).await;
backend
.generate_data_key(generate_request(key_id))
.await
.expect("cancelled key must be usable again");
// Cancel without a pending deletion is an invalid state transition.
expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "not pending deletion");
}
async fn local_fixture() -> (tempfile::TempDir, KmsConfig, LocalKmsBackend, String) {
let temp_dir = tempfile::tempdir().expect("temp dir should be created");
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
let backend = LocalKmsBackend::new(config.clone())
.await
.expect("local backend should build");
let created = backend
.create_key(create_request("contract-key".to_string()))
.await
.expect("key should be created");
(temp_dir, config, backend, created.key_id)
}
#[tokio::test]
async fn local_backend_state_machine_contract() {
let (_temp_dir, _config, backend, key_id) = local_fixture().await;
assert_state_machine_contract(&backend, &key_id).await;
}
/// SSE-shaped regression: disabling a key must not break decryption of data
/// keys created while it was enabled, while new data key creation must fail.
#[tokio::test]
async fn local_disabled_key_keeps_decrypting_existing_envelopes() {
let (_temp_dir, config, backend, key_id) = local_fixture().await;
let backend = Arc::new(backend);
let service = ObjectEncryptionService::new(KmsManager::new(backend.clone(), config));
let object_context = ObjectEncryptionContext::new("sse-bucket".to_string(), "dir/object.bin".to_string());
let kms_key = Some(key_id.clone());
let (_data_key, encrypted_blob) = service
.create_data_key(&kms_key, &object_context)
.await
.expect("data key creation must succeed while the key is enabled");
backend
.lifecycle_client()
.disable_key(&key_id, None)
.await
.expect("disable must succeed");
service
.decrypt_data_key(&encrypted_blob, &object_context)
.await
.expect("existing objects must stay readable after their KMS key is disabled");
expect_invalid_key_state(service.create_data_key(&kms_key, &object_context).await, "disabled");
}
/// Static is a stateless read-only backend: cryptographic operations always
/// work against the single configured key and every lifecycle mutation is
/// rejected as an invalid operation.
#[tokio::test]
async fn static_backend_stateless_contract() {
let key_id = "static-contract-key";
let mut raw_key = [0u8; 32];
rand::rng().fill(&mut raw_key[..]);
let config = KmsConfig::static_kms(key_id.to_string(), BASE64.encode(raw_key));
let static_backend = StaticKmsBackend::new(config).await.expect("static backend should build");
let backend: &dyn KmsBackend = &static_backend;
let data_key = backend
.generate_data_key(generate_request(key_id))
.await
.expect("static backend must generate data keys");
let decrypted = backend
.decrypt(decrypt_request(data_key.ciphertext_blob.clone()))
.await
.expect("static backend must decrypt its own envelopes");
assert_eq!(decrypted.plaintext, data_key.plaintext_key);
assert_key_state(backend, key_id, KeyState::Enabled).await;
expect_invalid_key_state(backend.create_key(create_request("another-key".to_string())).await, "read-only");
expect_invalid_key_state(backend.delete_key(schedule_request(key_id)).await, "read-only");
expect_invalid_key_state(backend.cancel_key_deletion(cancel_request(key_id)).await, "read-only");
// Enable/disable and rotation are capability gaps at the product
// surface, not state-machine rejections.
expect_unsupported(backend.enable_key(key_id).await);
expect_unsupported(backend.disable_key(key_id).await);
expect_unsupported(backend.rotate_key(key_id).await);
}
fn vault_dev_config(constructor: fn(url::Url, String) -> KmsConfig) -> KmsConfig {
let address = std::env::var("RUSTFS_KMS_VAULT_ADDR").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string());
let token = std::env::var("RUSTFS_KMS_VAULT_TOKEN").unwrap_or_else(|_| "dev-token".to_string());
let mut config = constructor(url::Url::parse(&address).expect("vault address should parse"), token);
config.allow_insecure_dev_defaults = true;
config
}
#[tokio::test]
#[ignore] // Requires a running Vault instance (dev mode) with a KV2 mount
async fn vault_kv2_backend_state_machine_contract() {
let config = vault_dev_config(KmsConfig::vault);
let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build");
let created = backend
.create_key(create_request(format!("contract-{}", uuid::Uuid::new_v4())))
.await
.expect("key should be created");
assert_state_machine_contract(&backend, &created.key_id).await;
// KV2 additionally supports version-retaining rotation, which must only
// work while the key is Enabled (the shared matrix covered the
// rejections).
backend
.rotate_key(&created.key_id)
.await
.expect("rotation of an Enabled KV2 key must succeed");
// Cleanup: leave the key pending deletion so repeated runs stay tidy.
let _ = backend.delete_key(schedule_request(&created.key_id)).await;
}
#[tokio::test]
#[ignore] // Requires a running Vault instance (dev mode) with the transit engine enabled
async fn vault_transit_backend_state_machine_contract() {
let config = vault_dev_config(KmsConfig::vault_transit);
let backend = VaultTransitKmsBackend::new(config)
.await
.expect("vault transit backend should build");
let created = backend
.create_key(create_request(format!("contract-{}", uuid::Uuid::new_v4())))
.await
.expect("key should be created");
assert_state_machine_contract(&backend, &created.key_id).await;
// Transit additionally supports rotation, which must only work while the
// key is Enabled (the shared matrix already covered the rejections).
backend
.rotate_key(&created.key_id)
.await
.expect("rotation of an Enabled transit key must succeed");
let _ = backend.delete_key(schedule_request(&created.key_id)).await;
}
crates/kms/src/backends/local.rs
+1526 -109
View File
File diff suppressed because it is too large Load Diff
crates/kms/src/backends/mod.rs
+377 -169
View File
@@ -14,144 +14,88 @@
//! KMS backend implementations
use crate::error::Result;
use crate::error::{KmsError, Result};
use crate::types::*;
use async_trait::async_trait;
use std::collections::HashMap;
use jiff::Zoned;
use serde::{Deserialize, Serialize};
#[cfg(test)]
mod contract_tests;
pub mod local;
#[cfg(test)]
pub(crate) mod scripted_vault;
pub mod static_kms;
pub mod vault;
pub(crate) mod vault_credentials;
pub mod vault_transit;
/// Abstract KMS client interface that all backends must implement
#[async_trait]
pub trait KmsClient: Send + Sync {
/// Generate a new data encryption key (DEK)
///
/// Creates a new data key using the specified master key. The returned DataKey
/// contains both the plaintext and encrypted versions of the key.
///
/// # Arguments
/// * `request` - The key generation request
/// * `context` - Optional operation context for auditing
///
/// # Returns
/// Returns a DataKey containing both plaintext and encrypted key material
async fn generate_data_key(&self, request: &GenerateKeyRequest, context: Option<&OperationContext>) -> Result<DataKeyInfo>;
/// Operations whose availability depends on the key's lifecycle state.
///
/// Decryption is deliberately absent: RustFS allows decryption with
/// `Disabled` and `PendingDeletion` keys — an explicit deviation from AWS
/// KMS — because rejecting it would break reads of every object encrypted
/// under a key the moment it is disabled. Deletion cancellation is also
/// absent: it is valid exactly when the key is `PendingDeletion`, which call
/// sites enforce directly.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum StateGatedOperation {
Encrypt,
GenerateDataKey,
Rotate,
Enable,
Disable,
ScheduleDeletion,
}
/// Encrypt data directly using a master key
///
/// Encrypts the provided plaintext using the specified master key.
/// This is different from generate_data_key as it encrypts user data directly.
///
/// # Arguments
/// * `request` - The encryption request containing plaintext and key ID
/// * `context` - Optional operation context for auditing
async fn encrypt(&self, request: &EncryptRequest, context: Option<&OperationContext>) -> Result<EncryptResponse>;
impl StateGatedOperation {
fn describe(self) -> &'static str {
match self {
Self::Encrypt => "encryption",
Self::GenerateDataKey => "data key generation",
Self::Rotate => "rotation",
Self::Enable => "enabling",
Self::Disable => "disabling",
Self::ScheduleDeletion => "deletion scheduling",
}
}
}
/// Decrypt data using a master key
///
/// Decrypts the provided ciphertext. The KMS automatically determines
/// which key was used for encryption based on the ciphertext metadata.
///
/// # Arguments
/// * `request` - The decryption request containing ciphertext
/// * `context` - Optional operation context for auditing
async fn decrypt(&self, request: &DecryptRequest, context: Option<&OperationContext>) -> Result<Vec<u8>>;
/// Enforce the shared key state × operation matrix.
///
/// - `Enabled`: every operation is allowed.
/// - `Disabled`: enabling, disabling (idempotent) and deletion scheduling are
/// allowed; encryption, data key generation and rotation are rejected.
/// - `PendingDeletion`: every state-gated operation is rejected, including a
/// repeated deletion schedule; only cancellation and decryption proceed.
/// - `PendingImport`/`Unavailable`: the key is not usable and is reported as
/// not found.
pub(crate) fn ensure_key_state_permits(key_id: &str, state: &KeyState, operation: StateGatedOperation) -> Result<()> {
match state {
KeyState::Enabled => Ok(()),
KeyState::Disabled => match operation {
StateGatedOperation::Enable | StateGatedOperation::Disable | StateGatedOperation::ScheduleDeletion => Ok(()),
StateGatedOperation::Encrypt | StateGatedOperation::GenerateDataKey | StateGatedOperation::Rotate => Err(
KmsError::invalid_key_state(format!("Key {key_id} is disabled: {} is not allowed", operation.describe())),
),
},
KeyState::PendingDeletion => Err(KmsError::invalid_key_state(format!(
"Key {key_id} is pending deletion: {} is not allowed",
operation.describe()
))),
KeyState::PendingImport | KeyState::Unavailable => Err(KmsError::key_not_found(key_id)),
}
}
/// Create a new master key
///
/// Creates a new master key in the KMS with the specified ID.
/// Returns an error if a key with the same ID already exists.
///
/// # Arguments
/// * `key_id` - Unique identifier for the new key
/// * `algorithm` - Key algorithm (e.g., "AES_256")
/// * `context` - Optional operation context for auditing
async fn create_key(&self, key_id: &str, algorithm: &str, context: Option<&OperationContext>) -> Result<MasterKeyInfo>;
/// Get information about a specific key
///
/// Returns metadata and information about the specified key.
///
/// # Arguments
/// * `key_id` - The key identifier
/// * `context` - Optional operation context for auditing
async fn describe_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result<KeyInfo>;
/// List available keys
///
/// Returns a paginated list of keys available in the KMS.
///
/// # Arguments
/// * `request` - List request parameters (pagination, filters)
/// * `context` - Optional operation context for auditing
async fn list_keys(&self, request: &ListKeysRequest, context: Option<&OperationContext>) -> Result<ListKeysResponse>;
/// Enable a key
///
/// Enables a previously disabled key, allowing it to be used for cryptographic operations.
///
/// # Arguments
/// * `key_id` - The key identifier
/// * `context` - Optional operation context for auditing
async fn enable_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result<()>;
/// Disable a key
///
/// Disables a key, preventing it from being used for new cryptographic operations.
/// Existing encrypted data can still be decrypted.
///
/// # Arguments
/// * `key_id` - The key identifier
/// * `context` - Optional operation context for auditing
async fn disable_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result<()>;
/// Schedule key deletion
///
/// Schedules a key for deletion after a specified number of days.
/// This allows for a grace period to recover the key if needed.
///
/// # Arguments
/// * `key_id` - The key identifier
/// * `pending_window_days` - Number of days before actual deletion
/// * `context` - Optional operation context for auditing
async fn schedule_key_deletion(
&self,
key_id: &str,
pending_window_days: u32,
context: Option<&OperationContext>,
) -> Result<()>;
/// Cancel key deletion
///
/// Cancels a previously scheduled key deletion.
///
/// # Arguments
/// * `key_id` - The key identifier
/// * `context` - Optional operation context for auditing
async fn cancel_key_deletion(&self, key_id: &str, context: Option<&OperationContext>) -> Result<()>;
/// Rotate a key
///
/// Creates a new version of the specified key. Previous versions remain
/// available for decryption but new operations will use the new version.
///
/// # Arguments
/// * `key_id` - The key identifier
/// * `context` - Optional operation context for auditing
async fn rotate_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result<MasterKeyInfo>;
/// Health check
///
/// Performs a health check on the KMS backend to ensure it's operational.
async fn health_check(&self) -> Result<()>;
/// Get backend information
///
/// Returns information about the KMS backend (type, version, etc.).
fn backend_info(&self) -> BackendInfo;
/// [`ensure_key_state_permits`] for backends that persist [`KeyStatus`].
pub(crate) fn ensure_key_status_permits(key_id: &str, status: &KeyStatus, operation: StateGatedOperation) -> Result<()> {
let state = match status {
KeyStatus::Active => KeyState::Enabled,
KeyStatus::Disabled => KeyState::Disabled,
KeyStatus::PendingDeletion => KeyState::PendingDeletion,
KeyStatus::Deleted => KeyState::Unavailable,
};
ensure_key_state_permits(key_id, &state, operation)
}
/// Simplified KMS backend interface for manager
@@ -181,58 +125,322 @@ pub trait KmsBackend: Send + Sync {
/// Cancel key deletion
async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse>;
/// Enable a disabled key so it can be used for cryptographic operations
/// again.
///
/// Backends that advertise [`BackendCapabilities::enable_disable`] must
/// override this method; the default rejects the operation.
async fn enable_key(&self, _key_id: &str) -> Result<()> {
Err(KmsError::unsupported_capability("backend without enable/disable support", "enable_key"))
}
/// Disable a key, rejecting new cryptographic use while existing data
/// remains decryptable.
///
/// Backends that advertise [`BackendCapabilities::enable_disable`] must
/// override this method; the default rejects the operation.
async fn disable_key(&self, _key_id: &str) -> Result<()> {
Err(KmsError::unsupported_capability("backend without enable/disable support", "disable_key"))
}
/// Rotate a key to a new version while prior versions remain available
/// for decryption.
///
/// Only backends that advertise [`BackendCapabilities::rotate`] (that is,
/// backends with retained version history) may override this method; the
/// default rejects the operation.
async fn rotate_key(&self, _key_id: &str) -> Result<()> {
Err(KmsError::unsupported_capability("backend without rotation support", "rotate_key"))
}
/// Health check
async fn health_check(&self) -> Result<bool>;
/// Report which operations this backend actually supports.
///
/// The default is conservative: only the operations every backend is
/// required to implement by this trait are advertised. Optional lifecycle
/// operations (rotation, enable/disable, deletion scheduling, ...) must be
/// opted in by overriding this method.
fn capabilities(&self) -> BackendCapabilities {
BackendCapabilities::minimal()
}
/// Remove a key whose scheduled deletion deadline has passed.
///
/// Used by the background deletion worker. Implementations must re-check
/// state and deadline under their own write synchronization so that a
/// concurrent cancellation observed after the caller's inspection wins
/// ([`ExpiredKeyRemoval::StateChanged`]), must write a tombstone (a
/// `Deleted`/`Unavailable` record) before destroying material so a crashed
/// removal can simply be re-run, and must treat an already-removed key as
/// success so the operation stays idempotent across restarts and nodes.
///
/// The default rejects the operation for backends without deletion
/// support.
async fn remove_expired_key(&self, _key_id: &str, _now: &Zoned) -> Result<ExpiredKeyRemoval> {
Err(KmsError::unsupported_capability("backend without deletion support", "remove_expired_key"))
}
}
/// Information about a KMS backend
#[derive(Debug, Clone)]
pub struct BackendInfo {
/// Backend type name (e.g., "local", "vault")
pub backend_type: String,
/// Backend version
pub version: String,
/// Backend endpoint or location
pub endpoint: String,
/// Whether the backend is currently healthy
pub healthy: bool,
/// Additional metadata about the backend
pub metadata: HashMap<String, String>,
/// Outcome of [`KmsBackend::remove_expired_key`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ExpiredKeyRemoval {
/// The key's record and material were removed, or were already gone.
Removed,
/// The key is no longer pending deletion (for example the deletion was
/// cancelled after the caller inspected it); nothing was removed.
StateChanged,
/// The key is pending deletion but its deadline has not passed, or it has
/// no persisted deadline (legacy record) and is never auto-removed.
NotExpired,
}
impl BackendInfo {
/// Create a new backend info
///
/// # Arguments
/// * `backend_type` - The type of the backend
/// * `version` - The version of the backend
/// * `endpoint` - The endpoint or location of the backend
/// * `healthy` - Whether the backend is healthy
///
/// # Returns
/// A new BackendInfo instance
///
pub fn new(backend_type: String, version: String, endpoint: String, healthy: bool) -> Self {
/// Set of operations a KMS backend supports.
///
/// Reported by [`KmsBackend::capabilities`] so callers (manager, admin API)
/// can discover what the active backend can do without probing individual
/// operations. Marked `#[non_exhaustive]` so new capability flags can be
/// added without breaking downstream code; construct values through
/// [`BackendCapabilities::minimal`] and the `with_*` builders.
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct BackendCapabilities {
/// Direct encryption of caller-provided plaintext with a master key
pub encrypt: bool,
/// Decryption of previously produced ciphertext
pub decrypt: bool,
/// Data encryption key (DEK) generation
pub generate_data_key: bool,
/// Key rotation that retains prior versions for decryption
pub rotate: bool,
/// Enabling and disabling keys
pub enable_disable: bool,
/// Scheduling key deletion with a pending window
pub schedule_deletion: bool,
/// Multiple key versions addressable after rotation
pub versioning: bool,
/// Irreversible physical deletion of key material
pub physical_delete: bool,
}
impl BackendCapabilities {
/// Conservative baseline: only the operations that every [`KmsBackend`]
/// implementation is required to provide by the trait. All optional
/// lifecycle capabilities default to unsupported.
pub const fn minimal() -> Self {
Self {
backend_type,
version,
endpoint,
healthy,
metadata: HashMap::new(),
encrypt: true,
decrypt: true,
generate_data_key: true,
rotate: false,
enable_disable: false,
schedule_deletion: false,
versioning: false,
physical_delete: false,
}
}
/// Add metadata to the backend info
///
/// # Arguments
/// * `key` - Metadata key
/// * `value` - Metadata value
///
/// # Returns
/// Updated BackendInfo instance
///
pub fn with_metadata(mut self, key: String, value: String) -> Self {
self.metadata.insert(key, value);
/// Set whether direct encryption is supported
pub const fn with_encrypt(mut self, encrypt: bool) -> Self {
self.encrypt = encrypt;
self
}
/// Set whether decryption is supported
pub const fn with_decrypt(mut self, decrypt: bool) -> Self {
self.decrypt = decrypt;
self
}
/// Set whether data key generation is supported
pub const fn with_generate_data_key(mut self, generate_data_key: bool) -> Self {
self.generate_data_key = generate_data_key;
self
}
/// Set whether version-retaining key rotation is supported
pub const fn with_rotate(mut self, rotate: bool) -> Self {
self.rotate = rotate;
self
}
/// Set whether enabling/disabling keys is supported
pub const fn with_enable_disable(mut self, enable_disable: bool) -> Self {
self.enable_disable = enable_disable;
self
}
/// Set whether scheduled deletion with a pending window is supported
pub const fn with_schedule_deletion(mut self, schedule_deletion: bool) -> Self {
self.schedule_deletion = schedule_deletion;
self
}
/// Set whether multiple key versions are supported
pub const fn with_versioning(mut self, versioning: bool) -> Self {
self.versioning = versioning;
self
}
/// Set whether physical deletion of key material is supported
pub const fn with_physical_delete(mut self, physical_delete: bool) -> Self {
self.physical_delete = physical_delete;
self
}
}
impl Default for BackendCapabilities {
fn default() -> Self {
Self::minimal()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::KmsConfig;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
/// Backend that implements only the trait-mandated operations and relies
/// on the default `capabilities` implementation.
struct MinimalBackend;
#[async_trait]
impl KmsBackend for MinimalBackend {
async fn create_key(&self, _request: CreateKeyRequest) -> Result<CreateKeyResponse> {
unimplemented!("not exercised by capability tests")
}
async fn encrypt(&self, _request: EncryptRequest) -> Result<EncryptResponse> {
unimplemented!("not exercised by capability tests")
}
async fn decrypt(&self, _request: DecryptRequest) -> Result<DecryptResponse> {
unimplemented!("not exercised by capability tests")
}
async fn generate_data_key(&self, _request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
unimplemented!("not exercised by capability tests")
}
async fn describe_key(&self, _request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
unimplemented!("not exercised by capability tests")
}
async fn list_keys(&self, _request: ListKeysRequest) -> Result<ListKeysResponse> {
unimplemented!("not exercised by capability tests")
}
async fn delete_key(&self, _request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
unimplemented!("not exercised by capability tests")
}
async fn cancel_key_deletion(&self, _request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
unimplemented!("not exercised by capability tests")
}
async fn health_check(&self) -> Result<bool> {
Ok(true)
}
}
fn capabilities_snapshot(capabilities: BackendCapabilities) -> std::collections::BTreeMap<String, bool> {
serde_json::from_value(serde_json::to_value(capabilities).expect("capabilities should serialize"))
.expect("capabilities should deserialize into a flat bool map")
}
#[test]
fn default_capabilities_are_conservative() {
let capabilities = MinimalBackend.capabilities();
assert_eq!(capabilities, BackendCapabilities::minimal());
assert_eq!(capabilities, BackendCapabilities::default());
// The conservative baseline advertises only trait-mandated operations.
assert!(capabilities.encrypt);
assert!(capabilities.decrypt);
assert!(capabilities.generate_data_key);
assert!(!capabilities.rotate);
assert!(!capabilities.enable_disable);
assert!(!capabilities.schedule_deletion);
assert!(!capabilities.versioning);
assert!(!capabilities.physical_delete);
}
#[tokio::test]
async fn default_lifecycle_operations_are_unsupported() {
for (operation, result) in [
("enable_key", MinimalBackend.enable_key("any-key").await),
("disable_key", MinimalBackend.disable_key("any-key").await),
("rotate_key", MinimalBackend.rotate_key("any-key").await),
] {
let error = result.expect_err("backends must opt in to lifecycle operations by overriding them");
assert!(
matches!(error, KmsError::UnsupportedCapability { .. }),
"expected UnsupportedCapability for {operation}, got {error:?}"
);
}
}
#[tokio::test]
async fn default_remove_expired_key_is_unsupported() {
let error = MinimalBackend
.remove_expired_key("any-key", &jiff::Zoned::now())
.await
.expect_err("backends without deletion support must reject expired-key removal");
assert!(
matches!(error, KmsError::UnsupportedCapability { .. }),
"expected UnsupportedCapability, got {error:?}"
);
}
#[tokio::test]
async fn local_backend_capabilities_golden() {
let temp_dir = tempfile::tempdir().expect("temp dir should be created");
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
let backend = local::LocalKmsBackend::new(config).await.expect("local backend should build");
insta::assert_json_snapshot!("local_backend_capabilities", capabilities_snapshot(backend.capabilities()));
}
#[tokio::test]
async fn vault_kv2_backend_capabilities_golden() {
let config = KmsConfig::vault(
url::Url::parse("http://127.0.0.1:8200").expect("vault URL should parse"),
"dev-token".to_string(),
)
.with_insecure_development_defaults();
// Constructing the client performs no network I/O with token auth.
let backend = vault::VaultKmsBackend::new(config)
.await
.expect("vault kv2 backend should build");
insta::assert_json_snapshot!("vault_kv2_backend_capabilities", capabilities_snapshot(backend.capabilities()));
}
#[tokio::test]
async fn vault_transit_backend_capabilities_golden() {
let config = KmsConfig::vault_transit(
url::Url::parse("http://127.0.0.1:8200").expect("vault URL should parse"),
"dev-token".to_string(),
)
.with_insecure_development_defaults();
// Constructing the client performs no network I/O with token auth.
let backend = vault_transit::VaultTransitKmsBackend::new(config)
.await
.expect("vault transit backend should build");
insta::assert_json_snapshot!("vault_transit_backend_capabilities", capabilities_snapshot(backend.capabilities()));
}
#[tokio::test]
async fn static_backend_capabilities_golden() {
let config = KmsConfig::static_kms("static-key".to_string(), BASE64.encode([0u8; 32]));
let backend = static_kms::StaticKmsBackend::new(config)
.await
.expect("static backend should build");
insta::assert_json_snapshot!("static_backend_capabilities", capabilities_snapshot(backend.capabilities()));
}
}
crates/kms/src/backends/scripted_vault.rs
+177
View File
@@ -0,0 +1,177 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Minimal scripted HTTP responder standing in for a Vault server.
//!
//! Wiring tests need to observe how many Vault requests a code path performs
//! (retries, read-confirm recovery) without a live Vault. The responder serves
//! one canned response per incoming request in order, closes the connection
//! after each response, and records the `METHOD /path` sequence for
//! assertions. It intentionally implements just enough HTTP/1.1 for the
//! `vaultrs` reqwest client: no keep-alive, no chunked bodies.
use std::sync::{Arc, Mutex};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
/// One canned HTTP response.
pub(crate) struct ScriptedResponse {
status: u16,
body: String,
}
impl ScriptedResponse {
/// A 200 response carrying `data` inside the standard Vault envelope.
pub(crate) fn ok(data: serde_json::Value) -> Self {
Self {
status: 200,
body: serde_json::json!({
"request_id": "scripted",
"lease_id": "",
"lease_duration": 0,
"renewable": false,
"data": data,
})
.to_string(),
}
}
/// An error response in Vault's `{"errors": [...]}` format.
pub(crate) fn error(status: u16, message: &str) -> Self {
Self {
status,
body: serde_json::json!({ "errors": [message] }).to_string(),
}
}
}
/// A scripted stand-in Vault listening on a loopback port.
pub(crate) struct ScriptedVault {
/// Base address (`http://127.0.0.1:port`) to point a Vault client at.
pub(crate) address: String,
requests: Arc<Mutex<Vec<(String, String)>>>,
}
impl ScriptedVault {
/// Bind a loopback listener and serve `responses` one per request.
///
/// Requests beyond the script get a 599 error so a test that under-scripts
/// fails loudly instead of hanging.
pub(crate) async fn serve(responses: Vec<ScriptedResponse>) -> Self {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind scripted vault listener");
let address = format!("http://{}", listener.local_addr().expect("scripted vault local addr"));
let requests = Arc::new(Mutex::new(Vec::new()));
let recorded = Arc::clone(&requests);
tokio::spawn(async move {
let mut responses = responses.into_iter();
loop {
let Ok((mut stream, _)) = listener.accept().await else {
return;
};
let Some(request) = read_request(&mut stream).await else {
continue;
};
recorded.lock().expect("scripted vault request log poisoned").push(request);
let response = responses
.next()
.unwrap_or_else(|| ScriptedResponse::error(599, "scripted vault: script exhausted"));
let payload = format!(
"HTTP/1.1 {} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
response.status,
response.body.len(),
response.body
);
let _ = stream.write_all(payload.as_bytes()).await;
let _ = stream.shutdown().await;
}
});
Self { address, requests }
}
/// The `METHOD /path` lines of every request served so far, in order.
pub(crate) fn requests(&self) -> Vec<String> {
self.requests
.lock()
.expect("scripted vault request log poisoned")
.iter()
.map(|(line, _)| line.clone())
.collect()
}
/// The request bodies, in the same order as [`Self::requests`]; empty for
/// bodyless requests. Lets tests assert what a write actually persisted
/// (record contents, check-and-set options), not just that a write happened.
pub(crate) fn request_bodies(&self) -> Vec<String> {
self.requests
.lock()
.expect("scripted vault request log poisoned")
.iter()
.map(|(_, body)| body.clone())
.collect()
}
}
/// Read one HTTP/1.1 request (head plus content-length body) and return its
/// `METHOD /path` line together with the body. Draining the body before
/// responding keeps the client from seeing a connection reset while it is
/// still writing.
async fn read_request(stream: &mut TcpStream) -> Option<(String, String)> {
let mut buffer = Vec::new();
let mut chunk = [0u8; 4096];
let head_end = loop {
if let Some(position) = buffer.windows(4).position(|window| window == b"\r\n\r\n") {
break position + 4;
}
let read = stream.read(&mut chunk).await.ok()?;
if read == 0 {
return None;
}
buffer.extend_from_slice(&chunk[..read]);
};
let head = String::from_utf8_lossy(&buffer[..head_end]).into_owned();
let mut lines = head.lines();
let request_line = lines.next()?;
let mut parts = request_line.split_whitespace();
let method = parts.next()?;
let path = parts.next()?;
// rustify appends a lone "?" when an endpoint has no query parameters;
// strip it so assertions can use the plain path.
let path = path.strip_suffix('?').unwrap_or(path);
let content_length: usize = lines
.filter_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse().ok())?
})
.next()
.unwrap_or(0);
let mut body = buffer[head_end..].to_vec();
let mut remaining = content_length.saturating_sub(body.len());
while remaining > 0 {
let read = stream.read(&mut chunk).await.ok()?;
if read == 0 {
break;
}
body.extend_from_slice(&chunk[..read]);
remaining = remaining.saturating_sub(read);
}
body.truncate(content_length);
Some((format!("{method} {path}"), String::from_utf8_lossy(&body).into_owned()))
}
crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__local_backend_capabilities.snap
+14
View File
@@ -0,0 +1,14 @@
---
source: crates/kms/src/backends/mod.rs
expression: capabilities_snapshot(backend.capabilities())
---
{
"decrypt": true,
"enable_disable": true,
"encrypt": true,
"generate_data_key": true,
"physical_delete": true,
"rotate": false,
"schedule_deletion": true,
"versioning": false
}
crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__static_backend_capabilities.snap
+14
View File
@@ -0,0 +1,14 @@
---
source: crates/kms/src/backends/mod.rs
expression: capabilities_snapshot(backend.capabilities())
---
{
"decrypt": true,
"enable_disable": false,
"encrypt": true,
"generate_data_key": true,
"physical_delete": false,
"rotate": false,
"schedule_deletion": false,
"versioning": false
}
crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap
+14
View File
@@ -0,0 +1,14 @@
---
source: crates/kms/src/backends/mod.rs
expression: capabilities_snapshot(backend.capabilities())
---
{
"decrypt": true,
"enable_disable": true,
"encrypt": true,
"generate_data_key": true,
"physical_delete": true,
"rotate": true,
"schedule_deletion": true,
"versioning": true
}
crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_transit_backend_capabilities.snap
+14
View File
@@ -0,0 +1,14 @@
---
source: crates/kms/src/backends/mod.rs
expression: capabilities_snapshot(backend.capabilities())
---
{
"decrypt": true,
"enable_disable": true,
"encrypt": true,
"generate_data_key": true,
"physical_delete": true,
"rotate": true,
"schedule_deletion": true,
"versioning": true
}
crates/kms/src/backends/static_kms.rs
+87 -142
View File
@@ -21,7 +21,7 @@
//!
//! encrypted_data(plaintext_len+16) || nonce (12 bytes)
use crate::backends::{BackendInfo, KmsBackend, KmsClient};
use crate::backends::{BackendCapabilities, KmsBackend};
use crate::config::{BackendConfig, KmsConfig};
use crate::encryption::DataKeyEnvelope;
use crate::error::{KmsError, Result};
@@ -98,9 +98,10 @@ impl StaticKmsBackend {
}
}
#[async_trait]
impl KmsClient for StaticKmsBackend {
async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result<DataKeyInfo> {
impl StaticKmsBackend {
/// Generate a fresh data key and wrap it in the standard KMS envelope,
/// authenticated against the canonical encryption context.
pub(crate) fn generate_data_key_envelope(&self, request: &GenerateKeyRequest) -> Result<DataKeyInfo> {
if request.master_key_id != self.key_id {
return Err(KmsError::key_not_found(&request.master_key_id));
}
@@ -137,6 +138,8 @@ impl KmsClient for StaticKmsBackend {
nonce: nonce_bytes.to_vec(),
encryption_context: request.encryption_context.clone(),
created_at: Zoned::now(),
// The static backend has a single fixed key with no rotation.
master_key_version: None,
};
let ciphertext = serde_json::to_vec(&envelope)?;
@@ -149,7 +152,8 @@ impl KmsClient for StaticKmsBackend {
))
}
async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result<EncryptResponse> {
/// Encrypt caller-provided plaintext into the standard KMS envelope.
pub(crate) fn encrypt_to_envelope(&self, request: &EncryptRequest) -> Result<EncryptResponse> {
if request.key_id != self.key_id {
return Err(KmsError::key_not_found(&request.key_id));
}
@@ -181,6 +185,8 @@ impl KmsClient for StaticKmsBackend {
nonce: nonce_bytes.to_vec(),
encryption_context: request.encryption_context.clone(),
created_at: Zoned::now(),
// The static backend has a single fixed key with no rotation.
master_key_version: None,
};
let ciphertext = serde_json::to_vec(&envelope)?;
@@ -192,7 +198,8 @@ impl KmsClient for StaticKmsBackend {
})
}
async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> {
/// Open a KMS envelope produced by this backend.
pub(crate) fn decrypt_envelope(&self, request: &DecryptRequest) -> Result<Vec<u8>> {
let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)
.map_err(|error| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {error}")))?;
if envelope.master_key_id != self.key_id {
@@ -231,14 +238,8 @@ impl KmsClient for StaticKmsBackend {
Ok(plaintext)
}
async fn create_key(&self, key_id: &str, _algorithm: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
if key_id == self.key_id {
return Err(KmsError::key_already_exists(key_id));
}
Err(KmsError::invalid_operation("Static KMS is read-only: cannot create new keys"))
}
async fn describe_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<KeyInfo> {
/// Describe the single configured key.
pub(crate) fn configured_key_info(&self, key_id: &str) -> Result<KeyInfo> {
if key_id != self.key_id {
return Err(KmsError::key_not_found(key_id));
}
@@ -259,7 +260,8 @@ impl KmsClient for StaticKmsBackend {
})
}
async fn list_keys(&self, request: &ListKeysRequest, _context: Option<&OperationContext>) -> Result<ListKeysResponse> {
/// List the single configured key, honouring the pagination marker.
pub(crate) fn list_configured_key(&self, request: &ListKeysRequest) -> Result<ListKeysResponse> {
let key_info = KeyInfo {
key_id: self.key_id.clone(),
description: Some("Static single-key KMS backend".to_string()),
@@ -291,57 +293,6 @@ impl KmsClient for StaticKmsBackend {
truncated: false,
})
}
async fn enable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> {
if key_id != self.key_id {
return Err(KmsError::key_not_found(key_id));
}
// Static KMS key is always enabled
Ok(())
}
async fn disable_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> {
if key_id != self.key_id {
return Err(KmsError::key_not_found(key_id));
}
Err(KmsError::invalid_operation("Static KMS is read-only: cannot disable keys"))
}
async fn schedule_key_deletion(
&self,
key_id: &str,
_pending_window_days: u32,
_context: Option<&OperationContext>,
) -> Result<()> {
if key_id != self.key_id {
return Err(KmsError::key_not_found(key_id));
}
Err(KmsError::invalid_operation("Static KMS is read-only: cannot schedule key deletion"))
}
async fn cancel_key_deletion(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<()> {
if key_id != self.key_id {
return Err(KmsError::key_not_found(key_id));
}
Err(KmsError::invalid_operation("Static KMS is read-only: cannot cancel key deletion"))
}
async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
if key_id != self.key_id {
return Err(KmsError::key_not_found(key_id));
}
Err(KmsError::invalid_operation("Static KMS is read-only: cannot rotate keys"))
}
async fn health_check(&self) -> Result<()> {
// Static KMS is always healthy if it was successfully initialized
Ok(())
}
fn backend_info(&self) -> BackendInfo {
BackendInfo::new("static".to_string(), env!("CARGO_PKG_VERSION").to_string(), "local".to_string(), true)
.with_metadata("key_id".to_string(), self.key_id.clone())
}
}
#[async_trait]
@@ -355,12 +306,12 @@ impl KmsBackend for StaticKmsBackend {
}
async fn encrypt(&self, request: EncryptRequest) -> Result<EncryptResponse> {
<Self as KmsClient>::encrypt(self, &request, None).await
self.encrypt_to_envelope(&request)
}
async fn decrypt(&self, request: DecryptRequest) -> Result<DecryptResponse> {
let key_id = self.key_id.clone();
let plaintext = <Self as KmsClient>::decrypt(self, &request, None).await?;
let plaintext = self.decrypt_envelope(&request)?;
Ok(DecryptResponse {
plaintext,
key_id,
@@ -376,7 +327,7 @@ impl KmsBackend for StaticKmsBackend {
encryption_context: request.encryption_context,
grant_tokens: Vec::new(),
};
let data_key = <Self as KmsClient>::generate_data_key(self, &gen_req, None).await?;
let data_key = self.generate_data_key_envelope(&gen_req)?;
let plaintext_key = data_key
.plaintext
@@ -391,7 +342,7 @@ impl KmsBackend for StaticKmsBackend {
}
async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
let key_info = <Self as KmsClient>::describe_key(self, &request.key_id, None).await?;
let key_info = self.configured_key_info(&request.key_id)?;
let key_metadata = KeyMetadata {
key_id: key_info.key_id.clone(),
key_state: if key_info.status == KeyStatus::Active {
@@ -411,7 +362,7 @@ impl KmsBackend for StaticKmsBackend {
}
async fn list_keys(&self, request: ListKeysRequest) -> Result<ListKeysResponse> {
<Self as KmsClient>::list_keys(self, &request, None).await
self.list_configured_key(&request)
}
async fn delete_key(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
@@ -431,12 +382,18 @@ impl KmsBackend for StaticKmsBackend {
async fn health_check(&self) -> Result<bool> {
Ok(true)
}
fn capabilities(&self) -> BackendCapabilities {
// Static KMS is a read-only single-key backend: it only performs
// cryptographic operations and rejects every lifecycle mutation.
BackendCapabilities::minimal()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::backends::{KmsBackend as KmsBackendTrait, KmsClient};
use crate::backends::KmsBackend as KmsBackendTrait;
use crate::config::{BackendConfig, KmsBackend, StaticConfig};
use crate::encryption::is_data_key_envelope;
use base64::Engine as _;
@@ -481,8 +438,8 @@ mod tests {
// Generate data key
let request = GenerateKeyRequest::new(key_id.clone(), "AES_256".to_string())
.with_context("bucket".to_string(), "test-bucket".to_string());
let data_key = KmsClient::generate_data_key(&backend, &request, None)
.await
let data_key = backend
.generate_data_key_envelope(&request)
.expect("Failed to generate data key");
assert_eq!(data_key.key_id, key_id);
@@ -499,9 +456,7 @@ mod tests {
// Decrypt the data key
let decrypt_request =
DecryptRequest::new(data_key.ciphertext.clone()).with_context("bucket".to_string(), "test-bucket".to_string());
let decrypted = KmsClient::decrypt(&backend, &decrypt_request, None)
.await
.expect("Failed to decrypt");
let decrypted = backend.decrypt_envelope(&decrypt_request).expect("Failed to decrypt");
assert_eq!(decrypted.as_slice(), data_key.plaintext.as_deref().expect("plaintext should exist"));
}
@@ -513,8 +468,8 @@ mod tests {
.with_context("bucket".to_string(), "source-bucket".to_string())
.with_context("object".to_string(), "source-object".to_string());
let data_key = KmsClient::generate_data_key(&backend, &request, None)
.await
let data_key = backend
.generate_data_key_envelope(&request)
.expect("generate static KMS data key");
assert!(
@@ -558,8 +513,8 @@ mod tests {
let (backend, key_id, _key) = create_test_backend().await;
let request = GenerateKeyRequest::new(key_id, "AES_256".to_string())
.with_context("bucket".to_string(), "source-bucket".to_string());
let generated = KmsClient::generate_data_key(&backend, &request, None)
.await
let generated = backend
.generate_data_key_envelope(&request)
.expect("generate context-bound data key");
let mut envelope: DataKeyEnvelope = serde_json::from_slice(&generated.ciphertext).expect("parse static KMS envelope");
envelope
@@ -568,8 +523,8 @@ mod tests {
let decrypt_request = DecryptRequest::new(serde_json::to_vec(&envelope).expect("serialize tampered envelope"))
.with_context("bucket".to_string(), "different-bucket".to_string());
let error = KmsClient::decrypt(&backend, &decrypt_request, None)
.await
let error = backend
.decrypt_envelope(&decrypt_request)
.expect_err("tampering with authenticated envelope context must fail");
assert!(matches!(error, KmsError::CryptographicError { .. }));
@@ -580,7 +535,7 @@ mod tests {
let (backend, _key_id, _key) = create_test_backend().await;
let request = GenerateKeyRequest::new("wrong-key-id".to_string(), "AES_256".to_string());
let result = KmsClient::generate_data_key(&backend, &request, None).await;
let result = backend.generate_data_key_envelope(&request);
assert!(result.is_err());
assert!(result.expect_err("should be Err").to_string().contains("wrong-key-id"));
}
@@ -592,7 +547,7 @@ mod tests {
// Ciphertext too short
let short = vec![0u8; 10];
let request = DecryptRequest::new(short);
let result = KmsClient::decrypt(&backend, &request, None).await;
let result = backend.decrypt_envelope(&request);
assert!(result.is_err());
}
@@ -602,9 +557,7 @@ mod tests {
// Generate a valid ciphertext first
let gen_request = GenerateKeyRequest::new(key_id, "AES_256".to_string());
let data_key = KmsClient::generate_data_key(&backend, &gen_request, None)
.await
.expect("generate");
let data_key = backend.generate_data_key_envelope(&gen_request).expect("generate");
// Tamper with the ciphertext (flip a bit in the encrypted portion)
let mut tampered = data_key.ciphertext.clone();
@@ -613,7 +566,7 @@ mod tests {
}
let request = DecryptRequest::new(tampered);
let result = KmsClient::decrypt(&backend, &request, None).await;
let result = backend.decrypt_envelope(&request);
assert!(result.is_err());
}
@@ -622,7 +575,14 @@ mod tests {
let (backend, key_id, _key) = create_test_backend().await;
// Creating the pre-configured key should return KeyAlreadyExists
let result = KmsClient::create_key(&backend, &key_id, "AES_256", None).await;
let result = KmsBackendTrait::create_key(
&backend,
CreateKeyRequest {
key_name: Some(key_id.clone()),
..Default::default()
},
)
.await;
assert!(result.is_err());
assert!(result.expect_err("should be Err").to_string().contains("already exists"));
}
@@ -632,7 +592,14 @@ mod tests {
let (backend, _key_id, _key) = create_test_backend().await;
// Creating any other key should return invalid operation (read-only)
let result = KmsClient::create_key(&backend, "other-key", "AES_256", None).await;
let result = KmsBackendTrait::create_key(
&backend,
CreateKeyRequest {
key_name: Some("other-key".to_string()),
..Default::default()
},
)
.await;
assert!(result.is_err());
let err_msg = result.expect_err("should be Err").to_string();
assert!(err_msg.contains("read-only") || err_msg.contains("cannot create"));
@@ -642,15 +609,13 @@ mod tests {
async fn test_describe_key() {
let (backend, key_id, _key) = create_test_backend().await;
let key_info = KmsClient::describe_key(&backend, &key_id, None)
.await
.expect("describe_key should succeed");
let key_info = backend.configured_key_info(&key_id).expect("describe_key should succeed");
assert_eq!(key_info.key_id, key_id);
assert_eq!(key_info.status, KeyStatus::Active);
assert_eq!(key_info.algorithm, "AES_256");
// Wrong key ID
let result = KmsClient::describe_key(&backend, "nonexistent", None).await;
let result = backend.configured_key_info("nonexistent");
assert!(result.is_err());
}
@@ -658,8 +623,8 @@ mod tests {
async fn test_list_keys() {
let (backend, key_id, _key) = create_test_backend().await;
let response = KmsClient::list_keys(&backend, &ListKeysRequest::default(), None)
.await
let response = backend
.list_configured_key(&ListKeysRequest::default())
.expect("list_keys should succeed");
assert_eq!(response.keys.len(), 1);
assert_eq!(response.keys[0].key_id, key_id);
@@ -667,61 +632,45 @@ mod tests {
}
#[tokio::test]
async fn test_disable_key_returns_error() {
async fn lifecycle_mutations_are_unsupported_at_the_product_surface() {
let (backend, key_id, _key) = create_test_backend().await;
let result = KmsClient::disable_key(&backend, &key_id, None).await;
assert!(result.is_err());
assert!(result.expect_err("should be Err").to_string().contains("read-only"));
}
#[tokio::test]
async fn test_enable_key_is_noop() {
let (backend, key_id, _key) = create_test_backend().await;
// Enable should succeed (no-op for static KMS)
KmsClient::enable_key(&backend, &key_id, None)
.await
.expect("enable_key should be no-op");
// Wrong key should still fail
let result = KmsClient::enable_key(&backend, "wrong", None).await;
assert!(result.is_err());
// The static backend advertises no enable/disable or rotation
// capability, so the shared KmsBackend defaults reject all three.
for result in [
KmsBackendTrait::enable_key(&backend, &key_id).await,
KmsBackendTrait::disable_key(&backend, &key_id).await,
KmsBackendTrait::rotate_key(&backend, &key_id).await,
] {
let error = result.expect_err("static lifecycle mutations must be rejected");
assert!(matches!(error, KmsError::UnsupportedCapability { .. }), "got {error:?}");
}
}
#[tokio::test]
async fn test_delete_key_returns_error() {
let (backend, key_id, _key) = create_test_backend().await;
let result = KmsClient::schedule_key_deletion(&backend, &key_id, 7, None).await;
let result = KmsBackendTrait::delete_key(
&backend,
DeleteKeyRequest {
key_id: key_id.clone(),
pending_window_in_days: Some(7),
force_immediate: None,
},
)
.await;
assert!(result.is_err());
assert!(result.expect_err("should be Err").to_string().contains("read-only"));
}
#[tokio::test]
async fn test_rotate_key_returns_error() {
let (backend, key_id, _key) = create_test_backend().await;
let result = KmsClient::rotate_key(&backend, &key_id, None).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_health_check() {
let (backend, _key_id, _key) = create_test_backend().await;
KmsClient::health_check(&backend).await.expect("health_check should succeed");
}
#[tokio::test]
async fn test_backend_info() {
let (backend, key_id, _key) = create_test_backend().await;
let info = KmsClient::backend_info(&backend);
assert_eq!(info.backend_type, "static");
assert_eq!(info.endpoint, "local");
assert!(info.healthy);
assert_eq!(info.metadata.get("key_id"), Some(&key_id));
KmsBackendTrait::health_check(&backend)
.await
.expect("health_check should succeed");
}
#[tokio::test]
@@ -730,17 +679,13 @@ mod tests {
let plaintext = b"Hello, static KMS world!";
let enc_request = EncryptRequest::new(key_id.clone(), plaintext.to_vec());
let enc_response = KmsClient::encrypt(&backend, &enc_request, None)
.await
.expect("encrypt should succeed");
let enc_response = backend.encrypt_to_envelope(&enc_request).expect("encrypt should succeed");
assert_eq!(enc_response.key_id, key_id);
assert!(!enc_response.ciphertext.is_empty());
let dec_request = DecryptRequest::new(enc_response.ciphertext);
let decrypted = KmsClient::decrypt(&backend, &dec_request, None)
.await
.expect("decrypt should succeed");
let decrypted = backend.decrypt_envelope(&dec_request).expect("decrypt should succeed");
assert_eq!(decrypted, plaintext);
}
crates/kms/src/backends/vault.rs
+2246 -270
View File
File diff suppressed because it is too large Load Diff
crates/kms/src/backends/vault_credentials.rs
+1357
View File
File diff suppressed because it is too large Load Diff
crates/kms/src/backends/vault_transit.rs
+1486 -238
View File
File diff suppressed because it is too large Load Diff
crates/kms/src/backup/capability.rs
+233
View File
@@ -0,0 +1,233 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Backup responsibility matrix: what a RustFS backup bundle owns per backend.
//!
//! The matrix is two-dimensional on purpose: responsibility is a function of
//! the backend *and* its at-rest protection state, not of the backend alone.
//! This keeps the schema stable when a backend changes protection direction —
//! switching Vault KV2 between storage-only and Transit-wrapped operation
//! selects a different existing row instead of changing the contract.
//!
//! These enums are backup-domain contract types. Once the backlog#1571
//! capability-discovery contract lands, the discovery surface is expected to
//! converge on (or map onto) the states defined here.
use serde::{Deserialize, Serialize};
/// Backend discriminant recorded in a backup manifest.
///
/// Wire names are aligned with [`crate::config::BackendConfig`] and
/// [`crate::config::KmsBackend`] (including the legacy `Vault` alias) so that
/// a manifest and a persisted KMS configuration never disagree about how the
/// same backend is spelled.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum BackupBackendKind {
/// Local file-based backend.
Local,
/// Vault KV v2 storage backend.
#[serde(rename = "VaultKV2", alias = "Vault")]
VaultKv2,
/// Vault Transit backend.
VaultTransit,
/// Static single-key backend.
Static,
}
/// At-rest protection state of master key material, as observed at snapshot
/// time.
///
/// The first three states mirror the Local backend's on-disk protection
/// marker (`StoredKeyProtection` in `backends/local.rs`, kebab-case wire
/// names). The remaining states describe the non-local backends.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum AtRestProtection {
/// Local key files AEAD-encrypted under the Argon2id-derived master key.
EncryptedMasterKey,
/// Local development-only plaintext key files. Such files must never
/// enter a bundle as-is; bundle artifacts are always re-wrapped under the
/// backup KEK.
PlaintextDevOnly,
/// Pre-beta.9 local key files without a protection marker; the effective
/// mode is resolved at read time. Treated like [`Self::PlaintextDevOnly`]
/// for bundling purposes: re-wrap is mandatory.
LegacyUnspecified,
/// Vault KV2 as currently shipped: material confidentiality relies on
/// Vault ACLs, KV2 at-rest encryption, and TLS only (the backend reports
/// `at_rest_protection = "vault-kv2-acl"`); RustFS applies no
/// cryptographic wrapping of its own.
StorageOnly,
/// Vault KV2 with material wrapped by Vault Transit before storage. Not
/// produced by any current backend; the row exists so a future direction
/// change selects a state instead of changing the schema.
TransitWrapped,
/// Vault Transit: the cryptographic root lives in Vault and is not
/// exportable. RustFS can only ever own metadata and references.
ExternalNonExportable,
/// Static backend: the secret is delivered externally at startup and
/// RustFS persists no key material at all.
ExternalSecretDelivery,
}
/// What a RustFS backup bundle is responsible for, per (backend, protection)
/// combination.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum BackupResponsibility {
/// The bundle carries the complete recoverable state: encrypted key
/// material for every version, salt, metadata, and configuration.
///
/// Restore precondition for the Local backend: the operator re-supplies
/// the master key out of band. The master key itself is outside the
/// backup domain; the manifest stores at most an opaque verifier.
FullMaterial,
/// The bundle carries only non-sensitive references and verification
/// information. The source of truth is external secret delivery and the
/// operator re-provides the secret during restore. Embedding the secret
/// itself in a bundle is forbidden.
ReferenceOnly,
/// The bundle carries RustFS-side metadata, configuration references and
/// verification data, while the cryptographic root is protected by the
/// external system's native snapshot/disaster-recovery flow (Vault/HSM).
/// Restore must re-establish the external trust root first.
MetadataPlusExternalRoot,
}
impl BackupResponsibility {
/// Resolve the responsibility matrix for one (backend, protection) cell.
///
/// Returns `None` for combinations that no supported deployment can
/// produce; manifests declaring such a combination are rejected as
/// corrupted. This function is total and the unit tests anchor every
/// cell, so any change to the matrix is a deliberate contract change.
pub fn for_backend(backend: BackupBackendKind, protection: AtRestProtection) -> Option<Self> {
use AtRestProtection::*;
use BackupBackendKind::*;
match (backend, protection) {
(Local, EncryptedMasterKey | PlaintextDevOnly | LegacyUnspecified) => Some(Self::FullMaterial),
(Local, _) => None,
// Storage-only KV2 offers no external cryptographic root, so the
// bundle must own the material (re-wrapped under the backup KEK).
(VaultKv2, StorageOnly) => Some(Self::FullMaterial),
(VaultKv2, TransitWrapped) => Some(Self::MetadataPlusExternalRoot),
(VaultKv2, _) => None,
(VaultTransit, ExternalNonExportable) => Some(Self::MetadataPlusExternalRoot),
(VaultTransit, _) => None,
(Static, ExternalSecretDelivery) => Some(Self::ReferenceOnly),
(Static, _) => None,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::KmsBackend;
fn json<T: Serialize>(value: &T) -> String {
serde_json::to_string(value).expect("serialization should succeed")
}
#[test]
fn backend_kind_wire_names_match_kms_backend() {
let pairs = [
(BackupBackendKind::Local, KmsBackend::Local),
(BackupBackendKind::VaultKv2, KmsBackend::VaultKv2),
(BackupBackendKind::VaultTransit, KmsBackend::VaultTransit),
(BackupBackendKind::Static, KmsBackend::Static),
];
for (backup_kind, config_kind) in pairs {
assert_eq!(json(&backup_kind), json(&config_kind), "wire name drifted for {backup_kind:?}");
}
}
#[test]
fn backend_kind_accepts_legacy_vault_alias() {
let decoded: BackupBackendKind = serde_json::from_str("\"Vault\"").expect("legacy alias should decode");
assert_eq!(decoded, BackupBackendKind::VaultKv2);
}
#[test]
fn responsibility_matrix_is_anchored_cell_by_cell() {
use AtRestProtection::*;
use BackupBackendKind::*;
use BackupResponsibility::*;
// Every (backend, protection) cell, exhaustively. Changing any row is
// a contract change and must be made here consciously.
let matrix = [
(Local, EncryptedMasterKey, Some(FullMaterial)),
(Local, PlaintextDevOnly, Some(FullMaterial)),
(Local, LegacyUnspecified, Some(FullMaterial)),
(Local, StorageOnly, None),
(Local, TransitWrapped, None),
(Local, ExternalNonExportable, None),
(Local, ExternalSecretDelivery, None),
(VaultKv2, EncryptedMasterKey, None),
(VaultKv2, PlaintextDevOnly, None),
(VaultKv2, LegacyUnspecified, None),
(VaultKv2, StorageOnly, Some(FullMaterial)),
(VaultKv2, TransitWrapped, Some(MetadataPlusExternalRoot)),
(VaultKv2, ExternalNonExportable, None),
(VaultKv2, ExternalSecretDelivery, None),
(VaultTransit, EncryptedMasterKey, None),
(VaultTransit, PlaintextDevOnly, None),
(VaultTransit, LegacyUnspecified, None),
(VaultTransit, StorageOnly, None),
(VaultTransit, TransitWrapped, None),
(VaultTransit, ExternalNonExportable, Some(MetadataPlusExternalRoot)),
(VaultTransit, ExternalSecretDelivery, None),
(Static, EncryptedMasterKey, None),
(Static, PlaintextDevOnly, None),
(Static, LegacyUnspecified, None),
(Static, StorageOnly, None),
(Static, TransitWrapped, None),
(Static, ExternalNonExportable, None),
(Static, ExternalSecretDelivery, Some(ReferenceOnly)),
];
assert_eq!(matrix.len(), 28, "matrix must stay exhaustive: 4 backends x 7 protection states");
for (backend, protection, expected) in matrix {
assert_eq!(
BackupResponsibility::for_backend(backend, protection),
expected,
"matrix cell drifted for ({backend:?}, {protection:?})"
);
}
}
#[test]
fn local_protection_wire_names_match_stored_key_protection() {
use crate::backends::local::StoredKeyProtection;
// The manifest must record exactly the marker values the Local
// backend writes to disk, or a restore could misread protection.
let pairs = [
(AtRestProtection::EncryptedMasterKey, StoredKeyProtection::EncryptedMasterKey),
(AtRestProtection::PlaintextDevOnly, StoredKeyProtection::PlaintextDevOnly),
(AtRestProtection::LegacyUnspecified, StoredKeyProtection::LegacyUnspecified),
];
for (backup_state, stored_state) in pairs {
assert_eq!(json(&backup_state), json(&stored_state), "wire name drifted for {backup_state:?}");
}
}
#[test]
fn responsibility_wire_names_are_frozen() {
assert_eq!(json(&BackupResponsibility::FullMaterial), "\"full-material\"");
assert_eq!(json(&BackupResponsibility::ReferenceOnly), "\"reference-only\"");
assert_eq!(json(&BackupResponsibility::MetadataPlusExternalRoot), "\"metadata-plus-external-root\"");
}
}
crates/kms/src/backup/dry_run.rs
+280
View File
@@ -0,0 +1,280 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Restore dry-run report contract.
//!
//! A restore dry-run is a zero-write preflight: it evaluates a bundle against
//! a target and reports every blocker, conflict, and external dependency
//! mismatch without modifying the target in any way. The report itself is
//! plain data — producing, serializing, or discarding it has no side effects,
//! and an implementation that writes anything during a dry-run violates this
//! contract. All values in a report are identifiers and references; secrets,
//! tokens, and key material never appear in it.
use crate::backup::error::BackupError;
use serde::{Deserialize, Serialize};
/// Machine-readable category of a restore blocker.
///
/// The first six codes mirror the [`BackupError`] variants; the remaining
/// codes cover preflight conditions that are not bundle defects.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum RestoreBlockerCode {
/// The bundle failed structural or integrity validation.
BundleCorrupted,
/// The manifest input ended prematurely.
BundleTruncated,
/// The manifest format version is unknown to this build.
UnknownFormatVersion,
/// The supplied backup KEK does not match the bundle's KEK.
WrongBackupKek,
/// A required artifact is absent from the bundle.
MissingArtifact,
/// The bundle has no completeness marker or is marked in-progress.
IncompleteBundle,
/// The target backend cannot satisfy the bundle's responsibility model.
UnsupportedBackend,
/// The bundle was produced by a different deployment than the target and
/// no explicit cross-deployment authorization applies.
DeploymentMismatch,
/// An external dependency (Vault cluster, mount, Transit key, ...) that
/// the bundle references is unreachable or missing.
ExternalDependencyUnavailable,
}
/// One condition that forbids the restore outright.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RestoreBlocker {
/// Machine-readable category.
pub code: RestoreBlockerCode,
/// Human-readable detail. Identifiers only; never secrets or material.
pub detail: String,
}
impl From<&BackupError> for RestoreBlocker {
fn from(error: &BackupError) -> Self {
let code = match error {
BackupError::Corrupted { .. } => RestoreBlockerCode::BundleCorrupted,
BackupError::Truncated { .. } => RestoreBlockerCode::BundleTruncated,
BackupError::UnknownVersion { .. } => RestoreBlockerCode::UnknownFormatVersion,
BackupError::WrongKek { .. } => RestoreBlockerCode::WrongBackupKek,
BackupError::MissingArtifact { .. } => RestoreBlockerCode::MissingArtifact,
BackupError::IncompleteBundle { .. } => RestoreBlockerCode::IncompleteBundle,
};
Self {
code,
detail: error.to_string(),
}
}
}
/// Kind of a conflict between bundle state and existing target state.
///
/// Restore is non-destructive by default: every conflict blocks the restore
/// unless an explicit, audited conflict policy resolves it. Silent overwrite
/// or merge is never an option.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum RestoreConflictKind {
/// The target already has a key with this stable id.
KeyAlreadyExists,
/// Restoring would lower a key version the target has already observed.
VersionRegression,
/// Restoring would lower the snapshot generation the target has already
/// observed.
GenerationRegression,
/// Restoring would revive a key the target has deleted or scheduled for
/// deletion.
StateRegression,
}
/// One conflict with existing target state.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RestoreConflict {
/// Stable key id the conflict concerns.
pub key_id: String,
/// Machine-readable category.
pub kind: RestoreConflictKind,
/// Human-readable detail. Identifiers only; never secrets or material.
pub detail: String,
}
/// A mismatch between an external dependency reference recorded in the bundle
/// and what the target environment observes.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ExternalDependencyMismatch {
/// Which dependency is affected (for example a Vault mount or Transit
/// key name). References only; never credentials.
pub dependency: String,
/// The value the bundle recorded.
pub expected: String,
/// The value the target environment reports.
pub observed: String,
}
/// Result of a restore dry-run preflight.
///
/// # Zero-write contract
///
/// A dry-run must not write to the target: no staging directories, no
/// repaired records, no metadata fixes triggered along the read path. The
/// report is pure data over values already known to the caller.
///
/// ```
/// use rustfs_kms::backup::{RestoreBlocker, RestoreBlockerCode, RestoreDryRunReport};
///
/// let clean = RestoreDryRunReport {
/// backup_id: "backup-0001".to_string(),
/// target_deployment_identity: "deployment-a".to_string(),
/// blockers: Vec::new(),
/// conflicts: Vec::new(),
/// external_mismatches: Vec::new(),
/// };
/// assert!(clean.restore_permitted());
///
/// let blocked = RestoreDryRunReport {
/// blockers: vec![RestoreBlocker {
/// code: RestoreBlockerCode::IncompleteBundle,
/// detail: "manifest has no completeness marker".to_string(),
/// }],
/// ..clean
/// };
/// assert!(!blocked.restore_permitted());
/// ```
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RestoreDryRunReport {
/// Identifier of the evaluated bundle.
pub backup_id: String,
/// Identity of the restore target the bundle was evaluated against.
pub target_deployment_identity: String,
/// Conditions that forbid the restore outright.
pub blockers: Vec<RestoreBlocker>,
/// Conflicts with existing target state.
pub conflicts: Vec<RestoreConflict>,
/// External dependency mismatches.
pub external_mismatches: Vec<ExternalDependencyMismatch>,
}
impl RestoreDryRunReport {
/// Whether the restore may proceed: true only when the preflight found
/// no blockers, no conflicts, and no external dependency mismatches.
pub fn restore_permitted(&self) -> bool {
self.blockers.is_empty() && self.conflicts.is_empty() && self.external_mismatches.is_empty()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn sample_report() -> RestoreDryRunReport {
RestoreDryRunReport {
backup_id: "backup-0001".to_string(),
target_deployment_identity: "deployment-b".to_string(),
blockers: vec![RestoreBlocker {
code: RestoreBlockerCode::DeploymentMismatch,
detail: "bundle was produced by deployment-a".to_string(),
}],
conflicts: vec![RestoreConflict {
key_id: "object-key".to_string(),
kind: RestoreConflictKind::VersionRegression,
detail: "target observed version 5, bundle carries version 3".to_string(),
}],
external_mismatches: vec![ExternalDependencyMismatch {
dependency: "vault transit key rustfs-master".to_string(),
expected: "min_version=2".to_string(),
observed: "min_version=4".to_string(),
}],
}
}
#[test]
fn report_round_trips_through_json() {
let report = sample_report();
let json = serde_json::to_string(&report).expect("serialization should succeed");
let decoded: RestoreDryRunReport = serde_json::from_str(&json).expect("deserialization should succeed");
assert_eq!(decoded, report);
}
#[test]
fn restore_permitted_requires_every_section_empty() {
assert!(!sample_report().restore_permitted());
let clean = RestoreDryRunReport {
blockers: Vec::new(),
conflicts: Vec::new(),
external_mismatches: Vec::new(),
..sample_report()
};
assert!(clean.restore_permitted());
for section in 0..3 {
let mut report = clean.clone();
match section {
0 => report.blockers = sample_report().blockers,
1 => report.conflicts = sample_report().conflicts,
_ => report.external_mismatches = sample_report().external_mismatches,
}
assert!(!report.restore_permitted(), "section {section} alone must block the restore");
}
}
#[test]
fn every_backup_error_maps_to_a_blocker_code() {
let cases = [
(BackupError::corrupted("x"), RestoreBlockerCode::BundleCorrupted),
(BackupError::truncated("x"), RestoreBlockerCode::BundleTruncated),
(
BackupError::UnknownVersion { found: 2, supported: 1 },
RestoreBlockerCode::UnknownFormatVersion,
),
(
BackupError::WrongKek {
required_kek_id: "a".to_string(),
required_kek_version: 1,
supplied_kek_id: "b".to_string(),
supplied_kek_version: 1,
},
RestoreBlockerCode::WrongBackupKek,
),
(BackupError::missing_artifact("key-material"), RestoreBlockerCode::MissingArtifact),
(BackupError::incomplete_bundle("x"), RestoreBlockerCode::IncompleteBundle),
];
for (error, expected_code) in cases {
let blocker = RestoreBlocker::from(&error);
assert_eq!(blocker.code, expected_code, "wrong code for {error:?}");
assert_eq!(blocker.detail, error.to_string());
}
}
/// The zero-write contract in practice: a report is plain serializable
/// data with no handles, no I/O, and no drop side effects.
#[test]
fn report_types_are_plain_data() {
fn assert_plain_data<T>()
where
T: serde::Serialize + serde::de::DeserializeOwned + Clone + PartialEq + std::fmt::Debug + Send + Sync + 'static,
{
}
assert_plain_data::<RestoreDryRunReport>();
assert_plain_data::<RestoreBlocker>();
assert_plain_data::<RestoreConflict>();
assert_plain_data::<ExternalDependencyMismatch>();
}
}
crates/kms/src/backup/error.rs
+133
View File
@@ -0,0 +1,133 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Typed failures for the backup/restore bundle contract.
use thiserror::Error;
/// Typed failures raised while decoding or validating a backup bundle.
///
/// Every variant is a fail-closed condition: a restore surface observing any
/// of them must abort before touching target state. Messages carry only
/// identifiers (backup ids, KEK ids, artifact kinds and paths) — never key
/// material, bundle plaintext, or credentials.
#[derive(Error, Debug, Clone, PartialEq, Eq)]
pub enum BackupError {
/// Manifest or bundle content fails structural, schema, or integrity
/// validation (unknown fields, duplicate fields, digest mismatch,
/// contradictory responsibility declarations, ...).
#[error("backup bundle corrupted: {reason}")]
Corrupted { reason: String },
/// Input ended before a complete manifest could be decoded.
#[error("backup manifest truncated: {reason}")]
Truncated { reason: String },
/// Manifest declares a format version this build does not understand.
/// Unknown versions are always rejected; there is no best-effort read.
#[error("unknown backup manifest format version {found} (this build supports version {supported})")]
UnknownVersion { found: u32, supported: u32 },
/// Bundle is protected by a different backup KEK than the one supplied.
#[error(
"backup bundle requires KEK '{required_kek_id}' version {required_kek_version}; \
supplied KEK '{supplied_kek_id}' version {supplied_kek_version} cannot open it"
)]
WrongKek {
required_kek_id: String,
required_kek_version: u32,
supplied_kek_id: String,
supplied_kek_version: u32,
},
/// Manifest requires an artifact that is not present in the bundle.
#[error("backup bundle is missing a required artifact: {artifact}")]
MissingArtifact { artifact: String },
/// Bundle has no completeness marker or records an in-progress state.
/// A bundle that never reached its completeness marker must never be
/// restored, regardless of how much of it is readable.
#[error("backup bundle is incomplete ({reason}); incomplete bundles must never be restored")]
IncompleteBundle { reason: String },
}
impl BackupError {
/// Create a corrupted-bundle error.
pub fn corrupted<S: Into<String>>(reason: S) -> Self {
Self::Corrupted { reason: reason.into() }
}
/// Create a truncated-manifest error.
pub fn truncated<S: Into<String>>(reason: S) -> Self {
Self::Truncated { reason: reason.into() }
}
/// Create an incomplete-bundle error.
pub fn incomplete_bundle<S: Into<String>>(reason: S) -> Self {
Self::IncompleteBundle { reason: reason.into() }
}
/// Create a missing-artifact error.
pub fn missing_artifact<S: Into<String>>(artifact: S) -> Self {
Self::MissingArtifact {
artifact: artifact.into(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error::KmsError;
#[test]
fn backup_errors_convert_into_kms_error_transparently() {
let error = BackupError::corrupted("manifest digest mismatch");
let kms_error: KmsError = error.clone().into();
assert_eq!(kms_error.to_string(), error.to_string());
assert!(matches!(kms_error, KmsError::Backup(inner) if inner == error));
}
#[test]
fn error_messages_carry_identifiers_only() {
// The display strings must stay descriptive without ever embedding
// material or bundle plaintext; each variant only interpolates the
// identifiers below.
let wrong_kek = BackupError::WrongKek {
required_kek_id: "backup-kek-1".to_string(),
required_kek_version: 3,
supplied_kek_id: "backup-kek-2".to_string(),
supplied_kek_version: 1,
};
assert_eq!(
wrong_kek.to_string(),
"backup bundle requires KEK 'backup-kek-1' version 3; supplied KEK 'backup-kek-2' version 1 cannot open it"
);
let unknown = BackupError::UnknownVersion { found: 9, supported: 1 };
assert_eq!(
unknown.to_string(),
"unknown backup manifest format version 9 (this build supports version 1)"
);
assert_eq!(
BackupError::missing_artifact("key-material").to_string(),
"backup bundle is missing a required artifact: key-material"
);
assert_eq!(
BackupError::incomplete_bundle("manifest has no completeness marker").to_string(),
"backup bundle is incomplete (manifest has no completeness marker); incomplete bundles must never be restored"
);
}
}
crates/kms/src/backup/local_export.rs
+1094
View File
File diff suppressed because it is too large Load Diff
crates/kms/src/backup/local_restore.rs
+1970
View File
File diff suppressed because it is too large Load Diff
crates/kms/src/backup/manifest.rs
+1274
View File
File diff suppressed because it is too large Load Diff
crates/kms/src/backup/mod.rs
+74
View File
@@ -0,0 +1,74 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Backup/restore contracts and backup production for KMS state.
//!
//! The contract side defines the versioned backup manifest, the per-backend
//! responsibility matrix, typed failure modes, and the restore dry-run
//! report. [`local_export`] implements the producer side and
//! [`local_restore`] the consumer side for the Local backend as
//! crate-internal APIs; the admin API builds on these pieces in follow-up
//! changes.
//!
//! # Bundle model
//!
//! A backup bundle is a set of AEAD-encrypted artifacts described by a single
//! [`BackupManifest`]. All state in a bundle belongs to one snapshot
//! generation — there is no partially consistent bundle. The bundle is
//! protected by a backup KEK that is deliberately outside the business KMS
//! trust hierarchy, and the manifest is sealed with a completeness marker and
//! a final digest; a bundle that never reached its marker is permanently
//! non-restorable.
//!
//! # Restore ordering
//!
//! Restore implementations must follow this order: re-establish the external
//! trust root first (Vault/HSM native restore where one exists), then
//! material and version records into staging, then metadata and
//! configuration, then verification, and only then an explicit atomic
//! cutover. A dry-run ([`RestoreDryRunReport`]) performs zero writes.
//!
//! # Deliberately unfrozen
//!
//! Fields whose shape depends on contracts still in flight are reserved
//! rather than guessed (see [`ReservedSlot`]): the per-key version inventory
//! (backlog#1565) and capability discovery (backlog#1571). Alias and policy
//! artifacts are reserved names for features that do not exist yet. Reserved
//! slots reject data in format version 1 and become real types in a later
//! format version.
mod capability;
mod dry_run;
mod error;
pub mod local_export;
pub mod local_restore;
mod manifest;
pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
pub use dry_run::{
ExternalDependencyMismatch, RestoreBlocker, RestoreBlockerCode, RestoreConflict, RestoreConflictKind, RestoreDryRunReport,
};
pub use error::BackupError;
pub use local_export::{
BackupKek, LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, decrypt_bundle_artifact, export_local_backup,
read_local_bundle_manifest,
};
pub use local_restore::{
LocalRestoreReport, LocalRestoreRequest, RestoreConflictPolicy, abort_local_restore, dry_run_local_restore,
restore_local_backup,
};
pub use manifest::{
AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest,
DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot,
};
crates/kms/src/config.rs
+577 -20
View File
@@ -29,8 +29,23 @@ pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRAN
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
pub const ENV_KMS_STATIC_SECRET_KEY: &str = "RUSTFS_KMS_STATIC_SECRET_KEY";
pub const ENV_KMS_STATIC_SECRET_KEY_FILE: &str = "RUSTFS_KMS_STATIC_SECRET_KEY_FILE";
pub const ENV_KMS_VAULT_APPROLE_ROLE_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_ROLE_ID";
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID";
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
/// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior.
///
/// Out-of-range values are clamped at use rather than rejected so existing
/// deployments with oversized settings keep starting after an upgrade.
pub(crate) const MAX_OPERATION_TIMEOUT: Duration = Duration::from_secs(300);
/// Upper bound applied to `KmsConfig::retry_attempts` when deriving backend behavior.
pub(crate) const MAX_RETRY_ATTEMPTS: u32 = 10;
fn default_vault_transit_metadata_kv_mount() -> String {
DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT.to_string()
@@ -40,6 +55,14 @@ fn default_vault_transit_metadata_key_prefix() -> String {
DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX.to_string()
}
fn default_vault_kv2_mount_path() -> String {
"transit".to_string()
}
fn default_vault_approle_mount() -> String {
DEFAULT_VAULT_APPROLE_MOUNT.to_string()
}
pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[
RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"),
RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"),
@@ -91,7 +114,9 @@ pub(crate) fn redacted_secret_option(value: Option<&str>) -> Option<&'static str
/// KMS backend types
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum KmsBackend {
/// Vault KV v2 + Transit backend (key metadata in KV, wrapping via Transit)
/// Vault KV v2 storage backend: master key material is stored directly in KV v2.
/// Confidentiality relies on Vault ACLs, KV v2 at-rest encryption, and TLS; the
/// backend performs no Transit wrapping of key material.
#[serde(rename = "VaultKV2", alias = "Vault")]
VaultKv2,
/// Vault Transit backend using Vault as the cryptographic source of truth
@@ -117,9 +142,15 @@ pub struct KmsConfig {
/// Allow development-only insecure defaults such as plaintext local keys or HTTP Vault.
#[serde(default)]
pub allow_insecure_dev_defaults: bool,
/// Operation timeout
/// Timeout for a single backend attempt.
///
/// This bounds one outbound request, not the whole operation: the operation
/// policy owns the total deadline across retries. Values above 300 seconds
/// are clamped at use (see `KmsConfig::effective_timeout`).
pub timeout: Duration,
/// Number of retry attempts
/// Number of retry attempts.
///
/// Values above 10 are clamped at use (see `KmsConfig::effective_retry_attempts`).
pub retry_attempts: u32,
/// Enable caching
pub enable_cache: bool,
@@ -147,7 +178,7 @@ impl Default for KmsConfig {
pub enum BackendConfig {
/// Local backend configuration
Local(LocalConfig),
/// Vault KV v2 + Transit backend configuration
/// Vault KV v2 storage backend configuration
#[serde(rename = "VaultKV2", alias = "Vault")]
VaultKv2(Box<VaultConfig>),
/// Vault Transit backend configuration
@@ -255,7 +286,11 @@ impl StaticConfig {
}
}
/// Vault KV v2 + Transit backend configuration (metadata in KV, key wrapping via Transit)
/// Vault KV v2 backend configuration.
///
/// Key material and metadata are stored directly in KV v2; any identity with KV read
/// access to the key path can recover plaintext master key material. Use the Vault
/// Transit backend when cryptographic isolation of key material is required.
#[derive(Clone, Serialize, Deserialize)]
pub struct VaultConfig {
/// Vault server URL
@@ -264,7 +299,10 @@ pub struct VaultConfig {
pub auth_method: VaultAuthMethod,
/// Vault namespace (Vault Enterprise)
pub namespace: Option<String>,
/// Transit engine mount path
/// Deprecated: legacy Transit engine mount path. The Vault KV2 backend never calls
/// the Transit engine, so this value is unused; the field is retained (and
/// defaulted) only so previously persisted configurations keep deserializing.
#[serde(default = "default_vault_kv2_mount_path")]
pub mount_path: String,
/// KV engine mount path for storing keys
pub kv_mount: String,
@@ -360,18 +398,94 @@ impl Default for VaultTransitConfig {
pub enum VaultAuthMethod {
/// Token authentication
Token { token: String },
/// AppRole authentication
AppRole { role_id: String, secret_id: String },
/// AppRole authentication: login with `role_id` + `secret_id` for a
/// lease-bound token that is renewed in the background.
AppRole {
role_id: String,
/// Inline secret_id; used only when `secret_id_file` is unset.
secret_id: String,
/// Path to a file holding the secret_id. Re-read on every login so an
/// externally rotated secret_id is picked up; takes precedence over the
/// inline value.
#[serde(default)]
secret_id_file: Option<PathBuf>,
/// AppRole auth engine mount path.
#[serde(default = "default_vault_approle_mount")]
mount: String,
/// Fail-closed margin in seconds: once the current token is within this
/// window of expiry without a successful refresh, requests are refused
/// instead of sent with a token that may lapse mid-flight. Defaults to
/// the per-attempt timeout.
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
/// Agent-managed token file (for example a Vault Agent auto-auth sink):
/// the token is read from `path` and re-read periodically so a token
/// rotated by the agent is picked up without a restart.
TokenFile {
path: PathBuf,
/// Seconds between token file re-reads. Each successful read also
/// extends the token's observed validity to twice this value, so a
/// file that stops being readable eventually trips the fail-closed
/// window. Defaults to 30 seconds.
#[serde(default)]
poll_interval_secs: Option<u64>,
/// Fail-closed margin in seconds, as on `AppRole`. Defaults to the
/// per-attempt timeout.
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
}
impl VaultAuthMethod {
/// AppRole authentication with the default mount and no secret-id file.
pub fn approle(role_id: String, secret_id: String) -> Self {
Self::AppRole {
role_id,
secret_id,
secret_id_file: None,
mount: default_vault_approle_mount(),
refresh_safety_window_secs: None,
}
}
/// Agent-managed token file with the default poll interval.
pub fn token_file(path: PathBuf) -> Self {
Self::TokenFile {
path,
poll_interval_secs: None,
refresh_safety_window_secs: None,
}
}
}
impl fmt::Debug for VaultAuthMethod {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Token { token } => f.debug_struct("Token").field("token", &redacted_secret(token)).finish(),
Self::AppRole { role_id, secret_id } => f
Self::AppRole {
role_id,
secret_id,
secret_id_file,
mount,
refresh_safety_window_secs,
} => f
.debug_struct("AppRole")
.field("role_id", role_id)
.field("secret_id", &redacted_secret(secret_id))
.field("secret_id_file", secret_id_file)
.field("mount", mount)
.field("refresh_safety_window_secs", refresh_safety_window_secs)
.finish(),
Self::TokenFile {
path,
poll_interval_secs,
refresh_safety_window_secs,
} => f
.debug_struct("TokenFile")
.field("path", path)
.field("poll_interval_secs", poll_interval_secs)
.field("refresh_safety_window_secs", refresh_safety_window_secs)
.finish(),
}
}
@@ -424,7 +538,12 @@ impl KmsConfig {
}
}
/// Create a new KMS configuration for Vault backend with token authentication (recommended for production)
/// Create a new KMS configuration for the Vault KV v2 backend with token authentication.
///
/// Master key material is stored directly in Vault KV v2: confidentiality relies on
/// Vault ACLs, KV v2 at-rest encryption, and TLS. KV read access to the key path is
/// equivalent to holding the plaintext master keys. Use [`KmsConfig::vault_transit`]
/// when key material must never be readable through Vault storage APIs.
pub fn vault(address: Url, token: String) -> Self {
Self {
backend: KmsBackend::VaultKv2,
@@ -437,13 +556,16 @@ impl KmsConfig {
}
}
/// Create a new KMS configuration for Vault backend with AppRole authentication (recommended for production)
/// Create a new KMS configuration for the Vault KV v2 backend with AppRole authentication.
///
/// Shares the security boundary described on [`KmsConfig::vault`]: key material lives
/// in KV v2 and is protected only by Vault ACLs and KV v2 at-rest encryption.
pub fn vault_approle(address: Url, role_id: String, secret_id: String) -> Self {
Self {
backend: KmsBackend::VaultKv2,
backend_config: BackendConfig::VaultKv2(Box::new(VaultConfig {
address: address.to_string(),
auth_method: VaultAuthMethod::AppRole { role_id, secret_id },
auth_method: VaultAuthMethod::approle(role_id, secret_id),
..Default::default()
})),
..Default::default()
@@ -536,6 +658,16 @@ impl KmsConfig {
self
}
/// Per-attempt timeout with the configured value clamped to the supported maximum.
pub(crate) fn effective_timeout(&self) -> Duration {
self.timeout.min(MAX_OPERATION_TIMEOUT)
}
/// Retry attempts with the configured value clamped to the supported maximum.
pub(crate) fn effective_retry_attempts(&self) -> u32 {
self.retry_attempts.min(MAX_RETRY_ATTEMPTS)
}
/// Validate the configuration
pub fn validate(&self) -> Result<()> {
// Validate timeout
@@ -548,6 +680,23 @@ impl KmsConfig {
return Err(KmsError::configuration_error("Retry attempts must be greater than 0"));
}
// Oversized values are clamped at use (not rejected) so pre-existing
// configurations cannot keep the service from starting after upgrade.
if self.timeout > MAX_OPERATION_TIMEOUT {
tracing::warn!(
configured_secs = self.timeout.as_secs(),
max_secs = MAX_OPERATION_TIMEOUT.as_secs(),
"KMS timeout exceeds the supported maximum; backend operations clamp it to the maximum"
);
}
if self.retry_attempts > MAX_RETRY_ATTEMPTS {
tracing::warn!(
configured = self.retry_attempts,
max = MAX_RETRY_ATTEMPTS,
"KMS retry_attempts exceeds the supported maximum; backend operations clamp it to the maximum"
);
}
// Validate backend-specific configuration
match &self.backend_config {
BackendConfig::Local(config) => {
@@ -574,13 +723,14 @@ impl KmsConfig {
return Err(KmsError::configuration_error("Vault KV2 address must use http or https scheme"));
}
validate_vault_auth_method("Vault KV2", &config.auth_method)?;
if !self.allow_insecure_dev_defaults {
validate_vault_development_defaults("Vault KV2", &config.address, &config.auth_method, config.tls.as_ref())?;
}
if config.mount_path.is_empty() {
return Err(KmsError::configuration_error("Vault KV2 mount path cannot be empty"));
}
// `mount_path` is deprecated and unused by this backend, so an empty value
// is deliberately not an error.
// Validate TLS configuration if using HTTPS
if config.address.starts_with("https://")
@@ -598,6 +748,8 @@ impl KmsConfig {
return Err(KmsError::configuration_error("Vault Transit address must use http or https scheme"));
}
validate_vault_auth_method("Vault Transit", &config.auth_method)?;
if !self.allow_insecure_dev_defaults {
validate_vault_development_defaults(
"Vault Transit",
@@ -702,14 +854,24 @@ impl KmsConfig {
}
KmsBackend::VaultKv2 => {
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
let token = get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token");
let auth_method = vault_auth_method_from_env()?;
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") {
Some(path) => {
tracing::warn!(
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
);
path
}
None => default_vault_kv2_mount_path(),
};
config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig {
address,
auth_method: VaultAuthMethod::Token { token },
auth_method,
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"),
mount_path,
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
tls: vault_tls_config(skip_tls_verify),
@@ -717,12 +879,12 @@ impl KmsConfig {
}
KmsBackend::VaultTransit => {
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
let token = get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token");
let auth_method = vault_auth_method_from_env()?;
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
config.backend_config = BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
address,
auth_method: VaultAuthMethod::Token { token },
auth_method,
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"),
metadata_kv_mount: get_env_str(
@@ -801,6 +963,95 @@ fn is_under_temp_dir(path: &Path) -> bool {
path.starts_with(std::env::temp_dir())
}
/// Resolve the Vault auth method from environment variables.
///
/// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication;
/// the secret_id then comes from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE`
/// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE`
/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the
/// file taking precedence. Without a role id the legacy token flow applies.
fn vault_auth_method_from_env() -> Result<VaultAuthMethod> {
if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) {
// A token file names one authoritative credential source; combining it
// with another one would leave the effective identity ambiguous, so
// that is a configuration error rather than a precedence rule.
if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() {
return Err(KmsError::configuration_error(format!(
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
)));
}
if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() {
return Err(KmsError::configuration_error(format!(
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method"
)));
}
return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file)));
}
let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else {
return Ok(VaultAuthMethod::Token {
token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"),
});
};
let secret_id_file = get_env_opt_str(ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE).map(PathBuf::from);
let secret_id = get_env_opt_str(ENV_KMS_VAULT_APPROLE_SECRET_ID).unwrap_or_default();
if secret_id.is_empty() && secret_id_file.is_none() {
return Err(KmsError::configuration_error(format!(
"Vault AppRole requires {ENV_KMS_VAULT_APPROLE_SECRET_ID} or {ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE} to be set"
)));
}
Ok(VaultAuthMethod::AppRole {
role_id,
secret_id,
secret_id_file,
mount: get_env_str(ENV_KMS_VAULT_APPROLE_MOUNT, DEFAULT_VAULT_APPROLE_MOUNT),
refresh_safety_window_secs: None,
})
}
fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod) -> Result<()> {
match auth_method {
VaultAuthMethod::Token { .. } => Ok(()),
VaultAuthMethod::AppRole {
role_id,
secret_id,
secret_id_file,
mount,
..
} => {
if role_id.is_empty() {
return Err(KmsError::configuration_error(format!("{backend_name} AppRole role_id cannot be empty")));
}
if secret_id.is_empty() && secret_id_file.is_none() {
return Err(KmsError::configuration_error(format!(
"{backend_name} AppRole requires a secret_id or a secret_id_file"
)));
}
if mount.is_empty() {
return Err(KmsError::configuration_error(format!("{backend_name} AppRole mount cannot be empty")));
}
Ok(())
}
VaultAuthMethod::TokenFile {
path,
poll_interval_secs,
..
} => {
if path.as_os_str().is_empty() {
return Err(KmsError::configuration_error(format!("{backend_name} token file path cannot be empty")));
}
if poll_interval_secs == &Some(0) {
return Err(KmsError::configuration_error(format!(
"{backend_name} token file poll interval must be greater than 0"
)));
}
Ok(())
}
}
}
fn validate_vault_development_defaults(
backend_name: &str,
address: &str,
@@ -855,6 +1106,31 @@ mod tests {
assert_eq!(local_config.key_dir, temp_dir.path());
}
#[test]
fn test_oversized_timeout_and_retries_clamped_not_rejected() {
let temp_dir = TempDir::new().expect("Failed to create temp dir");
let config = KmsConfig {
timeout: Duration::from_secs(3_600),
retry_attempts: 50,
..KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults()
};
// Out-of-range values must not keep the service from starting.
assert!(config.validate().is_ok());
assert_eq!(config.effective_timeout(), MAX_OPERATION_TIMEOUT);
assert_eq!(config.effective_retry_attempts(), MAX_RETRY_ATTEMPTS);
// In-range values pass through unchanged.
let config = KmsConfig {
timeout: Duration::from_secs(45),
retry_attempts: 5,
..config
};
assert!(config.validate().is_ok());
assert_eq!(config.effective_timeout(), Duration::from_secs(45));
assert_eq!(config.effective_retry_attempts(), 5);
}
#[test]
fn test_local_development_defaults_require_opt_in() {
let temp_dir = TempDir::new().expect("Failed to create temp dir");
@@ -977,6 +1253,73 @@ mod tests {
assert!(config.vault_config().is_some());
}
#[test]
fn test_persisted_vault_kv2_config_without_mount_path_deserializes() {
// Configurations persisted after mount_path was deprecated may omit the field;
// it must default instead of failing deserialization.
let raw = r#"{
"backend": "VaultKV2",
"backend_config": {
"VaultKV2": {
"address": "http://127.0.0.1:8200",
"auth_method": { "Token": { "token": "t" } },
"namespace": null,
"kv_mount": "secret",
"key_path_prefix": "rustfs/kms/keys",
"tls": null
}
},
"default_key_id": null,
"timeout": {"secs": 30, "nanos": 0},
"retry_attempts": 3,
"enable_cache": true,
"cache_config": {
"max_keys": 1000,
"ttl": {"secs": 3600, "nanos": 0},
"enable_metrics": true
}
}"#;
let config: KmsConfig = serde_json::from_str(raw).expect("persisted kms config without mount_path");
assert_eq!(config.backend, KmsBackend::VaultKv2);
let vault = config.vault_config().expect("vault-kv2 config");
assert_eq!(vault.mount_path, "transit");
}
#[test]
fn test_vault_kv2_empty_mount_path_passes_validation() {
let address = Url::parse("https://vault.example.com:8200").expect("Valid URL");
let mut config = KmsConfig::vault(address, "test-token".to_string());
if let BackendConfig::VaultKv2(vault) = &mut config.backend_config {
vault.mount_path = String::new();
}
assert!(config.validate().is_ok(), "deprecated mount_path must not be required");
}
#[test]
fn test_vault_kv2_sources_do_not_claim_transit_wrapping() {
let sources = [
("config.rs", include_str!("config.rs")),
("api_types.rs", include_str!("api_types.rs")),
("backends/vault.rs", include_str!("backends/vault.rs")),
("lib.rs", include_str!("lib.rs")),
];
// Assemble the needles at runtime so this guard does not match its own source.
let needles = [
format!("wrapping via {}", "Transit"),
format!("KV v2 + {}", "Transit"),
format!("KV2+{}", "Transit"),
format!("you would use Vault's {} engine", "transit"),
];
for (name, source) in sources {
for needle in &needles {
assert!(
!source.contains(needle.as_str()),
"{name} still describes the Vault KV2 backend with `{needle}`"
);
}
}
}
#[test]
fn test_legacy_persisted_vault_transit_config_uses_metadata_defaults() {
let raw = r#"{
@@ -1133,6 +1476,220 @@ mod tests {
);
}
#[test]
fn test_from_env_selects_approle_when_role_id_is_set() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault")),
("RUSTFS_KMS_VAULT_ADDRESS", Some("https://vault.example.com")),
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
(ENV_KMS_VAULT_APPROLE_SECRET_ID, Some("env-approle-secret-id")),
(ENV_KMS_VAULT_APPROLE_MOUNT, Some("approle-alt")),
// A stale token env var must not override the AppRole selection.
("RUSTFS_KMS_VAULT_TOKEN", Some("vault-token")),
],
|| {
let config = KmsConfig::from_env().expect("kms config should load from env");
let vault = config.vault_config().expect("vault backend config");
let VaultAuthMethod::AppRole {
role_id,
secret_id,
secret_id_file,
mount,
refresh_safety_window_secs,
} = &vault.auth_method
else {
panic!("role id in the environment must select AppRole auth, got {:?}", vault.auth_method);
};
assert_eq!(role_id, "env-role-id");
assert_eq!(secret_id, "env-approle-secret-id");
assert_eq!(secret_id_file, &None);
assert_eq!(mount, "approle-alt");
assert_eq!(refresh_safety_window_secs, &None);
},
);
}
#[test]
fn test_from_env_approle_secret_id_file_is_stored_as_path() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
("RUSTFS_KMS_VAULT_ADDRESS", Some("https://vault.example.com")),
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
(ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE, Some("/etc/rustfs/approle-secret-id")),
],
|| {
let config = KmsConfig::from_env().expect("kms config should load from env");
let vault = config.vault_transit_config().expect("vault transit backend config");
let VaultAuthMethod::AppRole {
secret_id,
secret_id_file,
mount,
..
} = &vault.auth_method
else {
panic!("role id in the environment must select AppRole auth");
};
// The path is stored, not read: the secret_id file is re-read on
// every login so external rotation is picked up.
assert_eq!(secret_id_file.as_deref(), Some(std::path::Path::new("/etc/rustfs/approle-secret-id")));
assert!(secret_id.is_empty());
assert_eq!(mount, DEFAULT_VAULT_APPROLE_MOUNT);
},
);
}
#[test]
fn test_from_env_approle_requires_secret_id_or_file() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault")),
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
(ENV_KMS_VAULT_APPROLE_SECRET_ID, None::<&str>),
(ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE, None::<&str>),
],
|| {
let error = KmsConfig::from_env().expect_err("approle without a secret_id source must be rejected");
assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_SECRET_ID));
},
);
}
#[test]
fn test_from_env_selects_token_file() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault")),
("RUSTFS_KMS_VAULT_ADDRESS", Some("https://vault.example.com")),
(ENV_KMS_VAULT_TOKEN_FILE, Some("/run/vault-agent/token")),
],
|| {
let config = KmsConfig::from_env().expect("kms config should load from env");
let vault = config.vault_config().expect("vault backend config");
let VaultAuthMethod::TokenFile {
path,
poll_interval_secs,
refresh_safety_window_secs,
} = &vault.auth_method
else {
panic!("token file in the environment must select TokenFile auth, got {:?}", vault.auth_method);
};
assert_eq!(path, std::path::Path::new("/run/vault-agent/token"));
assert_eq!(poll_interval_secs, &None);
assert_eq!(refresh_safety_window_secs, &None);
},
);
}
#[test]
fn test_from_env_token_file_is_mutually_exclusive_with_other_auth() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault")),
(ENV_KMS_VAULT_TOKEN_FILE, Some("/run/vault-agent/token")),
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
],
|| {
let error = KmsConfig::from_env().expect_err("token file combined with approle must be rejected");
assert!(error.to_string().contains(ENV_KMS_VAULT_TOKEN_FILE));
assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID));
},
);
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
(ENV_KMS_VAULT_TOKEN_FILE, Some("/run/vault-agent/token")),
("RUSTFS_KMS_VAULT_TOKEN", Some("vault-token")),
],
|| {
let error = KmsConfig::from_env().expect_err("token file combined with a static token must be rejected");
assert!(error.to_string().contains(ENV_KMS_VAULT_TOKEN_FILE));
assert!(error.to_string().contains("RUSTFS_KMS_VAULT_TOKEN"));
},
);
}
#[test]
fn test_validate_rejects_bad_token_file_settings() {
let vault_config = |auth_method: VaultAuthMethod| KmsConfig {
backend: KmsBackend::VaultKv2,
backend_config: BackendConfig::VaultKv2(Box::new(VaultConfig {
address: "https://vault.example.com:8200".to_string(),
auth_method,
..Default::default()
})),
..Default::default()
};
let error = vault_config(VaultAuthMethod::token_file(PathBuf::new()))
.validate()
.expect_err("empty token file path must be rejected");
assert!(error.to_string().contains("path"));
let error = vault_config(VaultAuthMethod::TokenFile {
path: PathBuf::from("/run/vault-agent/token"),
poll_interval_secs: Some(0),
refresh_safety_window_secs: None,
})
.validate()
.expect_err("zero poll interval must be rejected");
assert!(error.to_string().contains("poll interval"));
vault_config(VaultAuthMethod::token_file(PathBuf::from("/run/vault-agent/token")))
.validate()
.expect("well-formed token file auth must validate");
}
#[test]
fn test_approle_config_deserializes_legacy_shape_with_defaults() {
// Persisted configurations from before the AppRole implementation only
// carry role_id and secret_id; the new fields must fill with defaults.
let legacy = serde_json::json!({
"AppRole": {
"role_id": "legacy-role",
"secret_id": "legacy-secret-id",
}
});
let auth: VaultAuthMethod = serde_json::from_value(legacy).expect("legacy AppRole config must keep deserializing");
let VaultAuthMethod::AppRole {
role_id,
secret_id_file,
mount,
refresh_safety_window_secs,
..
} = auth
else {
panic!("expected AppRole");
};
assert_eq!(role_id, "legacy-role");
assert_eq!(secret_id_file, None);
assert_eq!(mount, DEFAULT_VAULT_APPROLE_MOUNT);
assert_eq!(refresh_safety_window_secs, None);
}
#[test]
fn test_validate_rejects_incomplete_approle() {
let mut config = KmsConfig::vault_approle(
Url::parse("https://vault.example.com:8200").expect("vault URL"),
String::new(),
"secret-id".to_string(),
);
let error = config.validate().expect_err("empty role_id must be rejected");
assert!(error.to_string().contains("role_id"));
config = KmsConfig::vault_approle(
Url::parse("https://vault.example.com:8200").expect("vault URL"),
"role-id".to_string(),
String::new(),
);
let error = config
.validate()
.expect_err("approle without secret_id or secret_id_file must be rejected");
assert!(error.to_string().contains("secret_id"));
}
#[test]
fn test_from_env_requires_vault_development_opt_in() {
with_vars(
crates/kms/src/deletion_worker.rs
+398
View File
@@ -0,0 +1,398 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Background worker that completes scheduled key deletions.
//!
//! Every sweep lists keys, picks the ones whose persisted deletion deadline
//! has passed (plus tombstones left by a crashed removal) and hands each to
//! [`KmsBackend::remove_expired_key`], which re-checks state under the
//! backend's own synchronization. The sweep is idempotent and keeps no state
//! of its own, so it is safe to re-run after a restart and safe to run on
//! every node of a deployment concurrently — a key is only ever removed while
//! its (re-read) record is an expired pending deletion or a tombstone.
use crate::backends::{ExpiredKeyRemoval, KmsBackend};
use crate::types::{KeyStatus, ListKeysRequest};
use async_trait::async_trait;
use jiff::Zoned;
use std::sync::Arc;
use std::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, warn};
/// How often the worker looks for expired pending deletions.
pub const DEFAULT_SWEEP_INTERVAL: Duration = Duration::from_secs(60);
/// Reports configuration that still references a KMS key.
///
/// Consulted before any material is destroyed; a non-empty result blocks the
/// removal until the references disappear. Implementations live where the
/// referencing configuration lives (for example bucket encryption settings in
/// the server) and are injected via
/// [`crate::service_manager::KmsServiceManager::set_deletion_reference_checker`].
#[async_trait]
pub trait DeletionReferenceChecker: Send + Sync {
/// Identifiers of configuration still referencing `key_id` (bucket names,
/// settings paths, ...). Errors must be reported as a reference so that
/// an unavailable checker never unblocks a deletion.
async fn references(&self, key_id: &str) -> Vec<String>;
}
/// Outcome of one sweep, for logging and tests.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct SweepReport {
/// Keys whose record and material were removed this sweep.
pub removed: Vec<String>,
/// Keys left in place because configuration still references them.
pub blocked: Vec<String>,
/// Keys that were pending but not yet due, without a persisted deadline,
/// or whose state changed between inspection and removal.
pub skipped: usize,
/// Keys whose removal attempt failed; retried on the next sweep.
pub failed: usize,
}
pub(crate) struct DeletionWorker {
backend: Arc<dyn KmsBackend>,
default_key_id: Option<String>,
reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
interval: Duration,
}
impl DeletionWorker {
pub(crate) fn new(
backend: Arc<dyn KmsBackend>,
default_key_id: Option<String>,
reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
) -> Self {
Self {
backend,
default_key_id,
reference_checker,
interval: DEFAULT_SWEEP_INTERVAL,
}
}
pub(crate) fn spawn(self, cancel: CancellationToken) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move { self.run(cancel).await })
}
async fn run(self, cancel: CancellationToken) {
let mut ticker = tokio::time::interval(self.interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
_ = cancel.cancelled() => {
debug!("KMS deletion worker stopped");
return;
}
_ = ticker.tick() => {}
}
let report = self.sweep(&Zoned::now()).await;
if !report.removed.is_empty() || !report.blocked.is_empty() || report.failed > 0 {
info!(
removed = ?report.removed,
blocked = ?report.blocked,
skipped = report.skipped,
failed = report.failed,
"KMS deletion sweep completed"
);
}
}
}
/// Run one sweep at the given time. Exposed separately so tests can drive
/// the expiry logic deterministically.
pub(crate) async fn sweep(&self, now: &Zoned) -> SweepReport {
let mut report = SweepReport::default();
let mut marker: Option<String> = None;
loop {
let request = ListKeysRequest {
limit: Some(100),
marker: marker.clone(),
usage_filter: None,
status_filter: None,
};
let response = match self.backend.list_keys(request).await {
Ok(response) => response,
Err(error) => {
warn!(%error, "KMS deletion sweep could not list keys");
report.failed += 1;
return report;
}
};
for key in &response.keys {
if matches!(key.status, KeyStatus::PendingDeletion | KeyStatus::Deleted) {
self.process_key(&key.key_id, now, &mut report).await;
}
}
if !response.truncated {
break;
}
match response.next_marker {
Some(next_marker) => marker = Some(next_marker),
None => break,
}
}
report
}
async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) {
// Never remove a key that live configuration still points at. The
// default key check is built in; broader references (bucket
// encryption settings, ...) come from the injected checker.
if self.default_key_id.as_deref() == Some(key_id) {
warn!(key_id, "expired KMS key is still the default key; refusing removal");
report.blocked.push(key_id.to_string());
return;
}
if let Some(checker) = &self.reference_checker {
let references = checker.references(key_id).await;
if !references.is_empty() {
warn!(key_id, ?references, "expired KMS key is still referenced; refusing removal");
report.blocked.push(key_id.to_string());
return;
}
}
// The backend re-checks state and deadline under its own write
// synchronization, so a cancellation racing this sweep wins there.
match self.backend.remove_expired_key(key_id, now).await {
Ok(ExpiredKeyRemoval::Removed) => report.removed.push(key_id.to_string()),
Ok(ExpiredKeyRemoval::StateChanged | ExpiredKeyRemoval::NotExpired) => report.skipped += 1,
Err(error) => {
warn!(key_id, %error, "failed to remove expired KMS key; will retry next sweep");
report.failed += 1;
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::backends::local::LocalKmsBackend;
use crate::config::KmsConfig;
use crate::error::KmsError;
use crate::types::{CreateKeyRequest, DeleteKeyRequest, DescribeKeyRequest, KeyState, KeyUsage};
async fn local_backend(temp_dir: &tempfile::TempDir) -> Arc<LocalKmsBackend> {
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
Arc::new(LocalKmsBackend::new(config).await.expect("local backend should build"))
}
async fn create_key(backend: &LocalKmsBackend, key_name: &str) -> String {
backend
.create_key(CreateKeyRequest {
key_name: Some(key_name.to_string()),
key_usage: KeyUsage::EncryptDecrypt,
..Default::default()
})
.await
.expect("key should be created")
.key_id
}
async fn schedule(backend: &LocalKmsBackend, key_id: &str) {
backend
.delete_key(DeleteKeyRequest {
key_id: key_id.to_string(),
pending_window_in_days: Some(7),
force_immediate: None,
})
.await
.expect("deletion should be scheduled");
}
fn worker(backend: Arc<LocalKmsBackend>) -> DeletionWorker {
DeletionWorker::new(backend, None, None)
}
fn after_window() -> Zoned {
Zoned::now() + Duration::from_secs(8 * 86400)
}
async fn assert_key_gone(backend: &LocalKmsBackend, key_id: &str) {
let error = backend
.describe_key(DescribeKeyRequest {
key_id: key_id.to_string(),
})
.await
.expect_err("removed key must not be describable");
assert!(matches!(error, KmsError::KeyNotFound { .. }), "expected KeyNotFound, got {error:?}");
}
#[tokio::test]
async fn sweep_removes_expired_pending_key_and_is_idempotent() {
let temp_dir = tempfile::tempdir().expect("temp dir");
let backend = local_backend(&temp_dir).await;
let key_id = create_key(&backend, "expired-key").await;
schedule(&backend, &key_id).await;
let worker = worker(backend.clone());
// Not yet due: nothing happens.
let report = worker.sweep(&Zoned::now()).await;
assert!(report.removed.is_empty());
assert_eq!(report.skipped, 1);
assert_eq!(report.failed, 0);
// Past the deadline: the key is removed.
let report = worker.sweep(&after_window()).await;
assert_eq!(report.removed, vec![key_id.clone()]);
assert_eq!(report.failed, 0);
assert_key_gone(&backend, &key_id).await;
// Re-running the sweep after the removal is a no-op.
let report = worker.sweep(&after_window()).await;
assert_eq!(report, SweepReport::default());
}
#[tokio::test]
async fn cancelled_deletion_always_beats_the_sweep() {
let temp_dir = tempfile::tempdir().expect("temp dir");
let backend = local_backend(&temp_dir).await;
let cancelled = create_key(&backend, "cancelled-key").await;
let doomed = create_key(&backend, "doomed-key").await;
schedule(&backend, &cancelled).await;
schedule(&backend, &doomed).await;
backend
.cancel_key_deletion(crate::types::CancelKeyDeletionRequest {
key_id: cancelled.clone(),
})
.await
.expect("cancel should succeed");
let report = worker(backend.clone()).sweep(&after_window()).await;
assert_eq!(report.removed, vec![doomed.clone()]);
assert_eq!(report.failed, 0);
// The cancelled key survives, enabled and usable.
let described = backend
.describe_key(DescribeKeyRequest {
key_id: cancelled.clone(),
})
.await
.expect("cancelled key must still exist");
assert_eq!(described.key_metadata.key_state, KeyState::Enabled);
assert_key_gone(&backend, &doomed).await;
}
#[tokio::test]
async fn default_key_and_external_references_block_removal() {
struct StaticReferences(Vec<String>);
#[async_trait]
impl DeletionReferenceChecker for StaticReferences {
async fn references(&self, _key_id: &str) -> Vec<String> {
self.0.clone()
}
}
let temp_dir = tempfile::tempdir().expect("temp dir");
let backend = local_backend(&temp_dir).await;
let key_id = create_key(&backend, "referenced-key").await;
schedule(&backend, &key_id).await;
// Blocked while it is the configured default key.
let as_default = DeletionWorker::new(backend.clone(), Some(key_id.clone()), None);
let report = as_default.sweep(&after_window()).await;
assert_eq!(report.blocked, vec![key_id.clone()]);
assert!(report.removed.is_empty());
// Blocked while external configuration still references it.
let with_references = DeletionWorker::new(
backend.clone(),
None,
Some(Arc::new(StaticReferences(vec!["bucket:sse-bucket".to_string()]))),
);
let report = with_references.sweep(&after_window()).await;
assert_eq!(report.blocked, vec![key_id.clone()]);
assert!(report.removed.is_empty());
backend
.describe_key(DescribeKeyRequest { key_id: key_id.clone() })
.await
.expect("blocked key must still exist");
// Removed once nothing references it anymore.
let unreferenced = DeletionWorker::new(backend.clone(), None, Some(Arc::new(StaticReferences(Vec::new()))));
let report = unreferenced.sweep(&after_window()).await;
assert_eq!(report.removed, vec![key_id.clone()]);
}
#[tokio::test]
async fn deadline_survives_backend_restart_and_sweep_completes_it() {
let temp_dir = tempfile::tempdir().expect("temp dir");
let key_id;
{
let backend = local_backend(&temp_dir).await;
key_id = create_key(&backend, "restart-key").await;
schedule(&backend, &key_id).await;
}
// "Restart": a fresh backend over the same directory must still see
// the persisted deadline...
let backend = local_backend(&temp_dir).await;
let described = backend
.describe_key(DescribeKeyRequest { key_id: key_id.clone() })
.await
.expect("key must survive the restart");
assert_eq!(described.key_metadata.key_state, KeyState::PendingDeletion);
assert!(
described.key_metadata.deletion_date.is_some(),
"deletion deadline must survive a backend restart"
);
// ...and the worker completes the deletion without any new schedule call.
let report = worker(backend.clone()).sweep(&after_window()).await;
assert_eq!(report.removed, vec![key_id.clone()]);
assert_key_gone(&backend, &key_id).await;
}
#[tokio::test(start_paused = true)]
async fn worker_loop_removes_due_keys_and_stops_on_cancel() {
let temp_dir = tempfile::tempdir().expect("temp dir");
let backend = local_backend(&temp_dir).await;
let key_id = create_key(&backend, "loop-key").await;
// A zero-day window through the lifecycle client produces a deadline
// that is already due for the worker's wall-clock sweep.
backend
.lifecycle_client()
.schedule_key_deletion(&key_id, 0, None)
.await
.expect("schedule with zero window");
let cancel = CancellationToken::new();
let task = worker(backend.clone()).spawn(cancel.clone());
// The paused clock auto-advances through the worker's interval ticks.
let mut removed = false;
for _ in 0..100 {
tokio::time::sleep(Duration::from_secs(1)).await;
if backend
.describe_key(DescribeKeyRequest { key_id: key_id.clone() })
.await
.is_err()
{
removed = true;
break;
}
}
assert!(removed, "worker loop must remove the due key");
cancel.cancel();
task.await.expect("worker task must stop after cancellation");
}
}

Some files were not shown because too many files have changed in this diff Show More