gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-04 19:25:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into overtrue/fix/lifecycle-rule-validation

Browse Source
This commit is contained in:
Zhengchao An
2026-09-04 23:04:01 +08:00
committed by GitHub
parent ef8f90be91 10ccf7c31a
commit 15b82db684
61 changed files with 10335 additions and 1425 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/e2e-nightly-selection.txt
+2 -1
View File
@@ -1 +1,2 @@
sha256=3da900ddcd7e2b9d3544bccfd69e468be5605a9fd2ea0ef5a6218304e42de5b4
sha256-darwin=a5665318c9bdc0947514fb7008ba1b83b114b739fac775c3c446f207058b7c7a
sha256-linux=45d80e1723de5d25bb5b81f3ef5c82f583efc3e4f036a8cd2bb99e4f1eca9e51
.config/nextest.toml
+2 -2
View File
@@ -69,7 +69,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_
setup = 'ecstore-large-stack'
[[profile.default.scripts]]
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
[[profile.default.scripts]]
@@ -210,7 +210,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_
setup = 'ecstore-large-stack'
[[profile.ci.scripts]]
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
Cargo.lock
Generated
+45 -74
View File
@@ -1328,9 +1328,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
version = "1.15.0"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "954c563ce84507722d2679f07a35d21b9c6466b3872d513020d0281fc8112ac9"
checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api-macros",
@@ -1368,9 +1368,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
version = "1.6.2"
version = "1.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fce83ce9abbb198d25bc7131e468d0f9fe1257125e58c39f3f9fc9f5098c9647"
checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9"
dependencies = [
"base64-simd",
"bytes",
@@ -5103,9 +5103,9 @@ dependencies = [
[[package]]
name = "hickory-net"
version = "0.26.1"
version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183"
checksum = "084e7bd6a377435d568f652153e571b50970d7ccc1d1eeec0519f834632287e1"
dependencies = [
"async-trait",
"cfg-if",
@@ -5127,9 +5127,9 @@ dependencies = [
[[package]]
name = "hickory-proto"
version = "0.26.1"
version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643"
checksum = "7e2da0694c15b44c6f68a6b05e0233617008c54080e31d6eb848d858a9c5b38d"
dependencies = [
"data-encoding",
"idna",
@@ -5147,9 +5147,9 @@ dependencies = [
[[package]]
name = "hickory-resolver"
version = "0.26.1"
version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c"
checksum = "0e4f9f4603319422d482738f3f6fe5aac03157fdbfed1cd85a3ff45adb09072f"
dependencies = [
"cfg-if",
"futures-util",
@@ -5226,9 +5226,9 @@ dependencies = [
[[package]]
name = "hotpath"
version = "0.24.0"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2645642a23d4061ec15a4a6e74f851a3145c3125356846cfc7772ff9c6f2737"
checksum = "2ec7782e005cabd5eaf350febde384cd799faa3a0e624587aa8759c240e0b592"
dependencies = [
"arc-swap",
"async-channel",
@@ -5240,7 +5240,6 @@ dependencies = [
"futures-util",
"hdrhistogram",
"hotpath-macros",
"hotpath-meta",
"http 1.5.0",
"libc",
"object 0.36.7",
@@ -5260,30 +5259,15 @@ dependencies = [
[[package]]
name = "hotpath-macros"
version = "0.24.0"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89a3d3cdf9b0d4d3d4f6d4a29798f3b9170401ba500eaa58dd8f890f926af0f1"
checksum = "929b2285d2cd21b2733a7fb6ebc843bb4f83dbd1db0122f5f9ebb9567b1e2613"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "hotpath-macros-meta"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e84cd2417fa60938241cf1cd6c03e09953f5c821122dc5da9b8f27975d136c5b"
[[package]]
name = "hotpath-meta"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d2c145b67b1a4e7bcefa918995e212c30a49a85f05cc5962fe1f717878d560b"
dependencies = [
"hotpath-macros-meta",
]
[[package]]
name = "htmlescape"
version = "0.3.1"
@@ -5642,18 +5626,6 @@ dependencies = [
"tempfile",
]
[[package]]
name = "internal-russh-num-bigint"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae8e22120c32fb4d19ec55fba35015f57095cd95a2e3b732e44457f5915b2ee8"
dependencies = [
"num-integer",
"num-traits",
"rand 0.10.2",
"rand_core 0.10.1",
]
[[package]]
name = "io-uring"
version = "0.7.14"
@@ -5904,17 +5876,20 @@ dependencies = [
[[package]]
name = "kafka-protocol"
version = "0.17.0"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66292444a1cd4d430d450d472c30cba839d0724229aba2d79affffcf901516e2"
checksum = "099d5c2f1b40cd830cbf18ca4d2a0805f2875b811ca18f0372b2433e68fe2dda"
dependencies = [
"anyhow",
"bytes",
"crc",
"crc32c",
"flate2",
"indexmap 2.14.1",
"paste",
"lz4",
"snap",
"uuid",
"zstd",
]
[[package]]
@@ -6923,6 +6898,8 @@ checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0"
dependencies = [
"num-integer",
"num-traits",
"rand 0.10.2",
"rand_core 0.10.1",
]
[[package]]
@@ -7515,9 +7492,9 @@ dependencies = [
[[package]]
name = "pageant"
version = "0.2.2"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3adadc44070da6f464b0918655a12f5792c156e088d8c4082d13e27d94c3e791"
checksum = "6d8eab09a361a4afe0b1668be978cd97e4f052e927a92b0b608cf902965d49ce"
dependencies = [
"base16ct 1.0.0",
"byteorder",
@@ -7606,12 +7583,6 @@ dependencies = [
"phc",
]
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "path-absolutize"
version = "4.0.1"
@@ -9250,9 +9221,9 @@ dependencies = [
[[package]]
name = "russh"
version = "0.63.1"
version = "0.63.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35bab1b87d915817d5d9cc352637cd40d5f0b298a48c6309af9156a4addc3031"
checksum = "8e134e2480f4e86f83e4aa45b4c0a9723f84beaffa694c54bdf056e74efdd7dd"
dependencies = [
"aes 0.9.3",
"aws-lc-rs",
@@ -9281,13 +9252,12 @@ dependencies = [
"hex-literal",
"hmac 0.13.0",
"inout 0.2.2",
"internal-russh-num-bigint",
"keccak",
"log",
"md5",
"ml-kem",
"module-lattice",
"num-bigint 0.4.8",
"num-bigint 0.5.1",
"p256 0.14.0",
"p384 0.14.0",
"p521",
@@ -9978,11 +9948,11 @@ dependencies = [
[[package]]
name = "rustfs-kafka"
version = "1.2.0"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4eee0644a99743fb2f51db7fbae1a6ca2d85f064daf7595784eaa52834a68c96"
checksum = "759f8ddf709b497006e0f422a89890f11ac86c84c2c39df282891477c22f1a13"
dependencies = [
"base64 0.22.1",
"base64 0.23.1",
"bytes",
"fnv",
"hmac 0.13.0",
@@ -9992,22 +9962,22 @@ dependencies = [
"pbkdf2 0.13.0",
"rand 0.10.2",
"rustls",
"rustls-native-certs",
"sha2 0.11.0",
"socket2",
"thiserror 2.0.20",
"tracing",
"twox-hash",
"uuid",
"webpki-roots 1.0.9",
]
[[package]]
name = "rustfs-kafka-async"
version = "1.2.0"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cd1997c3116cb94ede80d9a0b828f46dd27386cc93825fce141a92eb3aa9630"
checksum = "5aa574a13d91ef529e3a08f059a5c3ee8d4c3b16cf62f62c72595a6dc4f79f8f"
dependencies = [
"base64 0.22.1",
"base64 0.23.1",
"bytes",
"hmac 0.13.0",
"kafka-protocol",
@@ -10016,11 +9986,11 @@ dependencies = [
"rand 0.10.2",
"rustfs-kafka",
"rustls",
"rustls-native-certs",
"sha2 0.11.0",
"tokio",
"tokio-rustls",
"tracing",
"uuid",
"webpki-roots 1.0.9",
]
@@ -10194,18 +10164,18 @@ dependencies = [
[[package]]
name = "rustfs-mimalloc"
version = "0.5.2"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85d1a75bd188260754c4fa8ace1bb1f6a9956ee36c1d330f98fed5d25e39fa15"
checksum = "46a7b69356718defa4060de3059e609c9da6619c35924fefef55d902ccd94958"
dependencies = [
"rustfs-mimalloc-sys",
]
[[package]]
name = "rustfs-mimalloc-sys"
version = "0.5.2"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d3adbf24cbe37f040f856c04f8d4baf75626eacd4e8b17eb6b3d79de7fde692"
checksum = "1dd3ec9b7e7b9fed453acd8b4e32713ddc0e01e1aee31a3deec5b8025c0880c3"
dependencies = [
"cc",
]
@@ -12301,7 +12271,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.3.4",
"getrandom 0.4.3",
"once_cell",
"rustix",
"windows-sys 0.61.2",
@@ -12485,9 +12455,9 @@ dependencies = [
[[package]]
name = "tinyvec"
version = "1.12.0"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f"
checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b"
dependencies = [
"tinyvec_macros",
]
@@ -13652,13 +13622,14 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
[[package]]
name = "wnaf"
version = "0.14.0"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1"
checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa"
dependencies = [
"ff 0.14.0",
"group 0.14.0",
"hybrid-array",
"primefield",
]
[[package]]
Cargo.toml
+6 -6
View File
@@ -165,7 +165,7 @@ http-body = "1.1.0"
http-body-util = "0.1.5"
minlz = "1.2.3"
reqwest = "0.13.4"
rustfs-kafka-async = { version = "1.2.0" }
rustfs-kafka-async = { version = "1.3.1" }
socket2 = { version = "0.6.5" }
tokio = { version = "1.53.1" }
tokio-rustls = { default-features = false, version = "0.26.4" }
@@ -244,8 +244,8 @@ aws-sdk-kms = { default-features = false, version = "1.117.0" }
aws-sdk-s3 = { default-features = false, version = "1.144.0" }
aws-sdk-sts = { default-features = false, version = "1.113.0" }
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
aws-smithy-runtime-api = { version = "1.15.0" }
aws-smithy-types = { version = "1.6.2" }
aws-smithy-runtime-api = { version = "1.16.0" }
aws-smithy-types = { version = "1.6.3" }
base64-simd = "0.8.0"
brotli = "9.0.0"
clap = { version = "4.6.6" }
@@ -359,15 +359,15 @@ libunftp = { version = "0.23.0" }
unftp-core = "0.1.0"
suppaftp = { version = "11.0.0" }
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.63.1" }
russh = { version = "0.63.2" }
russh-sftp = "2.4.0"
# WebDAV
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.5.2" }
hotpath = { version = "0.24.0", default-features = false }
rustfs-mimalloc = { version = "0.5.3" }
hotpath = { version = "0.25.0", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.48" }
crates/e2e_test/src/reliant/grpc_lock_server.rs
+7
View File
@@ -856,6 +856,13 @@ impl NodeService for MinimalLockNodeService {
Err(Status::unimplemented("lock-only test server"))
}
async fn scanner_dirty_usage_snapshot(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ScannerDirtyUsageSnapshotRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ScannerDirtyUsageSnapshotResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn background_heal_status(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::BackgroundHealStatusRequest>,
crates/ecstore/src/api/mod.rs
+9 -2
View File
@@ -253,6 +253,13 @@ pub mod bucket {
};
}
pub mod sealed_credentials {
pub use crate::bucket::sealed_credentials::{
CredentialSealer, SEALED_CREDENTIAL_VERSION, SealScope, SealedCredential, SealedCredentialError,
SealedCredentialStore, credential_sealer, install_credential_sealer, seal_secret, unseal_secret,
};
}
pub mod replication {
pub use crate::bucket::replication::replication_pool::{
DurableMrfBacklogSummary, DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBacklogObservabilitySummary,
@@ -526,8 +533,8 @@ pub mod rpc {
pub use crate::cluster::rpc::{
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
ScannerBucketListing, ScannerPeerActivity, ScannerPublicationLease, TONIC_RPC_PREFIX, TonicInterceptor,
build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
ScannerBucketListing, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TONIC_RPC_PREFIX,
TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options,
encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs
+139
View File
@@ -16,6 +16,7 @@ use super::runtime_boundary as runtime_sources;
use crate::bucket::lifecycle::lifecycle;
use crate::object_api::ObjectInfo;
use crate::services::event_notification::{EventArgs, send_event};
use crate::storage_api_contracts::object::{DeletedObject, ObjectToDelete};
use rustfs_s3_types::EventName;
use rustfs_scanner_metrics::metrics::IlmAction;
@@ -76,6 +77,60 @@ pub(crate) fn emit_non_transitioned_expiration_event(action: IlmAction, source:
emit_lifecycle_event(event_name, deleted, LIFECYCLE_EXPIRY_USER_AGENT);
}
/// Emit the lifecycle expiration event for one version removed by the batch
/// `NewerNoncurrentVersions` expiry path.
///
/// That path never sent events, so a successful noncurrent-version expiry was
/// invisible to notification subscribers even though the equivalent
/// current-version path emits one (backlog#2202).
pub(crate) fn emit_noncurrent_expiration_event(bucket: &str, target: &ObjectToDelete, deleted: &DeletedObject, failed: bool) {
if let Some((event_name, object)) = noncurrent_expiration_event(bucket, target, deleted, failed) {
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT);
}
}
/// Decide which event a single batch entry earned, if any.
///
/// Only an entry that mutated something may be announced. "No error" is not
/// enough, and neither is `found`: the disk layer skips an absent version and
/// reports success (`delete_versions_internal` in `disk/local.rs` continues
/// past `FileVersionNotFound`), so a batch entry for a version that was
/// already gone comes back indistinguishable from a committed delete. The
/// delete plan's own source lookup is the signal that survives that, and the
/// lifecycle batch path always performs it because every target carries an
/// exact version identity.
fn noncurrent_expiration_event(
bucket: &str,
target: &ObjectToDelete,
deleted: &DeletedObject,
failed: bool,
) -> Option<(EventName, ObjectInfo)> {
if failed || !deleted.found || deleted.source_missing {
return None;
}
// A version removed by explicit version id is a plain versioned delete
// even when that version is itself a delete marker; only a request that
// carried no version id can publish a new delete marker. This is the rule
// the S3 DeleteObjects path applies (issue #6745). `delete_object_versions`
// now refuses targets without an exact version identity, so the
// marker-creation shape is unreachable from that caller; the mapping stays
// here so a future caller cannot silently announce the wrong mutation.
let created_delete_marker = deleted.delete_marker && target.version_id.is_none();
let (event_name, version_id) = if created_delete_marker {
(EventName::LifecycleExpirationDeleteMarkerCreated, deleted.delete_marker_version_id)
} else {
(EventName::LifecycleExpirationDelete, deleted.version_id.or(target.version_id))
};
let object = ObjectInfo {
bucket: bucket.to_string(),
name: target.object_name.clone(),
version_id,
delete_marker: deleted.delete_marker,
..Default::default()
};
Some((event_name, object))
}
fn emit_lifecycle_event(event_name: EventName, object: ObjectInfo, user_agent: &str) {
send_event(EventArgs {
event_name: event_name.to_string(),
@@ -113,6 +168,7 @@ fn non_transitioned_expiration_event_name(
#[cfg(test)]
mod tests {
use super::*;
use uuid::Uuid;
#[test]
fn transitioned_expiration_event_marks_delete_marker_creation() {
@@ -129,4 +185,87 @@ mod tests {
EventName::LifecycleExpirationDelete
);
}
fn deleted_version(version_id: Uuid) -> DeletedObject {
DeletedObject {
object_name: "object".to_string(),
version_id: Some(version_id),
found: true,
..Default::default()
}
}
fn target_version(version_id: Option<Uuid>) -> ObjectToDelete {
ObjectToDelete {
object_name: "object".to_string(),
version_id,
..Default::default()
}
}
#[test]
fn noncurrent_expiration_emits_versioned_delete_with_exact_identity() {
let version_id = Uuid::new_v4();
let (event_name, object) =
noncurrent_expiration_event("bucket", &target_version(Some(version_id)), &deleted_version(version_id), false)
.expect("a committed delete must emit");
assert_eq!(event_name, EventName::LifecycleExpirationDelete);
assert_eq!(object.bucket, "bucket");
assert_eq!(object.name, "object");
assert_eq!(object.version_id, Some(version_id));
}
/// Removing a noncurrent version that happens to be a delete marker is a
/// plain versioned delete, not a delete-marker creation.
#[test]
fn noncurrent_expiration_of_a_delete_marker_version_is_a_plain_delete() {
let version_id = Uuid::new_v4();
let deleted = DeletedObject {
delete_marker: true,
..deleted_version(version_id)
};
let (event_name, object) = noncurrent_expiration_event("bucket", &target_version(Some(version_id)), &deleted, false)
.expect("a committed delete must emit");
assert_eq!(event_name, EventName::LifecycleExpirationDelete);
assert_eq!(object.version_id, Some(version_id));
}
#[test]
fn noncurrent_expiration_reports_a_created_delete_marker() {
let marker_version_id = Uuid::new_v4();
let deleted = DeletedObject {
object_name: "object".to_string(),
delete_marker: true,
delete_marker_version_id: Some(marker_version_id),
found: true,
..Default::default()
};
let (event_name, object) =
noncurrent_expiration_event("bucket", &target_version(None), &deleted, false).expect("a committed delete must emit");
assert_eq!(event_name, EventName::LifecycleExpirationDeleteMarkerCreated);
assert_eq!(object.version_id, Some(marker_version_id));
}
/// A batch mixes successes with failures and versions that were already
/// gone; only a real mutation may produce an event. A version that was
/// already gone comes back with no error and `found` set, so
/// `source_missing` is the signal that keeps it silent.
#[test]
fn noncurrent_expiration_skips_failed_and_missing_versions() {
let version_id = Uuid::new_v4();
let target = target_version(Some(version_id));
assert!(noncurrent_expiration_event("bucket", &target, &deleted_version(version_id), true).is_none());
let absent = DeletedObject {
source_missing: true,
..deleted_version(version_id)
};
assert!(noncurrent_expiration_event("bucket", &target, &absent, false).is_none());
let not_found = DeletedObject {
found: false,
..deleted_version(version_id)
};
assert!(noncurrent_expiration_event("bucket", &target, &not_found, false).is_none());
}
}
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+420 -108
View File
@@ -87,16 +87,12 @@ use rustfs_filemeta::{
use rustfs_scanner_metrics::metrics::{
IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
};
use rustfs_utils::{
get_env_i64, get_env_usize,
path::encode_dir_object,
string::{parse_bool, strings_has_prefix_fold},
};
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::parse_bool};
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
Timestamp,
};
use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
use s3s::header::X_AMZ_RESTORE;
use sha2::{Digest, Sha256};
use std::any::Any;
use std::collections::{BTreeMap, HashMap, HashSet};
@@ -165,7 +161,6 @@ pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
pub const AMZ_ENCRYPTION_AES: &str = "AES256";
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
@@ -2777,7 +2772,8 @@ async fn run_tier_free_version_recovery_loop<F, Fut>(
match recovery_result {
Ok(stats) => {
let elapsed = started_at.elapsed();
schedule.record_success(&stats, elapsed);
schedule.record_success(&stats);
rustfs_io_metrics::record_stage_duration("lifecycle_free_version_recovery", elapsed.as_secs_f64() * 1000.0);
let (pending_tasks, active_tasks) = {
let state = expiry_state.read().await;
(state.pending_tasks(), state.stats.active_tasks())
@@ -2806,7 +2802,7 @@ async fn run_tier_free_version_recovery_loop<F, Fut>(
}
Err(err) => {
let elapsed = started_at.elapsed();
schedule.record_failure(elapsed);
schedule.record_failure();
rustfs_io_metrics::record_stage_duration(
"lifecycle_free_version_recovery_failed",
elapsed.as_secs_f64() * 1000.0,
@@ -2838,10 +2834,10 @@ async fn wait_for_tier_free_version_recovery(
} else {
schedule.next_delay
};
let sleep_delay = next_delay.saturating_sub(schedule.previous_run_duration);
schedule.previous_run_duration = StdDuration::ZERO;
// Recovery delays are completion-relative. Discounting the previous run
// would let a page that took at least one interval restart with no cooldown.
schedule.jitter_next_delay = false;
let sleep = tokio::time::sleep(sleep_delay);
let sleep = tokio::time::sleep(next_delay);
tokio::pin!(sleep);
let mut recovery_request_consumed = false;
@@ -2891,7 +2887,6 @@ struct TierFreeVersionRecoverySchedule {
next_delay: StdDuration,
idle_interval: StdDuration,
failure_interval: StdDuration,
previous_run_duration: StdDuration,
jitter_next_delay: bool,
bucket_marker: Option<String>,
object_marker: Option<String>,
@@ -2904,7 +2899,6 @@ impl Default for TierFreeVersionRecoverySchedule {
next_delay: StdDuration::ZERO,
idle_interval: TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL,
failure_interval: TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL,
previous_run_duration: StdDuration::ZERO,
jitter_next_delay: false,
bucket_marker: None,
object_marker: None,
@@ -2917,7 +2911,6 @@ impl TierFreeVersionRecoverySchedule {
fn reset_idle_interval(&mut self) {
self.idle_interval = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
self.next_delay = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
self.previous_run_duration = StdDuration::ZERO;
self.jitter_next_delay = false;
}
@@ -2928,18 +2921,15 @@ impl TierFreeVersionRecoverySchedule {
self.reset_idle_interval();
}
fn record_failure(&mut self, _run_duration: StdDuration) {
fn record_failure(&mut self) {
self.idle_interval = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
self.next_delay = self.failure_interval;
self.failure_interval =
std::cmp::min(self.failure_interval.saturating_mul(2), TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL);
// Keep the full backoff even after a long failed run, so a run whose
// duration exceeds the interval cannot restart immediately.
self.previous_run_duration = StdDuration::ZERO;
self.jitter_next_delay = false;
}
fn record_success(&mut self, stats: &FreeVersionRecoveryStats, run_duration: StdDuration) {
fn record_success(&mut self, stats: &FreeVersionRecoveryStats) {
self.failure_interval = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
if stats.enqueued > 0 || stats.failed > 0 {
self.follow_up_sweep = true;
@@ -2949,7 +2939,6 @@ impl TierFreeVersionRecoverySchedule {
self.object_marker = stats.next_object_marker.clone();
if stats.truncated {
self.next_delay = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
self.previous_run_duration = run_duration;
self.jitter_next_delay = false;
return;
}
@@ -2960,13 +2949,11 @@ impl TierFreeVersionRecoverySchedule {
self.follow_up_sweep = false;
self.idle_interval = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
self.next_delay = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL;
self.previous_run_duration = run_duration;
self.jitter_next_delay = false;
return;
}
self.next_delay = self.idle_interval;
self.previous_run_duration = run_duration;
self.jitter_next_delay = true;
self.idle_interval = std::cmp::min(self.idle_interval.saturating_mul(2), TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL);
}
@@ -4810,24 +4797,24 @@ fn attach_tier_operation_lease(mut reader: GetObjectReader, lease: TierOperation
reader
}
pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result<ObjectOptions, std::io::Error> {
/// Resolve the RestoreObject request options.
///
/// Returns the typed [`StorageError`]: flattening these into an opaque
/// `io::Error` string erased the identity the S3 layer needs to answer
/// InvalidArgument instead of a generic 500 (backlog#2205).
pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result<ObjectOptions, Error> {
let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
let vid = version_id.trim();
if !vid.is_empty() && vid != NULL_VERSION_ID {
if let Err(_err) = Uuid::parse_str(vid) {
return Err(std::io::Error::other(
StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()).to_string(),
));
return Err(StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()));
}
if !versioned && !version_suspended {
return Err(std::io::Error::other(
StorageError::InvalidArgument(
bucket.to_string(),
object.to_string(),
format!("version-id specified {} but versioning is not enabled on {}", vid, bucket),
)
.to_string(),
return Err(StorageError::InvalidArgument(
bucket.to_string(),
object.to_string(),
format!("version-id specified {vid} but versioning is not enabled on {bucket}"),
));
}
}
@@ -4880,43 +4867,18 @@ pub async fn put_restore_opts(
}
meta.insert(X_AMZ_STORAGE_CLASS.as_str().to_lowercase(), sc);*/
if let Some(type_) = &rreq.type_
&& type_.as_str() == RestoreRequestType::SELECT
// A SELECT restore must never reach the restore writer: the caller writes
// the retrieved bytes back to the source bucket/object, so building
// SELECT output options here produced a source overwrite carrying only
// the OutputLocation metadata instead of a write to `OutputLocation.S3`
// (backlog#1341). RestoreObject rejects SELECT at the API boundary; this
// is the fail-closed backstop for any other caller.
if rreq
.type_
.as_ref()
.is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
{
let Some(s3) = select_restore_s3_location(rreq)? else {
return Err(std::io::Error::other("OutputLocation.S3 required for SELECT requests"));
};
if let Some(user_metadata) = s3.user_metadata.as_ref() {
for metadata in user_metadata {
let name = metadata
.name
.as_deref()
.ok_or_else(|| std::io::Error::other("SELECT restore metadata name is required"))?;
let value = metadata.value.clone().unwrap_or_default();
if strings_has_prefix_fold(name, "x-amz-meta") {
meta.insert(name.to_string(), value);
} else {
meta.insert(format!("x-amz-meta-{name}"), value);
}
}
}
if let Some(tags) = &s3.tagging {
meta.insert(
AMZ_OBJECT_TAGGING.to_string(),
serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()),
);
}
if let Some(encryption) = &s3.encryption
&& encryption.encryption_type.as_str() != ""
{
meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string());
}
return Ok(ObjectOptions {
versioned: BucketVersioningSys::prefix_enabled(bucket, object).await,
version_suspended: BucketVersioningSys::prefix_suspended(bucket, object).await,
user_defined: meta,
..Default::default()
});
return Err(std::io::Error::other("SELECT restore requests are not supported"));
}
for (k, v) in oi.user_defined.iter() {
meta.insert(k.to_string(), v.clone());
@@ -5623,8 +5585,9 @@ mod tests {
use crate::bucket::lifecycle::replication_sink::{ReplicationStatusType, VersionPurgeStatusType};
use crate::bucket::lifecycle::runtime_boundary as runtime_sources;
use crate::bucket::lifecycle::tier_free_version_recovery::{
FreeVersionRecoveryStats, RecoveryWalkTestAction, list_tier_free_versions, recover_tier_free_versions_with_cancel,
set_recovery_bucket_list_wait_hook, set_recovery_walk_test_hook,
FreeVersionRecoveryStats, RecoveryWalkTestAction, RecoveryWorkBudget, list_tier_free_versions,
list_tier_free_versions_with_budget, recover_tier_free_versions_with_cancel, set_recovery_bucket_list_wait_hook,
set_recovery_walk_test_hook,
};
use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
use crate::bucket::lifecycle::tier_sweeper::Jentry;
@@ -5712,7 +5675,7 @@ mod tests {
assert_eq!(schedule.next_delay, StdDuration::ZERO);
for expected in [60, 120, 240, 480, 600, 600, 600, 600] {
schedule.record_success(&idle, StdDuration::ZERO);
schedule.record_success(&idle);
assert_eq!(schedule.next_delay, StdDuration::from_secs(expected));
}
assert_eq!(schedule.next_delay.as_secs() / TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL.as_secs(), 10);
@@ -5723,15 +5686,13 @@ mod tests {
let mut schedule = TierFreeVersionRecoverySchedule::default();
for expected in [60, 120, 240, 480, 600, 600] {
schedule.record_failure(StdDuration::from_secs(75));
schedule.record_failure();
assert_eq!(schedule.next_delay, StdDuration::from_secs(expected));
assert_eq!(schedule.previous_run_duration, StdDuration::ZERO);
}
schedule.record_success(&free_version_recovery_stats(0, 0, false), StdDuration::ZERO);
schedule.record_failure(StdDuration::from_secs(75));
schedule.record_success(&free_version_recovery_stats(0, 0, false));
schedule.record_failure();
assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
assert_eq!(schedule.previous_run_duration, StdDuration::ZERO);
}
#[test]
@@ -5746,22 +5707,22 @@ mod tests {
fn tier_free_version_recovery_pagination_preserves_full_sweep_backoff() {
let idle = free_version_recovery_stats(0, 0, false);
let mut schedule = TierFreeVersionRecoverySchedule::default();
schedule.record_success(&idle, StdDuration::ZERO);
schedule.record_success(&idle, StdDuration::ZERO);
schedule.record_success(&idle);
schedule.record_success(&idle);
assert_eq!(schedule.next_delay, StdDuration::from_secs(120));
assert_eq!(schedule.idle_interval, StdDuration::from_secs(240));
schedule.record_success(&free_version_recovery_stats(0, 0, true), StdDuration::ZERO);
schedule.record_success(&free_version_recovery_stats(0, 0, true));
assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
assert_eq!(schedule.idle_interval, StdDuration::from_secs(240));
assert_eq!(schedule.bucket_marker.as_deref(), Some("bucket"));
assert_eq!(schedule.object_marker.as_deref(), Some("object"));
schedule.record_success(&free_version_recovery_stats(0, 0, true), StdDuration::ZERO);
schedule.record_success(&free_version_recovery_stats(0, 0, true));
assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
assert_eq!(schedule.idle_interval, StdDuration::from_secs(240));
schedule.record_success(&idle, StdDuration::ZERO);
schedule.record_success(&idle);
assert_eq!(schedule.next_delay, StdDuration::from_secs(240));
assert_eq!(schedule.idle_interval, StdDuration::from_secs(480));
assert!(schedule.bucket_marker.is_none());
@@ -5771,7 +5732,7 @@ mod tests {
#[test]
fn tier_free_version_recovery_wake_during_pagination_keeps_one_full_follow_up() {
let mut schedule = TierFreeVersionRecoverySchedule::default();
schedule.record_success(&free_version_recovery_stats(0, 0, true), StdDuration::ZERO);
schedule.record_success(&free_version_recovery_stats(0, 0, true));
schedule.request_retry();
schedule.request_retry();
@@ -5779,7 +5740,7 @@ mod tests {
assert_eq!(schedule.bucket_marker.as_deref(), Some("bucket"));
assert_eq!(schedule.object_marker.as_deref(), Some("object"));
schedule.record_success(&free_version_recovery_stats(0, 0, false), StdDuration::ZERO);
schedule.record_success(&free_version_recovery_stats(0, 0, false));
assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
assert!(!schedule.follow_up_sweep);
assert!(schedule.bucket_marker.is_none());
@@ -5794,22 +5755,22 @@ mod tests {
free_version_recovery_stats(0, 1, true),
] {
let mut schedule = TierFreeVersionRecoverySchedule::default();
schedule.record_success(&idle, StdDuration::ZERO);
schedule.record_success(&idle, StdDuration::ZERO);
schedule.record_success(&idle);
schedule.record_success(&idle);
assert_eq!(schedule.idle_interval, StdDuration::from_secs(240));
schedule.record_success(&work, StdDuration::ZERO);
schedule.record_success(&work);
assert!(schedule.follow_up_sweep);
assert_eq!(schedule.idle_interval, StdDuration::from_secs(240));
assert!(!schedule.jitter_next_delay);
schedule.record_success(&idle, StdDuration::ZERO);
schedule.record_success(&idle);
assert!(!schedule.follow_up_sweep);
assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
assert_eq!(schedule.idle_interval, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
assert!(!schedule.jitter_next_delay);
schedule.record_success(&idle, StdDuration::ZERO);
schedule.record_success(&idle);
assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
assert_eq!(schedule.idle_interval, StdDuration::from_secs(120));
}
@@ -5823,17 +5784,17 @@ mod tests {
free_version_recovery_stats(0, 1, false),
] {
let mut schedule = TierFreeVersionRecoverySchedule::default();
schedule.record_success(&idle, StdDuration::ZERO);
schedule.record_success(&idle, StdDuration::ZERO);
schedule.record_success(&idle);
schedule.record_success(&idle);
assert_eq!(schedule.idle_interval, StdDuration::from_secs(240));
schedule.record_success(&work, StdDuration::ZERO);
schedule.record_success(&work);
assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
assert_eq!(schedule.idle_interval, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
assert!(!schedule.follow_up_sweep);
assert!(!schedule.jitter_next_delay);
schedule.record_success(&idle, StdDuration::ZERO);
schedule.record_success(&idle);
assert_eq!(schedule.next_delay, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL);
assert_eq!(schedule.idle_interval, StdDuration::from_secs(120));
}
@@ -5972,7 +5933,7 @@ mod tests {
);
}
async fn tier_free_version_recovery_page_call_times(run_duration: StdDuration) -> Vec<StdDuration> {
async fn tier_free_version_recovery_page_call_times(run_durations: &[StdDuration]) -> Vec<StdDuration> {
RECOVERY_JITTER_CALLS.store(0, Ordering::SeqCst);
let cancel = CancellationToken::new();
let state = ExpiryState::new();
@@ -5981,6 +5942,7 @@ mod tests {
let recorded_call_times = Arc::clone(&call_times);
let call_index = Arc::new(AtomicUsize::new(0));
let recorded_call_index = Arc::clone(&call_index);
let recovery_run_durations = run_durations.to_vec();
let loop_cancel = cancel.clone();
let recovery_cancel = cancel.clone();
let worker = tokio::spawn(async move {
@@ -5991,8 +5953,9 @@ mod tests {
.push(tokio::time::Instant::now().duration_since(started_at));
let index = recorded_call_index.fetch_add(1, Ordering::SeqCst);
let cancel = recovery_cancel.clone();
let run_duration = recovery_run_durations.get(index).copied();
async move {
if index == 0 {
if let Some(run_duration) = run_duration {
tokio::time::sleep(run_duration).await;
Ok(free_version_recovery_stats(0, 0, true))
} else {
@@ -6005,15 +5968,14 @@ mod tests {
});
tokio::task::yield_now().await;
tokio::time::advance(run_duration).await;
tokio::task::yield_now().await;
if run_duration < TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL {
let remaining = TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL - run_duration;
tokio::time::advance(remaining - StdDuration::from_secs(1)).await;
assert_eq!(call_index.load(Ordering::SeqCst), 1);
for (index, run_duration) in run_durations.iter().copied().enumerate() {
tokio::time::advance(run_duration).await;
tokio::task::yield_now().await;
tokio::time::advance(TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL - StdDuration::from_secs(1)).await;
assert_eq!(call_index.load(Ordering::SeqCst), index + 1);
tokio::time::advance(StdDuration::from_secs(1)).await;
tokio::task::yield_now().await;
}
tokio::task::yield_now().await;
worker.await.expect("recovery loop should stop after cancellation");
assert_eq!(
RECOVERY_JITTER_CALLS.load(Ordering::SeqCst),
@@ -6029,14 +5991,75 @@ mod tests {
#[tokio::test(start_paused = true)]
#[serial]
async fn tier_free_version_recovery_preserves_start_to_start_page_cadence() {
async fn tier_free_version_recovery_waits_after_each_page_completes() {
assert_eq!(
tier_free_version_recovery_page_call_times(StdDuration::from_secs(45)).await,
vec![StdDuration::ZERO, StdDuration::from_secs(60)]
tier_free_version_recovery_page_call_times(&[StdDuration::from_secs(45)]).await,
vec![StdDuration::ZERO, StdDuration::from_secs(105)]
);
assert_eq!(
tier_free_version_recovery_page_call_times(StdDuration::from_secs(75)).await,
vec![StdDuration::ZERO, StdDuration::from_secs(75)]
tier_free_version_recovery_page_call_times(&[StdDuration::from_secs(60)]).await,
vec![StdDuration::ZERO, StdDuration::from_secs(120)]
);
assert_eq!(
tier_free_version_recovery_page_call_times(&[StdDuration::from_secs(75)]).await,
vec![StdDuration::ZERO, StdDuration::from_secs(135)]
);
assert_eq!(
tier_free_version_recovery_page_call_times(&[StdDuration::from_secs(75), StdDuration::from_secs(45)]).await,
vec![StdDuration::ZERO, StdDuration::from_secs(135), StdDuration::from_secs(240)]
);
}
#[tokio::test(start_paused = true)]
async fn tier_free_version_recovery_notify_during_page_keeps_completion_cooldown() {
let cancel = CancellationToken::new();
let state = ExpiryState::new();
let recovery_notify = Arc::clone(&state.read().await.recovery_notify);
let started_at = tokio::time::Instant::now();
let call_times = Arc::new(StdMutex::new(Vec::new()));
let recorded_call_times = Arc::clone(&call_times);
let call_index = Arc::new(AtomicUsize::new(0));
let recorded_call_index = Arc::clone(&call_index);
let loop_cancel = cancel.clone();
let recovery_cancel = cancel.clone();
let worker = tokio::spawn(async move {
run_tier_free_version_recovery_loop(loop_cancel, state, std::convert::identity, move |_, _, _| {
recorded_call_times
.lock()
.expect("recovery call times lock should not be poisoned")
.push(tokio::time::Instant::now().duration_since(started_at));
let index = recorded_call_index.fetch_add(1, Ordering::SeqCst);
let cancel = recovery_cancel.clone();
async move {
if index == 0 {
tokio::time::sleep(StdDuration::from_secs(75)).await;
Ok(free_version_recovery_stats(0, 0, true))
} else {
cancel.cancel();
Ok(free_version_recovery_stats(0, 0, false))
}
}
})
.await;
});
tokio::task::yield_now().await;
tokio::time::advance(StdDuration::from_secs(30)).await;
recovery_notify.notify_one();
tokio::time::advance(StdDuration::from_secs(45)).await;
tokio::task::yield_now().await;
tokio::time::advance(StdDuration::from_secs(59)).await;
assert_eq!(call_index.load(Ordering::SeqCst), 1);
tokio::time::advance(StdDuration::from_secs(1)).await;
tokio::task::yield_now().await;
worker.await.expect("recovery loop should stop after cancellation");
assert_eq!(
call_times
.lock()
.expect("recovery call times lock should not be poisoned")
.as_slice(),
&[StdDuration::ZERO, StdDuration::from_secs(135)]
);
}
@@ -11283,6 +11306,120 @@ mod tests {
}
}
/// backlog#2202: the batch `NewerNoncurrentVersions` path used to delete
/// noncurrent versions without telling notification subscribers anything,
/// while the current-version path emitted a lifecycle expiration event.
/// Only versions this batch actually removed may produce an event.
#[tokio::test]
#[serial]
async fn lifecycle_noncurrent_batch_expiry_emits_events_only_for_committed_deletes() {
use crate::services::event_notification::test_recorder;
use rustfs_s3_types::EventName;
let (_disk_paths, ecstore) = setup_test_env().await;
let bucket = format!("lifecycle-noncurrent-events-{}", Uuid::new_v4().simple());
let object = "logs/object";
create_test_bucket(&ecstore, &bucket).await;
metadata_sys::update_in(
&ecstore.ctx,
&bucket,
BUCKET_VERSIONING_CONFIG,
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
)
.await
.expect("bucket versioning should be enabled");
let now = OffsetDateTime::now_utc();
let mut noncurrent_reader = PutObjReader::from_vec(b"noncurrent".to_vec());
let noncurrent = ecstore
.put_object(
&bucket,
object,
&mut noncurrent_reader,
&ObjectOptions {
versioned: true,
mod_time: Some(now - time::Duration::days(40)),
..Default::default()
},
)
.await
.expect("the noncurrent version should be created");
let noncurrent_version_id = noncurrent.version_id.expect("a versioned PUT has an exact identity");
let mut current_reader = PutObjReader::from_vec(b"current".to_vec());
let current = ecstore
.put_object(
&bucket,
object,
&mut current_reader,
&ObjectOptions {
versioned: true,
mod_time: Some(now - time::Duration::days(2)),
..Default::default()
},
)
.await
.expect("the current version should be created");
let current_version_id = current.version_id.expect("a versioned PUT has an exact identity");
let incarnation = ecstore
.bucket_incarnation_id_from_disk(&bucket)
.await
.expect("bucket incarnation should be available");
test_recorder::install();
// One version that exists and one that never did: `delete_objects`
// suppresses the not-found error, so only the committed delete may be
// announced.
let missing_version_id = Uuid::new_v4();
let targets = vec![
ObjectToDelete {
object_name: object.to_string(),
version_id: Some(noncurrent_version_id),
..Default::default()
},
ObjectToDelete {
object_name: object.to_string(),
version_id: Some(missing_version_id),
..Default::default()
},
];
let failed = crate::bucket::lifecycle::object_handlers_common::delete_object_versions(
&ecstore,
&bucket,
&targets,
lifecycle::Event::default(),
incarnation,
)
.await;
assert_eq!(failed, 0, "a missing version is not a batch failure");
let events = test_recorder::recorded_for_bucket(&bucket);
let announced = events.iter().map(|event| event.version_id).collect::<Vec<_>>();
assert_eq!(
announced,
vec![Some(noncurrent_version_id)],
"only the committed noncurrent delete should be announced \
(noncurrent={noncurrent_version_id}, current={current_version_id}, missing={missing_version_id}), got {events:?}"
);
assert_eq!(events[0].event_name, EventName::LifecycleExpirationDelete.to_string());
assert_eq!(events[0].object, object);
assert!(!events[0].delete_marker);
let remaining = ecstore
.clone()
.list_object_versions(&bucket, object, None, None, None, 10)
.await
.expect("remaining versions should be listable")
.objects
.into_iter()
.filter(|candidate| candidate.name == object)
.collect::<Vec<_>>();
assert_eq!(remaining.len(), 1);
assert_eq!(remaining[0].version_id, Some(current_version_id));
}
#[tokio::test]
#[serial]
async fn lifecycle_deletes_only_the_historical_null_version_after_versioning_is_reenabled() {
@@ -12557,6 +12694,181 @@ mod tests {
.expect("empty recovery test bucket should be removed");
}
#[tokio::test]
#[serial]
async fn tier_free_version_recovery_object_budget_is_global_across_buckets() {
let (_disk_paths, ecstore) = setup_test_env().await;
for budget in [
RecoveryWorkBudget {
max_objects: 0,
max_buckets: 1,
},
RecoveryWorkBudget {
max_objects: 1,
max_buckets: 0,
},
] {
let err = list_tier_free_versions_with_budget(Arc::clone(&ecstore), 1, None, None, CancellationToken::new(), budget)
.await
.expect_err("a zero recovery work budget must fail before starting an unbounded walk");
assert!(matches!(err, Error::Io(_)));
}
let suffix = Uuid::new_v4().simple();
let buckets = [
format!("zzzz-recovery-budget-{suffix}-a"),
format!("zzzz-recovery-budget-{suffix}-b"),
format!("zzzz-recovery-budget-{suffix}-c"),
];
for bucket in &buckets {
create_test_bucket(&ecstore, bucket).await;
}
let first_bucket = buckets[0].clone();
let first_bucket_for_hook = first_bucket.clone();
let _walk = set_recovery_walk_test_hook(move |bucket| {
(bucket == first_bucket_for_hook).then(|| {
RecoveryWalkTestAction::SendItems(
["ordinary-object-0", "ordinary-object-0", "ordinary-object-1"]
.into_iter()
.map(|name| ObjectInfo {
bucket: first_bucket_for_hook.clone(),
name: name.to_string(),
..Default::default()
})
.collect(),
)
})
});
let page = list_tier_free_versions_with_budget(
Arc::clone(&ecstore),
1,
None,
None,
CancellationToken::new(),
RecoveryWorkBudget {
max_objects: 1,
max_buckets: 10,
},
)
.await
.expect("the decoded-object boundary should enforce the recovery budget");
assert!(page.items.is_empty());
assert_eq!(page.scanned_entries, 3);
assert_eq!(page.buckets_scanned, 1);
assert!(page.truncated);
assert_eq!(page.next_bucket_marker.as_deref(), Some(first_bucket.as_str()));
assert_eq!(page.next_object_marker.as_deref(), Some("ordinary-object-0"));
drop(_walk);
let walked = Arc::new(StdMutex::new(Vec::new()));
let walked_by_hook = Arc::clone(&walked);
let test_buckets = buckets.clone();
let _walk = set_recovery_walk_test_hook(move |bucket| {
test_buckets
.iter()
.find(|candidate| candidate.as_str() == bucket)
.map(|bucket| {
walked_by_hook
.lock()
.expect("recovery walk log should not be poisoned")
.push(bucket.clone());
RecoveryWalkTestAction::SendItems(vec![ObjectInfo {
bucket: bucket.clone(),
name: "ordinary-object".to_string(),
..Default::default()
}])
})
});
let page = list_tier_free_versions_with_budget(
Arc::clone(&ecstore),
1,
None,
None,
CancellationToken::new(),
RecoveryWorkBudget {
max_objects: 2,
max_buckets: 10,
},
)
.await
.expect("the bounded recovery page should be listed");
assert!(page.items.is_empty());
assert_eq!(page.scanned_entries, 2);
assert_eq!(page.buckets_scanned, 2);
assert!(page.truncated);
assert_eq!(page.next_bucket_marker.as_deref(), Some(buckets[1].as_str()));
assert_eq!(page.next_object_marker.as_deref(), Some("ordinary-object"));
assert_eq!(walked.lock().expect("recovery walk log should not be poisoned").as_slice(), &buckets[..2]);
for bucket in &buckets {
ecstore
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("empty recovery test bucket should be removed");
}
}
#[tokio::test]
#[serial]
async fn tier_free_version_recovery_bucket_budget_resumes_at_unscanned_bucket() {
let (_disk_paths, ecstore) = setup_test_env().await;
let suffix = Uuid::new_v4().simple();
let buckets = [
format!("zzzz-recovery-buckets-{suffix}-a"),
format!("zzzz-recovery-buckets-{suffix}-b"),
];
for bucket in &buckets {
create_test_bucket(&ecstore, bucket).await;
}
let test_buckets = buckets.clone();
let _walk = set_recovery_walk_test_hook(move |bucket| {
test_buckets
.iter()
.any(|candidate| candidate.as_str() == bucket)
.then(|| RecoveryWalkTestAction::SendItems(Vec::new()))
});
let budget = RecoveryWorkBudget {
max_objects: 10,
max_buckets: 1,
};
let first = list_tier_free_versions_with_budget(Arc::clone(&ecstore), 1, None, None, CancellationToken::new(), budget)
.await
.expect("the first bucket-bounded page should be listed");
assert_eq!(first.buckets_scanned, 1);
assert!(first.truncated);
assert_eq!(first.next_bucket_marker.as_deref(), Some(buckets[1].as_str()));
assert!(first.next_object_marker.is_none());
let second = list_tier_free_versions_with_budget(
Arc::clone(&ecstore),
1,
first.next_bucket_marker,
first.next_object_marker,
CancellationToken::new(),
budget,
)
.await
.expect("the second bucket-bounded page should resume");
assert_eq!(second.buckets_scanned, 1);
assert!(!second.truncated);
assert!(second.next_bucket_marker.is_none());
assert!(second.next_object_marker.is_none());
for bucket in &buckets {
ecstore
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("empty recovery test bucket should be removed");
}
}
#[tokio::test]
#[serial]
async fn tier_free_version_recovery_propagates_walk_item_error() {
crates/ecstore/src/bucket/lifecycle/durable_namespace.rs
+217 -9
View File
@@ -182,6 +182,16 @@ pub(crate) enum DurableIlmRecordCheckpoint {
identity_sha256: String,
state: tier_delete_journal::TierDeleteDispatchManifestState,
},
TierDeleteDispatchParent {
content_sha256: String,
identity_sha256: String,
revision: u64,
next_chunk_sequence: u64,
completed_journal_count: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
active_chunk_identity_sha256: Option<String>,
completed: bool,
},
TransitionTransaction {
content_sha256: String,
identity_sha256: String,
@@ -220,6 +230,7 @@ impl DurableIlmRecordCheckpoint {
match self {
Self::TierDeleteJournal { content_sha256, .. }
| Self::TierDeleteDispatchManifest { content_sha256, .. }
| Self::TierDeleteDispatchParent { content_sha256, .. }
| Self::TransitionTransaction { content_sha256, .. }
| Self::ManualTransitionJob { content_sha256, .. }
| Self::ManualTransitionScope { content_sha256, .. }
@@ -341,6 +352,52 @@ impl DurableIlmRecordCheckpoint {
(Preparing, DispatchAuthorized | Aborting) | (Aborting, Aborted) | (DispatchAuthorized, Completed)
)
}
(
Self::TierDeleteDispatchParent {
identity_sha256: previous_identity,
revision: previous_revision,
next_chunk_sequence: previous_sequence,
completed_journal_count: previous_completed_journals,
active_chunk_identity_sha256: previous_active,
completed: previous_completed,
..
},
Self::TierDeleteDispatchParent {
identity_sha256: next_identity,
revision: next_revision,
next_chunk_sequence: next_sequence,
completed_journal_count: next_completed_journals,
active_chunk_identity_sha256: next_active,
completed: next_completed,
..
},
) => {
let Some((sequence_delta, completed_journal_delta)) = tier_delete_dispatch_parent_progress_delta(
*previous_sequence,
*previous_completed_journals,
*next_sequence,
*next_completed_journals,
) else {
return Err(Error::other("durable ILM record generation is not a monotonic successor"));
};
let same_position_transition = sequence_delta == 0
&& completed_journal_delta == 0
&& matches!(
(previous_active.as_ref(), next_active.as_ref(), previous_completed, next_completed),
(None, Some(_), false, false) | (Some(_), None, false, false) | (None, None, false, true)
);
let progress_transition = sequence_delta > 0
&& completed_journal_delta > 0
&& !matches!(
(previous_active.as_ref(), next_active.as_ref()),
(Some(previous), Some(next)) if previous == next
);
previous_identity == next_identity
&& !previous_completed
&& next_revision > previous_revision
&& (!next_completed || next_active.is_none())
&& (same_position_transition || progress_transition)
}
(
Self::TransitionTransaction {
identity_sha256: previous_identity,
@@ -475,11 +532,58 @@ impl DurableIlmRecordCheckpoint {
..
},
) => previous_identity == terminal_identity,
(
Self::TierDeleteDispatchParent {
identity_sha256: previous_identity,
revision: previous_revision,
next_chunk_sequence: previous_sequence,
completed_journal_count: previous_completed_journals,
active_chunk_identity_sha256: previous_active,
completed: false,
..
},
Self::TierDeleteDispatchParent {
identity_sha256: terminal_identity,
revision: terminal_revision,
next_chunk_sequence: terminal_sequence,
completed_journal_count: terminal_completed_journals,
active_chunk_identity_sha256: None,
completed: true,
..
},
) => {
previous_identity == terminal_identity
&& terminal_revision > previous_revision
&& tier_delete_dispatch_parent_progress_delta(
*previous_sequence,
*previous_completed_journals,
*terminal_sequence,
*terminal_completed_journals,
)
.is_some_and(|(sequence_delta, completed_journal_delta)| {
if sequence_delta == 0 && completed_journal_delta == 0 {
previous_active.is_none()
} else {
sequence_delta > 0 && completed_journal_delta > 0
}
})
}
_ => false,
}
}
}
fn tier_delete_dispatch_parent_progress_delta(
previous_sequence: u64,
previous_completed_journals: u64,
next_sequence: u64,
next_completed_journals: u64,
) -> Option<(u64, u64)> {
let sequence_delta = next_sequence.checked_sub(previous_sequence)?;
let completed_journal_delta = next_completed_journals.checked_sub(previous_completed_journals)?;
(sequence_delta <= completed_journal_delta).then_some((sequence_delta, completed_journal_delta))
}
fn transition_state_distance(
from: transition_transaction::TransitionTransactionState,
to: transition_transaction::TransitionTransactionState,
@@ -913,17 +1017,42 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
)
}
DurableIlmRecordKind::TierDeleteDispatchManifest => {
let (operation_id, identity_sha256, state) =
tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)?;
(
"operation_id",
hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
DurableIlmRecordCheckpoint::TierDeleteDispatchManifest {
content_sha256,
match tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)? {
tier_delete_journal::TierDeleteDispatchDurableRecord::Manifest {
operation_id,
identity_sha256,
state,
},
)
} => (
"operation_id",
hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
DurableIlmRecordCheckpoint::TierDeleteDispatchManifest {
content_sha256,
identity_sha256,
state,
},
),
tier_delete_journal::TierDeleteDispatchDurableRecord::Parent {
operation_id,
identity_sha256,
revision,
next_chunk_sequence,
completed_journal_count,
active_chunk_identity_sha256,
completed,
} => (
"operation_id",
hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
DurableIlmRecordCheckpoint::TierDeleteDispatchParent {
content_sha256,
identity_sha256,
revision,
next_chunk_sequence,
completed_journal_count,
active_chunk_identity_sha256,
completed,
},
),
}
}
DurableIlmRecordKind::TransitionTransaction => {
let transaction = transition_transaction::decode_transition_transaction_record(path, data)
@@ -1143,6 +1272,85 @@ mod tests {
assert!(aborted.validate_successor(&preparing).is_err());
}
#[test]
fn tier_delete_dispatch_parent_checkpoint_is_monotonic_across_chunks() {
let identity = "a".repeat(64);
let checkpoint = |revision, sequence, completed_journals, active: Option<&str>, completed| {
DurableIlmRecordCheckpoint::TierDeleteDispatchParent {
content_sha256: format!("{revision:064x}"),
identity_sha256: identity.clone(),
revision,
next_chunk_sequence: sequence,
completed_journal_count: completed_journals,
active_chunk_identity_sha256: active.map(ToOwned::to_owned),
completed,
}
};
let idle = checkpoint(0, 0, 0, None, false);
let first_child = "b".repeat(64);
let second_child = "c".repeat(64);
let bound = checkpoint(1, 0, 0, Some(&first_child), false);
let advanced = checkpoint(2, 1, 2, None, false);
let next_bound = checkpoint(3, 1, 2, Some(&second_child), false);
let completed = checkpoint(4, 2, 3, None, true);
let terminal_after_more_chunks = checkpoint(6, 4, 7, None, true);
idle.validate_successor(&bound).expect("an idle parent may bind one child");
bound
.validate_successor(&advanced)
.expect("a completed child may advance the parent sequence");
advanced
.validate_successor(&next_bound)
.expect("the next sequence may bind a new immutable child");
next_bound
.validate_successor(&completed)
.expect("receipt progress may skip directly to a later terminal checkpoint");
assert!(
bound.is_predecessor_of_terminal(&terminal_after_more_chunks),
"terminal cleanup may still recognize a valid multi-chunk predecessor"
);
assert!(
advanced.is_predecessor_of_terminal(&terminal_after_more_chunks),
"terminal cleanup may still skip over later valid parent generations"
);
assert!(
idle.validate_successor(&checkpoint(1, 0, 1, Some(&first_child), false))
.is_err()
);
assert!(bound.validate_successor(&checkpoint(2, 1, 0, None, false)).is_err());
assert!(
bound.validate_successor(&checkpoint(2, 2, 1, None, false)).is_err(),
"sequence cannot advance beyond completed journal evidence"
);
assert!(
advanced.validate_successor(&checkpoint(3, 1, 3, None, false)).is_err(),
"completed journal count cannot grow without a completed child sequence"
);
assert!(
bound.validate_successor(&checkpoint(2, 0, 0, None, true)).is_err(),
"an active child cannot be marked completed without completion evidence"
);
assert!(
bound
.validate_successor(&checkpoint(2, 0, 0, Some(&second_child), false))
.is_err(),
"an active child cannot be replaced at the same parent sequence"
);
assert!(
bound
.validate_successor(&checkpoint(2, 1, 1, Some(&first_child), false))
.is_err(),
"sequence growth cannot retain the same active child identity"
);
assert!(
!bound.is_predecessor_of_terminal(&checkpoint(2, 0, 0, None, true)),
"terminal cleanup must not treat an active child as completed without count evidence"
);
assert!(completed.validate_successor(&checkpoint(5, 3, 4, None, true)).is_err());
assert!(completed.validate_successor(&advanced).is_err());
assert!(advanced.validate_successor(&idle).is_err());
}
#[test]
fn tier_delete_journal_checkpoint_binds_dispatch_and_full_state_monotonically() {
use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs
+7
View File
@@ -20,6 +20,7 @@ const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
const EVENT_LIFECYCLE_CLEANUP_SKIPPED: &str = "lifecycle_cleanup_skipped";
const EVENT_LIFECYCLE_CLEANUP_FAILED: &str = "lifecycle_cleanup_failed";
use crate::bucket::lifecycle::bucket_lifecycle_audit::emit_noncurrent_expiration_event;
use crate::bucket::lifecycle::lifecycle;
use crate::bucket::lifecycle::replication_sink::{self, ReplicationObjectBridge};
use crate::object_api::ObjectOptions;
@@ -98,6 +99,12 @@ pub async fn delete_object_versions(
// version so it does not sit resident until TTL (ODC-26).
if let Some(target) = to_del.get(i) {
crate::object_api::notify_object_mutation(bucket, &target.object_name).await;
// Announce the version this batch actually removed. Cache
// eviction and replication scheduling keep their existing
// order and admission; the event is derived from the committed
// result, and a send failure never rolls back a delete that
// already happened (backlog#2202).
emit_noncurrent_expiration_event(bucket, target, deleted_obj, false);
}
if deleted_obj.replication_state.is_none() {
continue;
crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs
+1764 -290
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs
+77 -3
View File
@@ -32,7 +32,10 @@ use crate::store::ECStore;
use rustfs_filemeta::FileInfo;
pub const DEFAULT_FREE_VERSION_RECOVERY_LIMIT: usize = 1_000;
// These are page-wide repair budgets. Applying them per bucket would still let
// one recovery pass walk an unbounded namespace before the scheduler can cool down.
const DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT: usize = 10_000;
const DEFAULT_FREE_VERSION_RECOVERY_BUCKET_LIMIT: usize = 100;
#[cfg(not(test))]
const BACKGROUND_WALK_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
#[cfg(test)]
@@ -41,6 +44,12 @@ const BACKGROUND_WALK_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(100);
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, crate::error::Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct RecoveryWorkBudget {
pub(super) max_objects: usize,
pub(super) max_buckets: usize,
}
fn recovery_walk_options(limit: usize, marker: Option<String>) -> WalkOptions {
WalkOptions {
include_free_versions: true,
@@ -58,6 +67,7 @@ fn recovery_walk_options(limit: usize, marker: Option<String>) -> WalkOptions {
#[cfg(test)]
pub(super) enum RecoveryWalkTestAction {
SendItems(Vec<ObjectInfo>),
SendItemsThenError(Vec<ObjectInfo>, crate::error::Error),
SendItemsThenHang(Vec<ObjectInfo>, Arc<tokio::sync::Notify>),
SendItemsUntilReceiverCloses(Arc<tokio::sync::Notify>),
@@ -277,6 +287,17 @@ pub(super) async fn list_tier_free_versions(
bucket_marker: Option<String>,
object_marker: Option<String>,
cancel_token: CancellationToken,
) -> Result<FreeVersionRecoveryPage> {
list_tier_free_versions_with_budget(api, limit, bucket_marker, object_marker, cancel_token, recovery_work_budget(limit)).await
}
pub(super) async fn list_tier_free_versions_with_budget(
api: Arc<ECStore>,
limit: usize,
bucket_marker: Option<String>,
object_marker: Option<String>,
cancel_token: CancellationToken,
work_budget: RecoveryWorkBudget,
) -> Result<FreeVersionRecoveryPage> {
let mut page = FreeVersionRecoveryPage {
items: Vec::new(),
@@ -290,6 +311,9 @@ pub(super) async fn list_tier_free_versions(
if limit == 0 {
return Ok(page);
}
if work_budget.max_objects == 0 || work_budget.max_buckets == 0 {
return Err(std::io::Error::other("free-version recovery work budget must be greater than zero").into());
}
let bucket_options = BucketOptions::default();
let list_buckets = async {
@@ -313,7 +337,7 @@ pub(super) async fn list_tier_free_versions(
};
let mut bucket_seen = bucket_marker.is_none();
let mut truncated_after: Option<RecoveryCursor> = None;
let walk_scan_limit = recovery_walk_scan_limit(limit);
let mut remaining_scan_objects = work_budget.max_objects;
for bucket in buckets {
if cancel_token.is_cancelled() {
@@ -329,12 +353,20 @@ pub(super) async fn list_tier_free_versions(
bucket_seen = true;
}
if page.buckets_scanned >= work_budget.max_buckets {
page.truncated = true;
page.next_bucket_marker = Some(bucket.name);
page.next_object_marker = None;
break;
}
page.buckets_scanned += 1;
let bucket_object_marker = if bucket_marker.as_deref() == Some(bucket.name.as_str()) {
object_marker.clone()
} else {
None
};
let bucket_walk_limit = remaining_scan_objects;
let (tx, mut rx) = mpsc::channel::<ObjectInfoOrErr>(100);
let cancel = cancel_token.child_token();
@@ -358,6 +390,21 @@ pub(super) async fn list_tier_free_versions(
#[cfg(test)]
if let Some(action) = test_action {
match action {
RecoveryWalkTestAction::SendItems(items) => {
for item in items {
if tx
.send(ObjectInfoOrErr {
item: Some(item),
err: None,
})
.await
.is_err()
{
return Ok(());
}
}
return Ok(());
}
RecoveryWalkTestAction::SendItemsThenError(items, err) => {
for item in items {
if tx
@@ -425,7 +472,7 @@ pub(super) async fn list_tier_free_versions(
}
}
api.walk(cancel, &bucket_name, "", tx, recovery_walk_options(walk_scan_limit, object_marker))
api.walk(cancel, &bucket_name, "", tx, recovery_walk_options(bucket_walk_limit, object_marker))
.await
}
});
@@ -466,6 +513,18 @@ pub(super) async fn list_tier_free_versions(
let Some(oi) = item.item else {
continue;
};
if last_seen_object.as_deref() != Some(oi.name.as_str()) && scanned_objects >= bucket_walk_limit {
// The disk listing limit follows S3-visible counting rules and
// does not charge metadata containing only hidden/free versions.
// Enforce the repair budget again at the decoded-object boundary.
page.truncated = true;
page.next_bucket_marker = Some(bucket.name.clone());
page.next_object_marker = last_seen_object.clone();
cancel.cancel();
draining_after_truncation = true;
drain_deadline = Some(tokio::time::Instant::now() + BACKGROUND_WALK_SHUTDOWN_TIMEOUT);
continue;
}
record_scanned_object(&mut last_seen_object, &mut scanned_objects, &oi.name);
if let Some(cursor) = &truncated_after
&& (cursor.bucket.as_str() != bucket.name.as_str() || cursor.object.as_str() != oi.name.as_str())
@@ -509,7 +568,8 @@ pub(super) async fn list_tier_free_versions(
return Err(err);
}
walk_result?;
mark_scan_truncated_if_needed(&mut page, scanned_objects, walk_scan_limit, &bucket.name, last_seen_object.as_deref());
remaining_scan_objects = remaining_scan_objects.saturating_sub(scanned_objects);
mark_scan_truncated_if_needed(&mut page, scanned_objects, bucket_walk_limit, &bucket.name, last_seen_object.as_deref());
if page.truncated {
break;
@@ -528,6 +588,13 @@ fn recovery_walk_scan_limit(limit: usize) -> usize {
DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT.max(limit.saturating_add(1))
}
fn recovery_work_budget(limit: usize) -> RecoveryWorkBudget {
RecoveryWorkBudget {
max_objects: recovery_walk_scan_limit(limit),
max_buckets: DEFAULT_FREE_VERSION_RECOVERY_BUCKET_LIMIT,
}
}
fn record_scanned_object(last_seen_object: &mut Option<String>, scanned_objects: &mut usize, object: &str) {
if last_seen_object.as_deref() == Some(object) {
return;
@@ -700,6 +767,13 @@ mod tests {
recovery_walk_scan_limit(DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT),
DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT + 1
);
assert_eq!(
recovery_work_budget(DEFAULT_FREE_VERSION_RECOVERY_LIMIT),
RecoveryWorkBudget {
max_objects: DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT,
max_buckets: DEFAULT_FREE_VERSION_RECOVERY_BUCKET_LIMIT,
}
);
}
#[test]
crates/ecstore/src/bucket/mod.rs
+1
View File
@@ -31,6 +31,7 @@ pub mod policy_sys;
pub mod quota;
pub mod remote_s3_client;
pub mod replication;
pub mod sealed_credentials;
pub mod tagging;
pub mod target;
pub mod utils;
crates/ecstore/src/bucket/on_demand_migration/source_client.rs
+203 -65
View File
@@ -14,9 +14,9 @@
//! Outbound client for an on-demand migration source bucket.
//!
//! `SourceClient` wraps an `aws_sdk_s3::Client` built through the shared
//! remote builder and exposes the read-only surface the migration path
//! needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a
//! `SourceClient` maps local keys onto a read-only `SourceBackend`. The
//! S3 backend uses the shared remote builder and exposes the surface the
//! migration path needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a
//! probe for admin validation). Every request carries the
//! `source-proxy-request` anti-loop marker in both the `x-rustfs-` and
//! `x-minio-` prefixes so a RustFS/MinIO source answers locally instead of
@@ -511,7 +511,7 @@ pub struct SourceObject {
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct SourcePage {
pub objects: Vec<SourceObject>,
/// Rolled-up prefixes, in the local namespace; always empty when the
/// Rolled-up prefixes, in the same namespace as `objects`; always empty when the
/// request carried no delimiter.
pub common_prefixes: Vec<String>,
pub is_truncated: bool,
@@ -519,7 +519,8 @@ pub struct SourcePage {
}
/// One `ListObjectsV2` page request against the source. Keys are given in the
/// local namespace; `SourceClient` maps them through `source_prefix`.
/// local namespace at `SourceClient`, and in the source namespace at
/// `SourceBackend`; `SourceClient` maps them through `source_prefix`.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct SourceListRequest<'a> {
pub prefix: Option<&'a str>,
@@ -575,8 +576,30 @@ impl Intercept for SourceProxyMarkerInterceptor {
}
}
pub struct SourceClient {
/// Read-only provider operations in the source bucket namespace.
///
/// Implementations must preserve streaming, honor the requested range and
/// pagination cursor, and classify failures without including credentials.
/// `SourceClient` owns prefix mapping so every provider shares the same local
/// namespace. Continuation tokens are opaque and must never be prefix-mapped.
#[async_trait::async_trait]
pub trait SourceBackend: Send + Sync {
async fn head(&self, key: &str) -> Result<SourceHead, SourceError>;
async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError>;
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError>;
async fn tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError>;
/// Verify bucket access; `SourceClient` separately probes a filtered listing.
async fn probe(&self) -> Result<(), SourceError>;
}
/// S3-compatible implementation, including request signing and anti-loop headers.
pub struct S3SourceBackend {
client: S3Client,
bucket: String,
}
pub struct SourceClient {
backend: Box<dyn SourceBackend>,
endpoint: String,
bucket: String,
source_prefix: Option<String>,
@@ -609,7 +632,10 @@ impl SourceClient {
fn from_config_builder(config: aws_sdk_s3::config::Builder, endpoint: String, spec: &SourceClientSpec) -> Self {
let client = S3Client::from_conf(config.interceptor(SourceProxyMarkerInterceptor::new()).build());
Self {
client,
backend: Box::new(S3SourceBackend {
client,
bucket: spec.bucket.clone(),
}),
endpoint,
bucket: spec.bucket.clone(),
source_prefix: spec.source_prefix.clone().filter(|prefix| !prefix.is_empty()),
@@ -652,35 +678,15 @@ impl SourceClient {
}
pub async fn head_object(&self, key: &str) -> Result<SourceHead, SourceError> {
let output = self
.client
.head_object()
.bucket(&self.bucket)
.key(self.source_key(key))
.send()
.await
.map_err(classify_sdk_error)?;
source_head_from_head_output(output)
self.backend.head(&self.source_key(key)).await
}
/// Streams the object; `range` is passed through as an HTTP `Range`
/// header and omitted entirely when `None`.
/// Streams the object, preserving an optional HTTP byte range.
pub async fn get_object(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
let range = range.map(range_header_value).transpose()?;
let output = self
.client
.get_object()
.bucket(&self.bucket)
.key(self.source_key(key))
.set_range(range)
.send()
.await
.map_err(classify_sdk_error)?;
source_get_from_output(output)
self.backend.get(&self.source_key(key), range).await
}
/// Lists one page under the local `prefix`. Keys are returned in the
/// local namespace; entries outside `source_prefix` are skipped.
/// Lists one page under the local prefix.
pub async fn list_objects_v2(
&self,
prefix: Option<&str>,
@@ -696,9 +702,81 @@ impl SourceClient {
.await
}
/// [`Self::list_objects_v2`] with the delimiter and start-after the
/// list-through merge needs (rustfs/backlog#2164).
/// Maps keys and common prefixes while leaving opaque cursors untouched.
pub async fn list_page(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
let prefix = self.source_key(request.prefix.unwrap_or_default());
let start_after = request.start_after.map(|key| self.source_key(key));
let mut page = self
.backend
.list(&SourceListRequest {
prefix: Some(&prefix),
start_after: start_after.as_deref(),
..*request
})
.await?;
page.objects = page
.objects
.into_iter()
.filter_map(|object| self.local_object(object))
.collect();
page.common_prefixes = page
.common_prefixes
.into_iter()
.filter_map(|prefix| self.local_key(&prefix).map(str::to_string))
.collect();
Ok(page)
}
fn local_object(&self, mut object: SourceObject) -> Option<SourceObject> {
object.key = self.local_key(&object.key)?.to_string();
Some(object)
}
pub async fn get_object_tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
self.backend.tagging(&self.source_key(key)).await
}
pub async fn probe(&self) -> Result<SourceProbe, SourceError> {
self.backend.probe().await?;
let page = self.list_objects_v2(None, None, 1).await?;
Ok(SourceProbe {
sample_object: page.objects.into_iter().next(),
has_more_objects: page.is_truncated,
})
}
}
#[async_trait::async_trait]
impl SourceBackend for S3SourceBackend {
async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
let output = self
.client
.head_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
.map_err(classify_sdk_error)?;
source_head_from_head_output(output)
}
/// Streams the object; `range` is passed through as an HTTP `Range`
/// header and omitted entirely when `None`.
async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
let range = range.map(range_header_value).transpose()?;
let output = self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.set_range(range)
.send()
.await
.map_err(classify_sdk_error)?;
source_get_from_output(output)
}
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
// `start_after` is silently ignored by S3 once a continuation token is
// present; refuse the ambiguous pair rather than list from the wrong
// position.
@@ -711,9 +789,9 @@ impl SourceClient {
.client
.list_objects_v2()
.bucket(&self.bucket)
.prefix(self.source_key(request.prefix.unwrap_or_default()))
.prefix(request.prefix.unwrap_or_default())
.set_delimiter(request.delimiter.map(str::to_string))
.set_start_after(request.start_after.map(|after| self.source_key(after)))
.set_start_after(request.start_after.map(str::to_string))
.set_continuation_token(request.continuation_token.map(str::to_string))
.max_keys(request.max_keys)
.send()
@@ -731,13 +809,13 @@ impl SourceClient {
.contents
.unwrap_or_default()
.into_iter()
.filter_map(|object| self.source_object(object))
.filter_map(s3_source_object)
.collect();
let common_prefixes = output
.common_prefixes
.unwrap_or_default()
.into_iter()
.filter_map(|prefix| Some(self.local_key(prefix.prefix.as_deref()?)?.to_string()))
.filter_map(|prefix| prefix.prefix)
.collect();
Ok(SourcePage {
@@ -748,48 +826,43 @@ impl SourceClient {
})
}
fn source_object(&self, object: SdkObject) -> Option<SourceObject> {
let key = self.local_key(object.key.as_deref()?)?.to_string();
let etag = normalize_etag(object.e_tag);
let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
Some(SourceObject {
key,
etag,
size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0),
last_modified: system_time(object.last_modified),
storage_class: object.storage_class.map(|class| class.as_str().to_string()),
is_multipart_etag,
})
}
pub async fn get_object_tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
async fn tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
let output = self
.client
.get_object_tagging()
.bucket(&self.bucket)
.key(self.source_key(key))
.key(key)
.send()
.await
.map_err(classify_sdk_error)?;
Ok(output.tag_set.into_iter().map(|tag| (tag.key, tag.value)).collect())
}
/// Admin validation: HeadBucket plus a one-key listing under the prefix.
pub async fn probe(&self) -> Result<SourceProbe, SourceError> {
async fn probe(&self) -> Result<(), SourceError> {
self.client
.head_bucket()
.bucket(&self.bucket)
.send()
.await
.map_err(classify_sdk_error)?;
let page = self.list_objects_v2(None, None, 1).await?;
Ok(SourceProbe {
sample_object: page.objects.into_iter().next(),
has_more_objects: page.is_truncated,
})
Ok(())
}
}
fn s3_source_object(object: SdkObject) -> Option<SourceObject> {
let key = object.key?;
let etag = normalize_etag(object.e_tag);
let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
Some(SourceObject {
key,
etag,
size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0),
last_modified: system_time(object.last_modified),
storage_class: object.storage_class.map(|class| class.as_str().to_string()),
is_multipart_etag,
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1194,6 +1267,68 @@ mod tests {
assert!(requests[1].uri.contains("continuation-token=token-1"), "{}", requests[1].uri);
}
#[tokio::test]
async fn list_page_maps_delimiter_prefixes_and_start_after_but_not_cursors() {
let body = r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<IsTruncated>true</IsTruncated><NextContinuationToken>data/opaque</NextContinuationToken>
<CommonPrefixes><Prefix>data/photos/</Prefix></CommonPrefixes>
<CommonPrefixes><Prefix>outside/</Prefix></CommonPrefixes>
</ListBucketResult>"#;
let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await;
let first = client
.list_page(&SourceListRequest {
prefix: Some("photos/"),
delimiter: Some("/"),
start_after: Some("photos/a"),
max_keys: 2,
..Default::default()
})
.await
.expect("delimiter listing should succeed");
assert_eq!(first.common_prefixes, vec!["photos/"]);
assert_eq!(first.next_continuation_token.as_deref(), Some("data/opaque"));
let second = client
.list_page(&SourceListRequest {
continuation_token: first.next_continuation_token.as_deref(),
max_keys: 2,
..Default::default()
})
.await
.expect("opaque continuation should succeed");
assert_eq!(second.common_prefixes, first.common_prefixes);
let requests = recorded(&requests);
let query = |request: &RecordedRequest| {
Url::parse(&request.uri)
.expect("request URI")
.query_pairs()
.into_owned()
.collect::<HashMap<_, _>>()
};
let first_query = query(&requests[0]);
assert_eq!(first_query.get("prefix").map(String::as_str), Some("data/photos/"));
assert_eq!(first_query.get("start-after").map(String::as_str), Some("data/photos/a"));
assert_eq!(first_query.get("delimiter").map(String::as_str), Some("/"));
let second_query = query(&requests[1]);
assert_eq!(second_query.get("continuation-token").map(String::as_str), Some("data/opaque"));
assert!(!second_query.contains_key("start-after"));
}
#[tokio::test]
async fn list_page_rejects_ambiguous_cursor_before_sending() {
let (client, requests) = scripted_client(&spec(Some("data/")), vec![]).await;
let err = client
.list_page(&SourceListRequest {
start_after: Some("a"),
continuation_token: Some("opaque"),
max_keys: 1,
..Default::default()
})
.await
.expect_err("ambiguous list position must fail");
assert!(matches!(err, SourceError::Other(_)));
assert!(recorded(&requests).is_empty(), "invalid request must never reach the source");
}
#[tokio::test]
async fn list_objects_v2_rejects_truncated_page_without_token() {
let (client, _) = scripted_client(&spec(None), vec![ok(Vec::new(), LIST_TRUNCATED_WITHOUT_TOKEN)]).await;
@@ -1357,11 +1492,14 @@ mod tests {
fn prefix_client(prefix: Option<String>) -> SourceClient {
SourceClient {
client: S3Client::from_conf(
aws_sdk_s3::Config::builder()
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
.build(),
),
backend: Box::new(S3SourceBackend {
client: S3Client::from_conf(
aws_sdk_s3::Config::builder()
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
.build(),
),
bucket: "bucket".to_string(),
}),
endpoint: "https://source.example.com".to_string(),
bucket: "bucket".to_string(),
source_prefix: prefix.filter(|prefix| !prefix.is_empty()),
crates/ecstore/src/bucket/sealed_credentials.rs
+351
View File
@@ -0,0 +1,351 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Sealed remote credentials shared by the three stores that persist one
//! (rustfs/backlog#2168): replication targets (`bucket-targets.json`), remote
//! tiers (`tier-config.bin`) and on-demand migration sources
//! (`on-demand-migration.json`).
//!
//! The design record is `docs/architecture/remote-credential-sealing-adr.md`.
//! What this module owns: the versioned envelope, the encryption context that
//! binds a ciphertext to the record owning it, the sealer registration point,
//! and the fail-closed error type. What it deliberately does not own: any KMS
//! call (ECStore does not depend on `rustfs-kms`; the binary installs a
//! sealer, exactly like `ON_DEMAND_MIGRATION_CONFIG_HOOK` and the event
//! dispatch hook in `crates/ecstore/src/services/event_notification.rs`), and
//! any decision about which stored field a consumer writes.
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt;
use std::sync::{Arc, OnceLock};
/// Envelope format this build writes. A reader accepts only versions it
/// knows; an unknown version is a typed error, never a fallback.
pub const SEALED_CREDENTIAL_VERSION: u8 = 1;
/// Which store a sealed value belongs to. Part of the encryption context, so
/// a ciphertext cannot be replayed into a different store.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SealedCredentialStore {
/// `bucket-targets.json` (replication and other bucket targets).
BucketTargets,
/// `tier-config.bin` (remote tiers).
TierConfig,
/// `on-demand-migration.json` (migration sources).
OnDemandMigration,
}
impl SealedCredentialStore {
pub fn as_str(self) -> &'static str {
match self {
SealedCredentialStore::BucketTargets => "bucket-targets",
SealedCredentialStore::TierConfig => "tier-config",
SealedCredentialStore::OnDemandMigration => "on-demand-migration",
}
}
}
/// Identity of the record a secret belongs to: the store, its owner (bucket
/// name, tier name, or target ARN) and the field name. Rendered into the KMS
/// encryption context so a ciphertext moved between buckets, tiers or fields
/// fails to decrypt instead of silently authorizing a different remote.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SealScope {
pub store: SealedCredentialStore,
pub owner: String,
pub field: &'static str,
}
impl SealScope {
pub fn new(store: SealedCredentialStore, owner: impl Into<String>, field: &'static str) -> Self {
Self {
store,
owner: owner.into(),
field,
}
}
/// The encryption context handed to the sealer. Keys are stable: they are
/// part of the on-disk contract, because a ciphertext only decrypts under
/// the same context.
pub fn encryption_context(&self) -> HashMap<String, String> {
HashMap::from([
("rustfs:store".to_string(), self.store.as_str().to_string()),
("rustfs:owner".to_string(), self.owner.clone()),
("rustfs:field".to_string(), self.field.to_string()),
])
}
}
/// A sealed secret as persisted. `Debug` prints no ciphertext: a sealed value
/// is not a secret, but it is noise in a log line and an operator reading one
/// should see the key it is wrapped under, not the bytes.
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct SealedCredential {
/// Envelope version; see [`SEALED_CREDENTIAL_VERSION`].
pub v: u8,
/// KMS master key id the data key is wrapped under.
pub key_id: String,
/// Master key version, when the backend reports one. Carried so the KMS
/// re-wrap job (`docs/architecture/kms-bulk-rekey-contract.md`) can tell
/// stale envelopes apart; nothing here rotates on its own.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub key_version: Option<String>,
/// Algorithm label reported by the sealer, for forensics and migration.
pub alg: String,
/// Ciphertext blob as produced by the sealer, base64 (standard, padded)
/// in the JSON stores and raw inside the tier msgpack payload.
pub ct: String,
}
impl fmt::Debug for SealedCredential {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SealedCredential")
.field("v", &self.v)
.field("key_id", &self.key_id)
.field("key_version", &self.key_version)
.field("alg", &self.alg)
.field("ct", &format_args!("<{} bytes sealed>", self.ct.len()))
.finish()
}
}
impl SealedCredential {
/// Rejects an envelope this build cannot read. Called before every
/// unseal so an unknown version fails here rather than inside a backend.
pub fn check_version(&self) -> Result<(), SealedCredentialError> {
if self.v == SEALED_CREDENTIAL_VERSION {
Ok(())
} else {
Err(SealedCredentialError::UnsupportedVersion(self.v))
}
}
}
/// Why a seal or unseal did not produce a usable value. Every variant is
/// terminal for the record that carried it: a caller reports the remote as
/// unusable, and never substitutes a default or empty credential.
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum SealedCredentialError {
/// No sealer is installed: KMS is not configured, or the process has not
/// finished startup. Reading a sealed record is impossible here.
#[error("no credential sealer is installed")]
NoSealer,
/// The stored envelope is from a newer (or otherwise unknown) format.
#[error("unsupported sealed credential envelope version {0}")]
UnsupportedVersion(u8),
/// The stored bytes are not a well-formed envelope.
#[error("malformed sealed credential: {0}")]
Malformed(String),
/// The sealer refused: wrong encryption context, missing key, revoked
/// access, or a failed authentication tag.
#[error("sealed credential could not be unsealed: {0}")]
Kms(String),
}
/// The KMS-backed half, installed by the binary at startup.
#[async_trait]
pub trait CredentialSealer: Send + Sync + 'static {
/// Wraps `plaintext` under the scope's encryption context.
async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError>;
/// Unwraps a stored envelope. Must fail when the envelope was sealed
/// under a different scope.
async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError>;
}
static CREDENTIAL_SEALER: OnceLock<Arc<dyn CredentialSealer>> = OnceLock::new();
/// Installs the process-wide sealer. Returns `false` when one is already
/// installed, matching the other ECStore hooks.
pub fn install_credential_sealer(sealer: Arc<dyn CredentialSealer>) -> bool {
CREDENTIAL_SEALER.set(sealer).is_ok()
}
/// The installed sealer, or `None` when KMS is not wired. Callers that only
/// need to know whether sealing is possible use this; callers that must have
/// it use [`seal_secret`] / [`unseal_secret`] and get the typed error.
pub fn credential_sealer() -> Option<Arc<dyn CredentialSealer>> {
CREDENTIAL_SEALER.get().cloned()
}
/// Seals one secret field. Fails closed: without a sealer the caller must
/// reject the write rather than persist the secret in clear text after the
/// operator asked for sealing.
pub async fn seal_secret(plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError> {
let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?;
sealer.seal(plaintext, scope).await
}
/// Unseals one secret field, rejecting an unknown envelope version first.
pub async fn unseal_secret(sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError> {
sealed.check_version()?;
let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?;
sealer.unseal(sealed, scope).await
}
#[cfg(test)]
mod tests {
use super::*;
use parking_lot::Mutex;
/// Stands in for the KMS-backed sealer: records the context it was called
/// with, and refuses a ciphertext presented under a different one.
#[derive(Default)]
struct FakeSealer {
sealed_contexts: Mutex<Vec<HashMap<String, String>>>,
}
#[async_trait]
impl CredentialSealer for FakeSealer {
async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError> {
let context = scope.encryption_context();
self.sealed_contexts.lock().push(context.clone());
let mut bound = serde_json::to_string(&context).expect("context serializes");
bound.push('|');
bound.push_str(plaintext);
Ok(SealedCredential {
v: SEALED_CREDENTIAL_VERSION,
key_id: "key-1".to_string(),
key_version: Some("3".to_string()),
alg: "AES-256-GCM".to_string(),
ct: base64_simd::STANDARD.encode_to_string(bound.as_bytes()),
})
}
async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError> {
let raw = base64_simd::STANDARD
.decode_to_vec(sealed.ct.as_bytes())
.map_err(|err| SealedCredentialError::Malformed(err.to_string()))?;
let bound = String::from_utf8(raw).map_err(|err| SealedCredentialError::Malformed(err.to_string()))?;
let expected = serde_json::to_string(&scope.encryption_context()).expect("context serializes");
bound
.strip_prefix(&expected)
.and_then(|rest| rest.strip_prefix('|'))
.map(str::to_string)
.ok_or_else(|| SealedCredentialError::Kms("encryption context mismatch".to_string()))
}
}
fn scope(owner: &str) -> SealScope {
SealScope::new(SealedCredentialStore::OnDemandMigration, owner, "secret_key")
}
#[tokio::test]
async fn seal_round_trips_and_binds_the_scope() {
let sealer = Arc::new(FakeSealer::default());
let sealed = sealer.seal("super-secret", &scope("photos")).await.expect("seal");
assert_eq!(sealed.v, SEALED_CREDENTIAL_VERSION);
assert_eq!(sealed.key_version.as_deref(), Some("3"));
assert_eq!(sealer.unseal(&sealed, &scope("photos")).await.expect("unseal"), "super-secret");
// The same ciphertext under another bucket must not unseal.
let err = sealer
.unseal(&sealed, &scope("other-bucket"))
.await
.expect_err("a ciphertext must not move between owners");
assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}");
// Nor under another field of the same record.
let other_field = SealScope::new(SealedCredentialStore::OnDemandMigration, "photos", "session_token");
let err = sealer
.unseal(&sealed, &other_field)
.await
.expect_err("a ciphertext must not move between fields");
assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}");
let contexts = sealer.sealed_contexts.lock();
assert_eq!(contexts.len(), 1);
assert_eq!(contexts[0]["rustfs:store"], "on-demand-migration");
assert_eq!(contexts[0]["rustfs:owner"], "photos");
assert_eq!(contexts[0]["rustfs:field"], "secret_key");
}
#[tokio::test]
async fn an_unknown_envelope_version_is_rejected_before_the_sealer_is_asked() {
let sealed = SealedCredential {
v: SEALED_CREDENTIAL_VERSION + 1,
key_id: "key-1".to_string(),
key_version: None,
alg: "AES-256-GCM".to_string(),
ct: "Zm9v".to_string(),
};
assert_eq!(
sealed.check_version().expect_err("a newer envelope must not be read"),
SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1)
);
// The global helper reports the version, not "no sealer", even in a
// process where none is installed.
assert_eq!(
unseal_secret(&sealed, &scope("photos")).await.expect_err("version first"),
SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1)
);
}
#[tokio::test]
async fn without_a_sealer_both_directions_fail_closed() {
// This test binary installs no sealer, so the global helpers must
// report NoSealer rather than fall back to clear text.
assert!(credential_sealer().is_none(), "no sealer is installed in unit tests");
assert_eq!(
seal_secret("super-secret", &scope("photos")).await.expect_err("seal"),
SealedCredentialError::NoSealer
);
let sealed = SealedCredential {
v: SEALED_CREDENTIAL_VERSION,
key_id: "key-1".to_string(),
key_version: None,
alg: "AES-256-GCM".to_string(),
ct: "Zm9v".to_string(),
};
assert_eq!(
unseal_secret(&sealed, &scope("photos")).await.expect_err("unseal"),
SealedCredentialError::NoSealer
);
}
#[test]
fn debug_and_serde_keep_the_on_disk_shape_stable() {
let sealed = SealedCredential {
v: 1,
key_id: "key-1".to_string(),
key_version: None,
alg: "AES-256-GCM".to_string(),
ct: "Zm9v".to_string(),
};
// key_version is omitted when absent, so an envelope from a backend
// without version history stays compact.
assert_eq!(
serde_json::to_string(&sealed).expect("serialize"),
r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"#
);
let parsed: SealedCredential = serde_json::from_str(r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"#)
.expect("an envelope without key_version parses");
assert_eq!(parsed, sealed);
let rendered = format!("{sealed:?}");
assert!(rendered.contains("key-1"), "{rendered}");
assert!(!rendered.contains("Zm9v"), "Debug must not print the ciphertext: {rendered}");
}
#[test]
fn a_malformed_envelope_is_a_typed_error() {
let err = serde_json::from_str::<SealedCredential>(r#"{"v":1,"key_id":"key-1"}"#)
.map_err(|err| SealedCredentialError::Malformed(err.to_string()))
.expect_err("a truncated envelope must not parse");
assert!(matches!(err, SealedCredentialError::Malformed(_)), "{err}");
}
}
crates/ecstore/src/cluster/rpc/mod.rs
+1 -1
View File
@@ -48,7 +48,7 @@ pub use internode_data_transport::build_internode_data_transport_from_env;
pub(crate) use peer_rest_client::TierConfigReloadOutcome;
pub use peer_rest_client::{
KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPublicationLease,
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease,
};
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
pub use peer_s3_client::{
crates/ecstore/src/cluster/rpc/peer_rest_client.rs
+263 -13
View File
@@ -20,7 +20,8 @@ use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_bo
use crate::error::{Error, Result};
use crate::storage_api_contracts::internode::{
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION,
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
};
use crate::{
bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats, TierDailyStatsWire},
@@ -47,18 +48,19 @@ use rustfs_protos::proto_gen::node_service::{
HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
StartProfilingRequest, StopRebalanceRequest, TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest,
TierMutationControlResponse, TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest,
node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient,
ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageSnapshotRequest, ScannerDirtyUsageSnapshotResponse,
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, ServerInfoRequest,
SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient,
};
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
use rustfs_utils::XHost;
use serde::{Deserialize, Serialize as _};
use std::{
collections::HashMap,
collections::{BTreeMap, HashMap},
io::Cursor,
sync::{
Arc, Weak,
@@ -185,18 +187,31 @@ pub struct ScannerPeerActivity {
pub publication_blocked: Option<bool>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ScannerPeerDirtyUsageSnapshot {
pub instance_id: String,
pub generation: u64,
pub pending_bucket_count: u64,
pub protocol_version: u32,
pub complete: bool,
pub buckets: BTreeMap<String, u64>,
}
fn scanner_instance_id_is_valid(instance_id: &str) -> bool {
instance_id.len() == 32
&& instance_id
.as_bytes()
.iter()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte))
}
fn decode_scanner_activity_with_verifier(
response: ScannerActivityResponse,
challenge: &[u8; 16],
verify_proof: impl FnOnce(&[u8], &[u8]) -> Result<()>,
) -> Result<ScannerPeerActivity> {
let instance_id = &response.instance_id;
if instance_id.len() != 32
|| !instance_id
.as_bytes()
.iter()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte))
{
if !scanner_instance_id_is_valid(instance_id) {
return Err(Error::other("peer returned an invalid scanner activity instance ID"));
}
let (
@@ -318,6 +333,82 @@ fn decode_scanner_activity(response: ScannerActivityResponse, challenge: &[u8; 1
})
}
fn decode_scanner_dirty_usage_snapshot_with_verifier(
response: ScannerDirtyUsageSnapshotResponse,
challenge: &[u8; 16],
verify_proof: impl FnOnce(&[u8], &[u8]) -> Result<()>,
) -> Result<ScannerPeerDirtyUsageSnapshot> {
let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_response_body(challenge, &response)
.map_err(|_| Error::other("peer scanner dirty usage snapshot is too large to authenticate"))?;
verify_proof(&canonical, &response.response_proof)?;
if response.protocol_version != SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION {
return Err(Error::other("peer returned unsupported scanner dirty usage snapshot protocol"));
}
if !scanner_instance_id_is_valid(&response.instance_id) {
return Err(Error::other("peer returned an invalid scanner dirty usage snapshot instance ID"));
}
if response.generation == u64::MAX {
return Err(Error::other("peer scanner dirty usage snapshot exhausted its generation"));
}
if response.pending_bucket_count > 0 && response.generation == 0 {
return Err(Error::other("peer scanner dirty usage snapshot has pending buckets without a generation"));
}
if response.buckets.len() > SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES {
return Err(Error::other("peer scanner dirty usage snapshot exceeds the entry limit"));
}
let bucket_count = u64::try_from(response.buckets.len())
.map_err(|_| Error::other("peer scanner dirty usage snapshot entry count cannot be represented"))?;
let max_entries = u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES).unwrap_or(u64::MAX);
if response.complete {
if response.pending_bucket_count != bucket_count {
return Err(Error::other(
"complete peer scanner dirty usage snapshot has an inconsistent bucket count",
));
}
} else if !response.buckets.is_empty() || response.pending_bucket_count <= max_entries {
return Err(Error::other(
"incomplete peer scanner dirty usage snapshot must represent an entry-limit overflow",
));
}
for pair in response.buckets.windows(2) {
if pair[0].bucket >= pair[1].bucket {
return Err(Error::other("peer scanner dirty usage snapshot buckets are not strictly ordered"));
}
}
for bucket in &response.buckets {
if bucket.bucket.is_empty() {
return Err(Error::other("peer scanner dirty usage snapshot contains an empty bucket name"));
}
if bucket.generation == 0 || bucket.generation > response.generation {
return Err(Error::other("peer scanner dirty usage snapshot contains an invalid bucket generation"));
}
}
Ok(ScannerPeerDirtyUsageSnapshot {
instance_id: response.instance_id,
generation: response.generation,
pending_bucket_count: response.pending_bucket_count,
protocol_version: response.protocol_version,
complete: response.complete,
buckets: response
.buckets
.into_iter()
.map(|bucket| (bucket.bucket, bucket.generation))
.collect(),
})
}
fn decode_scanner_dirty_usage_snapshot(
response: ScannerDirtyUsageSnapshotResponse,
challenge: &[u8; 16],
) -> Result<ScannerPeerDirtyUsageSnapshot> {
decode_scanner_dirty_usage_snapshot_with_verifier(response, challenge, |canonical, proof| {
verify_tonic_rpc_response_proof(canonical, proof)
.map_err(|_| Error::other("peer returned an invalid scanner dirty usage snapshot response proof"))
})
}
fn scanner_activity_protocol_unsupported(err: &Error) -> bool {
matches!(
err,
@@ -1935,6 +2026,30 @@ impl PeerRestClient {
}
}
pub async fn scanner_dirty_usage_snapshot(&self) -> Result<ScannerPeerDirtyUsageSnapshot> {
self.finalize_result(
async {
let challenge = Uuid::new_v4();
let mut client = self
.get_client()
.await?
.max_decoding_message_size(SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE)
.max_encoding_message_size(SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE);
let mut request = Request::new(ScannerDirtyUsageSnapshotRequest {
challenge: challenge.as_bytes().to_vec().into(),
protocol_version: SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
});
let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_request_body(request.get_ref())
.map_err(|_| Error::other("scanner dirty usage snapshot request is too large to authenticate"))?;
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.scanner_dirty_usage_snapshot(request).await?.into_inner();
decode_scanner_dirty_usage_snapshot(response, challenge.as_bytes())
}
.await,
)
.await
}
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
let result = self
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
@@ -2640,6 +2755,141 @@ mod tests {
})
}
fn decode_test_scanner_dirty_usage_snapshot(
response: ScannerDirtyUsageSnapshotResponse,
) -> Result<ScannerPeerDirtyUsageSnapshot> {
decode_scanner_dirty_usage_snapshot_with_verifier(response, &[9; 16], |_canonical, proof| {
(proof == b"proof")
.then_some(())
.ok_or_else(|| Error::other("peer returned an invalid scanner dirty usage snapshot response proof"))
})
}
fn test_scanner_dirty_usage_snapshot_response() -> ScannerDirtyUsageSnapshotResponse {
ScannerDirtyUsageSnapshotResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
generation: 7,
pending_bucket_count: 2,
protocol_version: SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
complete: true,
buckets: vec![
rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
bucket: "archive".to_string(),
generation: 3,
},
rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
bucket: "photos".to_string(),
generation: 7,
},
],
response_proof: b"proof".to_vec().into(),
}
}
#[test]
fn scanner_dirty_usage_snapshot_requires_a_complete_authenticated_ordered_view() {
let decoded = decode_test_scanner_dirty_usage_snapshot(test_scanner_dirty_usage_snapshot_response())
.expect("a complete authenticated dirty usage snapshot should decode");
assert_eq!(decoded.instance_id, "0123456789abcdef0123456789abcdef");
assert_eq!(decoded.generation, 7);
assert_eq!(decoded.pending_bucket_count, 2);
assert_eq!(decoded.protocol_version, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION);
assert!(decoded.complete);
assert_eq!(decoded.buckets.get("archive"), Some(&3));
assert_eq!(decoded.buckets.get("photos"), Some(&7));
let overflow_count =
u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1).expect("the test snapshot entry limit should fit in u64");
let overflow = decode_test_scanner_dirty_usage_snapshot(ScannerDirtyUsageSnapshotResponse {
pending_bucket_count: overflow_count,
complete: false,
buckets: Vec::new(),
..test_scanner_dirty_usage_snapshot_response()
})
.expect("an explicit all-or-nothing overflow snapshot should decode");
assert!(!overflow.complete);
assert!(overflow.buckets.is_empty());
}
#[test]
fn scanner_dirty_usage_snapshot_rejects_inconsistent_or_partial_peer_data() {
let mut cases = Vec::new();
let mut invalid_instance = test_scanner_dirty_usage_snapshot_response();
invalid_instance.instance_id = "ABCDEF0123456789ABCDEF0123456789".to_string();
cases.push((invalid_instance, "instance ID"));
let mut unsupported = test_scanner_dirty_usage_snapshot_response();
unsupported.protocol_version = SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION + 1;
cases.push((unsupported, "unsupported"));
let mut exhausted = test_scanner_dirty_usage_snapshot_response();
exhausted.generation = u64::MAX;
cases.push((exhausted, "exhausted"));
let mut inconsistent_count = test_scanner_dirty_usage_snapshot_response();
inconsistent_count.pending_bucket_count = 3;
cases.push((inconsistent_count, "bucket count"));
let mut unordered = test_scanner_dirty_usage_snapshot_response();
unordered.buckets.reverse();
cases.push((unordered, "strictly ordered"));
let mut future_bucket = test_scanner_dirty_usage_snapshot_response();
future_bucket.buckets[0].generation = 8;
cases.push((future_bucket, "bucket generation"));
let mut zero_generation = test_scanner_dirty_usage_snapshot_response();
zero_generation.buckets[0].generation = 0;
cases.push((zero_generation, "bucket generation"));
let mut empty_bucket = test_scanner_dirty_usage_snapshot_response();
empty_bucket.buckets[0].bucket.clear();
cases.push((empty_bucket, "empty bucket name"));
let mut partial = test_scanner_dirty_usage_snapshot_response();
partial.complete = false;
cases.push((partial, "entry-limit overflow"));
let too_many_buckets = ScannerDirtyUsageSnapshotResponse {
generation: 1,
pending_bucket_count: u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1)
.expect("the test snapshot entry limit should fit in u64"),
buckets: (0..=SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES)
.map(|index| rustfs_protos::proto_gen::node_service::ScannerDirtyUsageBucket {
bucket: format!("bucket-{index:04}"),
generation: 1,
})
.collect(),
..test_scanner_dirty_usage_snapshot_response()
};
cases.push((too_many_buckets, "exceeds the entry limit"));
let overflow_count =
u64::try_from(SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES + 1).expect("the test snapshot entry limit should fit in u64");
let invalid_overflow = ScannerDirtyUsageSnapshotResponse {
generation: 0,
pending_bucket_count: overflow_count,
complete: false,
buckets: Vec::new(),
..test_scanner_dirty_usage_snapshot_response()
};
cases.push((invalid_overflow, "without a generation"));
for (response, expected) in cases {
let err =
decode_test_scanner_dirty_usage_snapshot(response).expect_err("malformed dirty usage snapshots must fail closed");
assert!(err.to_string().contains(expected), "expected {expected:?} in {err}");
}
let mut invalid_proof = test_scanner_dirty_usage_snapshot_response();
invalid_proof.protocol_version = SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION + 1;
invalid_proof.response_proof = b"invalid".to_vec().into();
let err = decode_test_scanner_dirty_usage_snapshot(invalid_proof)
.expect_err("an invalid response proof must fail before peer fields are trusted");
assert!(err.to_string().contains("response proof"));
}
#[test]
fn build_clients_from_slots_preserves_missing_remote_topology_slots() {
let slots = vec![
crates/ecstore/src/runtime/instance.rs
-8
View File
@@ -213,7 +213,6 @@ pub struct InstanceContext {
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
#[cfg(test)]
tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
}
@@ -260,7 +259,6 @@ impl InstanceContext {
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
#[cfg(test)]
tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
}
}
@@ -655,16 +653,10 @@ impl InstanceContext {
.insert(store_id)
}
#[cfg(test)]
#[allow(
dead_code,
reason = "driven by the tier-delete-journal recovery test behind `--features test-util` (backlog#1823)"
)]
pub(crate) fn wake_tier_delete_journal_recovery(&self) {
self.tier_delete_journal_recovery_wakeup.notify_one();
}
#[cfg(test)]
pub(crate) async fn wait_for_tier_delete_journal_recovery(&self) {
self.tier_delete_journal_recovery_wakeup.notified().await;
}
crates/ecstore/src/services/event_notification.rs
+68 -9
View File
@@ -117,26 +117,85 @@ pub fn send_event(args: EventArgs) {
);
}
/// Shared event recorder for this crate's tests.
///
/// [`register_event_dispatch_hook`] backs a `OnceLock`, so only the first
/// caller in a test binary can install a hook. Every test that needs to
/// observe dispatched events must therefore go through this single recorder
/// instead of registering its own.
#[cfg(test)]
pub(crate) mod test_recorder {
use super::register_event_dispatch_hook;
use std::sync::{Mutex, OnceLock};
use uuid::Uuid;
/// The fields a test needs from a dispatched event. `EventArgs` itself is
/// not `Clone`, and recording a reduced shape keeps this test seam from
/// constraining the production type.
#[derive(Clone, Debug)]
pub(crate) struct RecordedEvent {
pub(crate) event_name: String,
pub(crate) bucket: String,
pub(crate) object: String,
pub(crate) version_id: Option<Uuid>,
pub(crate) delete_marker: bool,
}
static RECORDED: OnceLock<Mutex<Vec<RecordedEvent>>> = OnceLock::new();
fn recorded() -> &'static Mutex<Vec<RecordedEvent>> {
RECORDED.get_or_init(|| Mutex::new(Vec::new()))
}
pub(crate) fn install() {
static INSTALLED: OnceLock<()> = OnceLock::new();
INSTALLED.get_or_init(|| {
assert!(
register_event_dispatch_hook(|args| {
recorded().lock().unwrap_or_else(|err| err.into_inner()).push(RecordedEvent {
event_name: args.event_name,
bucket: args.bucket_name,
object: args.object.name,
version_id: args.object.version_id,
delete_marker: args.object.delete_marker,
});
}),
"the test event recorder must own this binary's dispatch hook"
);
});
}
/// Everything recorded for one bucket. Tests select by their own unique
/// bucket name rather than draining, because tests that are not
/// `#[serial]` may dispatch events concurrently.
pub(crate) fn recorded_for_bucket(bucket: &str) -> Vec<RecordedEvent> {
recorded()
.lock()
.unwrap_or_else(|err| err.into_inner())
.iter()
.filter(|event| event.bucket == bucket)
.cloned()
.collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
static DISPATCH_COUNT: AtomicUsize = AtomicUsize::new(0);
#[test]
fn send_event_dispatches_to_registered_hook() {
let _ = register_event_dispatch_hook(|_args| {
DISPATCH_COUNT.fetch_add(1, Ordering::Relaxed);
});
let before = DISPATCH_COUNT.load(Ordering::Relaxed);
test_recorder::install();
let bucket = format!("event-dispatch-{}", uuid::Uuid::new_v4().simple());
send_event(EventArgs {
event_name: "s3:ObjectCreated:Put".to_string(),
bucket_name: "demo".to_string(),
bucket_name: bucket.clone(),
..Default::default()
});
assert_eq!(DISPATCH_COUNT.load(Ordering::Relaxed), before + 1);
let dispatched = test_recorder::recorded_for_bucket(&bucket);
assert_eq!(dispatched.len(), 1);
assert_eq!(dispatched[0].event_name, "s3:ObjectCreated:Put");
}
}
crates/ecstore/src/services/notification_sys.rs
+123 -1
View File
@@ -13,7 +13,9 @@
// limitations under the License.
use crate::bucket::lifecycle::tier_last_day_stats::DailyAllTierStats;
use crate::cluster::rpc::{PeerRestClient, ScannerPeerActivity, ScannerPublicationLease, TierConfigReloadOutcome};
use crate::cluster::rpc::{
PeerRestClient, ScannerPeerActivity, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, TierConfigReloadOutcome,
};
use crate::diagnostics::admin_server_info::get_commit_id;
use crate::disk::DiskAPI;
use crate::error::{Error, Result};
@@ -2080,6 +2082,32 @@ impl NotificationSys {
Ok(generations)
}
pub async fn scanner_dirty_usage_snapshots(&self) -> Result<Vec<(String, ScannerPeerDirtyUsageSnapshot)>> {
if self.peer_clients.is_empty() {
return Err(Error::other("scanner dirty usage snapshot probe has no remote peers"));
}
if self.all_peer_clients.len() != self.peer_clients.len() + 1 {
return Err(Error::other("scanner dirty usage snapshot peer topology is incomplete"));
}
let mut futures = Vec::with_capacity(self.peer_clients.len());
for client in self.peer_clients.iter().cloned() {
futures.push(async move {
let client = client.ok_or_else(|| Error::other("scanner dirty usage snapshot peer is unreachable"))?;
let host = client.grid_host.clone();
scanner_dirty_usage_snapshot_with_retry(&client, &host)
.await
.map(|snapshot| (host, snapshot))
});
}
let mut snapshots = Vec::with_capacity(futures.len());
for result in join_all(futures).await {
snapshots.push(result?);
}
Ok(snapshots)
}
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<(String, String, u64)>) -> Result<bool> {
let mut by_host = HashMap::with_capacity(acknowledgements.len());
for (host, instance_id, generation) in acknowledgements {
@@ -2591,6 +2619,54 @@ async fn scanner_activity_with_retry(client: &PeerRestClient, host: &str) -> Res
}
}
async fn scanner_dirty_usage_snapshot_with_retry(client: &PeerRestClient, host: &str) -> Result<ScannerPeerDirtyUsageSnapshot> {
let first = timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, client.scanner_dirty_usage_snapshot()).await;
let should_retry = match &first {
Ok(Ok(_)) => false,
Ok(Err(err)) => scanner_activity_should_retry(Some(err), false),
Err(_) => scanner_activity_should_retry(None, true),
};
match first {
Ok(Ok(snapshot)) => return Ok(snapshot),
Ok(Err(err)) if !should_retry => return Err(err),
Ok(Err(err)) => {
debug!(
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
result = "retrying",
capability = "scanner_dirty_usage_snapshot",
peer = host,
error = %err,
"notification capability probe retrying"
);
client.prepare_retry().await;
}
Err(_) => {
debug!(
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
result = "retrying",
capability = "scanner_dirty_usage_snapshot",
peer = host,
timeout = ?SCANNER_ACTIVITY_PROBE_TIMEOUT,
"notification capability probe retrying"
);
client.prepare_retry().await;
}
}
match timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, client.scanner_dirty_usage_snapshot()).await {
Ok(result) => result,
Err(_) => {
client.evict_connection().await;
Err(Error::Timeout)
}
}
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
async fn call_peer_with_timeout<F, Fut>(
timeout_dur: Duration,
@@ -3688,6 +3764,52 @@ mod tests {
assert!(err.to_string().contains("peer topology is incomplete"));
}
#[tokio::test]
async fn scanner_dirty_usage_snapshot_probe_rejects_unusable_peer_topologies() {
let unreachable = NotificationSys {
peer_clients: vec![None],
all_peer_clients: vec![None, None],
peer_topology_hosts: vec!["node-a:9000".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
tier_config_reload_workers: Default::default(),
};
let err = unreachable
.scanner_dirty_usage_snapshots()
.await
.expect_err("an unreachable peer must invalidate the distributed dirty usage snapshot");
assert!(err.to_string().contains("peer is unreachable"));
let empty = NotificationSys {
peer_clients: Vec::new(),
all_peer_clients: Vec::new(),
peer_topology_hosts: Vec::new(),
peer_admin_caches: Vec::new(),
tier_config_reload_workers: Default::default(),
};
let err = empty
.scanner_dirty_usage_snapshots()
.await
.expect_err("an empty peer set must not produce a distributed dirty usage snapshot");
assert!(err.to_string().contains("no remote peers"));
let client = PeerRestClient::new(
"127.0.0.1:9000".to_string().try_into().expect("peer host should parse"),
"http://127.0.0.1:9000".to_string(),
);
let incomplete = NotificationSys {
peer_clients: vec![Some(client)],
all_peer_clients: vec![None],
peer_topology_hosts: vec!["127.0.0.1:9000".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
tier_config_reload_workers: Default::default(),
};
let err = incomplete
.scanner_dirty_usage_snapshots()
.await
.expect_err("an incomplete topology must not produce a distributed dirty usage snapshot");
assert!(err.to_string().contains("peer topology is incomplete"));
}
#[tokio::test]
async fn server_info_no_client_slot_uses_topology_host_without_counting_rpc_failure() {
let sys = NotificationSys {
crates/ecstore/src/services/tier/tier.rs
+1976 -170
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/services/tier/tier_admin.rs
+187 -9
View File
@@ -18,25 +18,203 @@
#![allow(unused_must_use)]
#![allow(clippy::all)]
use serde::{Deserialize, Serialize};
use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[derive(Serialize, Deserialize, Default, Clone)]
#[serde(default)]
pub struct TierServicePrincipalAuth {
#[serde(rename = "TenantID", alias = "tenantID", alias = "tenant_id")]
pub tenant_id: String,
#[serde(rename = "ClientID", alias = "clientID", alias = "client_id")]
pub client_id: String,
#[serde(rename = "ClientSecret", alias = "clientSecret", alias = "client_secret")]
pub client_secret: String,
}
impl TierServicePrincipalAuth {
pub(crate) fn is_empty(&self) -> bool {
self.tenant_id.is_empty() && self.client_id.is_empty() && self.client_secret.is_empty()
}
}
impl std::fmt::Debug for TierServicePrincipalAuth {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TierServicePrincipalAuth")
.field("tenant_id", &self.tenant_id)
.field("client_id", &self.client_id)
.field("client_secret", &"REDACTED")
.finish()
}
}
#[derive(Serialize, Deserialize, Default, Clone)]
#[serde(default)]
pub struct TierCreds {
#[serde(rename = "accessKey")]
#[serde(rename = "access", alias = "accessKey")]
pub access_key: String,
#[serde(rename = "secretKey")]
#[serde(rename = "secret", alias = "secretKey")]
pub secret_key: String,
#[serde(rename = "awsRole")]
#[serde(rename = "awsrole", alias = "awsRole")]
pub aws_role: bool,
#[serde(rename = "awsRoleWebIdentityTokenFile")]
#[serde(rename = "awsroleWebIdentity", alias = "awsRoleWebIdentityTokenFile")]
pub aws_role_web_identity_token_file: String,
#[serde(rename = "awsRoleArn")]
#[serde(rename = "awsroleARN", alias = "awsRoleArn", alias = "awsRoleARN")]
pub aws_role_arn: String,
//azsp: ServicePrincipalAuth,
#[serde(rename = "azSP", alias = "azsp", skip_serializing_if = "TierServicePrincipalAuth::is_empty")]
pub azure_service_principal: TierServicePrincipalAuth,
//#[serde(rename = "credsJson")]
#[serde(
rename = "creds",
alias = "credsJson",
alias = "credsJSON",
alias = "creds_json",
default,
skip_serializing_if = "Vec::is_empty",
with = "base64_bytes"
)]
pub creds_json: Vec<u8>,
}
impl std::fmt::Debug for TierCreds {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TierCreds")
.field("access_key", &self.access_key)
.field("secret_key", &"REDACTED")
.field("aws_role", &self.aws_role)
.field(
"aws_role_web_identity_token_file",
&(!self.aws_role_web_identity_token_file.is_empty()).then_some("REDACTED"),
)
.field("aws_role_arn", &self.aws_role_arn)
.field("azure_service_principal", &self.azure_service_principal)
.field("creds_json", &(!self.creds_json.is_empty()).then_some("REDACTED"))
.finish()
}
}
mod base64_bytes {
use super::*;
pub(super) fn serialize<S>(value: &[u8], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&base64_simd::STANDARD.encode_to_string(value))
}
pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(untagged)]
enum EncodedBytes {
Base64(String),
Legacy(Vec<u8>),
}
match EncodedBytes::deserialize(deserializer)? {
EncodedBytes::Base64(value) => base64_simd::STANDARD
.decode_to_vec(value.as_bytes())
.or_else(|_| base64_simd::STANDARD_NO_PAD.decode_to_vec(value.as_bytes()))
.or_else(|_| base64_simd::URL_SAFE.decode_to_vec(value.as_bytes()))
.or_else(|_| base64_simd::URL_SAFE_NO_PAD.decode_to_vec(value.as_bytes()))
.map_err(de::Error::custom),
EncodedBytes::Legacy(value) => Ok(value),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tier_creds_accepts_madmin_wire_names_and_base64_gcs_json() {
let service_account = r#"{"type":"service_account","project_id":"tier-🚀x"}"#.as_bytes();
let encoded = "eyJ0eXBlIjoic2VydmljZV9hY2NvdW50IiwicHJvamVjdF9pZCI6InRpZXIt8J+agHgifQ==";
let creds: TierCreds = serde_json::from_value(serde_json::json!({
"access": "access",
"secret": "secret",
"awsrole": false,
"creds": encoded,
}))
.expect("madmin tier credentials should decode");
assert_eq!(creds.access_key, "access");
assert_eq!(creds.secret_key, "secret");
assert_eq!(creds.creds_json.as_slice(), &service_account[..]);
let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode");
assert_eq!(wire["access"], "access");
assert_eq!(wire["secret"], "secret");
assert_eq!(wire["creds"], encoded);
assert!(wire.get("accessKey").is_none());
assert!(wire.get("secretKey").is_none());
let legacy: TierCreds = serde_json::from_value(serde_json::json!({
"accessKey": "legacy-access",
"secretKey": "legacy-secret",
"credsJson": service_account,
}))
.expect("the former RustFS field names and byte-array encoding should remain readable");
assert_eq!(legacy.access_key, "legacy-access");
assert_eq!(legacy.secret_key, "legacy-secret");
assert_eq!(legacy.creds_json.as_slice(), &service_account[..]);
}
#[test]
fn tier_creds_accepts_all_supported_base64_alphabets_and_padding_modes() {
let service_account = r#"{"type":"service_account","project_id":"tier-🚀"}"#.as_bytes();
for encoder in [
base64_simd::STANDARD,
base64_simd::STANDARD_NO_PAD,
base64_simd::URL_SAFE,
base64_simd::URL_SAFE_NO_PAD,
] {
let encoded = encoder.encode_to_string(service_account);
let creds: TierCreds = serde_json::from_value(serde_json::json!({ "creds": encoded }))
.expect("all supported madmin base64 forms should decode");
assert_eq!(creds.creds_json, service_account);
}
}
#[test]
fn tier_creds_debug_redacts_secret_payloads() {
let creds = TierCreds {
access_key: "access".to_string(),
secret_key: "tier-secret-value".to_string(),
aws_role_web_identity_token_file: "/var/run/private-token".to_string(),
creds_json: br#"{"private_key":"gcs-private-key-value"}"#.to_vec(),
..Default::default()
};
let rendered = format!("{creds:?}");
assert!(!rendered.contains("tier-secret-value"));
assert!(!rendered.contains("/var/run/private-token"));
assert!(!rendered.contains("gcs-private-key-value"));
}
#[test]
fn tier_creds_accepts_canonical_madmin_azure_service_principal_wire_shape() {
let creds: TierCreds = serde_json::from_value(serde_json::json!({
"azSP": {
"TenantID": "tenant",
"ClientID": "client",
"ClientSecret": "service-principal-secret"
}
}))
.expect("canonical madmin azure service principal credentials should decode");
assert_eq!(creds.azure_service_principal.tenant_id, "tenant");
assert_eq!(creds.azure_service_principal.client_id, "client");
assert_eq!(creds.azure_service_principal.client_secret, "service-principal-secret");
let wire = serde_json::to_value(&creds).expect("canonical madmin credentials should encode");
assert_eq!(wire["azSP"]["TenantID"], "tenant");
assert_eq!(wire["azSP"]["ClientID"], "client");
assert_eq!(wire["azSP"]["ClientSecret"], "service-principal-secret");
assert!(!format!("{creds:?}").contains("service-principal-secret"));
}
}
crates/ecstore/src/services/tier/tier_config.rs
+295 -123
View File
@@ -42,7 +42,7 @@ const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
pub enum TierType {
#[default]
Unsupported,
#[serde(rename = "s3")]
#[serde(rename = "s3", alias = "S3")]
S3,
#[serde(rename = "wasabi")]
Wasabi,
@@ -58,7 +58,7 @@ pub enum TierType {
Huaweicloud,
#[serde(rename = "azure")]
Azure,
#[serde(rename = "gcs")]
#[serde(rename = "gcs", alias = "GCS")]
GCS,
#[serde(rename = "r2")]
R2,
@@ -138,16 +138,18 @@ impl TierType {
}
}
#[derive(Default, Debug, Serialize, Deserialize)]
pub(crate) const TIER_CREDENTIAL_REDACTED: &str = "REDACTED";
#[derive(Default, Serialize, Deserialize)]
#[serde(default)]
pub struct TierConfig {
#[serde(skip)]
pub version: String,
#[serde(rename = "type")]
#[serde(rename = "type", alias = "Type")]
pub tier_type: TierType,
#[serde(skip)]
#[serde(rename = "Name", alias = "name", skip_serializing)]
pub name: String,
#[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
#[serde(rename = "s3", alias = "S3", skip_serializing_if = "Option::is_none")]
pub s3: Option<TierS3>,
#[serde(rename = "wasabi", skip_serializing_if = "Option::is_none")]
pub wasabi: Option<TierWasabi>,
@@ -159,7 +161,7 @@ pub struct TierConfig {
pub huaweicloud: Option<TierHuaweicloud>,
#[serde(rename = "azure", skip_serializing_if = "Option::is_none")]
pub azure: Option<TierAzure>,
#[serde(rename = "gcs", skip_serializing_if = "Option::is_none")]
#[serde(rename = "gcs", alias = "GCS", skip_serializing_if = "Option::is_none")]
pub gcs: Option<TierGCS>,
#[serde(rename = "r2", skip_serializing_if = "Option::is_none")]
pub r2: Option<TierR2>,
@@ -170,109 +172,91 @@ pub struct TierConfig {
}
impl Clone for TierConfig {
fn clone(&self) -> TierConfig {
let mut s3 = None;
let mut wasabi = None;
let mut r = None;
let mut compatible_backend = None;
let mut aliyun = None;
let mut tencent = None;
let mut huaweicloud = None;
let mut azure = None;
let mut gcs = None;
let mut r2 = None;
match self.tier_type {
TierType::S3 => {
if let Some(s3_) = self.s3.as_ref() {
let mut s3_clone = s3_.clone();
s3_clone.secret_key = "REDACTED".to_string();
s3 = Some(s3_clone);
}
}
TierType::Wasabi => {
if let Some(wasabi_) = self.wasabi.as_ref() {
let mut wasabi_clone = wasabi_.clone();
wasabi_clone.secret_key = "REDACTED".to_string();
wasabi = Some(wasabi_clone);
}
}
TierType::RustFS => {
if let Some(r_) = self.rustfs.as_ref() {
let mut r_clone = r_.clone();
r_clone.secret_key = "REDACTED".to_string();
r = Some(r_clone);
}
}
TierType::MinIO => {
if let Some(compatible_backend_) = self.minio.as_ref() {
let mut compatible_backend_clone = compatible_backend_.clone();
compatible_backend_clone.secret_key = "REDACTED".to_string();
compatible_backend = Some(compatible_backend_clone);
}
}
TierType::Aliyun => {
if let Some(aliyun_) = self.aliyun.as_ref() {
let mut aliyun_clone = aliyun_.clone();
aliyun_clone.secret_key = "REDACTED".to_string();
aliyun = Some(aliyun_clone);
}
}
TierType::Tencent => {
if let Some(tencent_) = self.tencent.as_ref() {
let mut tencent_clone = tencent_.clone();
tencent_clone.secret_key = "REDACTED".to_string();
tencent = Some(tencent_clone);
}
}
TierType::Huaweicloud => {
if let Some(huaweicloud_) = self.huaweicloud.as_ref() {
let mut huaweicloud_clone = huaweicloud_.clone();
huaweicloud_clone.secret_key = "REDACTED".to_string();
huaweicloud = Some(huaweicloud_clone);
}
}
TierType::Azure => {
if let Some(azure_) = self.azure.as_ref() {
let mut azure_clone = azure_.clone();
azure_clone.secret_key = "REDACTED".to_string();
azure = Some(azure_clone);
}
}
TierType::GCS => {
if let Some(gcs_) = self.gcs.as_ref() {
let mut gcs_clone = gcs_.clone();
gcs_clone.creds = "REDACTED".to_string();
gcs = Some(gcs_clone);
}
}
TierType::R2 => {
if let Some(r2_) = self.r2.as_ref() {
let mut r2_clone = r2_.clone();
r2_clone.secret_key = "REDACTED".to_string();
r2 = Some(r2_clone);
}
}
_ => (),
}
TierConfig {
version: self.version.clone(),
tier_type: self.tier_type.clone(),
name: self.name.clone(),
s3,
wasabi,
rustfs: r,
minio: compatible_backend,
aliyun,
tencent,
huaweicloud,
azure,
gcs,
r2,
}
fn clone(&self) -> Self {
self.redacted()
}
}
impl TierConfig {
pub(crate) fn redacted(&self) -> Self {
let mut redacted = Self {
version: self.version.clone(),
tier_type: self.tier_type.clone(),
name: self.name.clone(),
..Default::default()
};
match self.tier_type {
TierType::S3 => {
redacted.s3 = self.s3.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
if !backend.aws_role_web_identity_token_file.is_empty() {
backend.aws_role_web_identity_token_file = TIER_CREDENTIAL_REDACTED.to_string();
}
backend
});
}
TierType::Wasabi => {
redacted.wasabi = self.wasabi.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::RustFS => {
redacted.rustfs = self.rustfs.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::MinIO => {
redacted.minio = self.minio.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::Aliyun => {
redacted.aliyun = self.aliyun.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::Tencent => {
redacted.tencent = self.tencent.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::Huaweicloud => {
redacted.huaweicloud = self.huaweicloud.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::Azure => {
redacted.azure = self.azure.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
if !backend.sp_auth.client_secret.is_empty() {
backend.sp_auth.client_secret = TIER_CREDENTIAL_REDACTED.to_string();
}
backend
});
}
TierType::GCS => {
redacted.gcs = self.gcs.clone().map(|mut backend| {
backend.creds = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::R2 => {
redacted.r2 = self.r2.clone().map(|mut backend| {
backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
backend
});
}
TierType::Unsupported => {}
}
redacted
}
pub(crate) fn clone_with_credentials(&self) -> Self {
Self {
version: self.version.clone(),
@@ -372,31 +356,61 @@ impl TierConfig {
}
}
impl std::fmt::Debug for TierConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let redacted = self.redacted();
f.debug_struct("TierConfig")
.field("version", &redacted.version)
.field("tier_type", &redacted.tier_type)
.field("name", &redacted.name)
.field("s3", &redacted.s3)
.field("wasabi", &redacted.wasabi)
.field("aliyun", &redacted.aliyun)
.field("tencent", &redacted.tencent)
.field("huaweicloud", &redacted.huaweicloud)
.field("azure", &redacted.azure)
.field("gcs", &redacted.gcs)
.field("r2", &redacted.r2)
.field("rustfs", &redacted.rustfs)
.field("minio", &redacted.minio)
.finish()
}
}
//type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierS3 {
#[serde(alias = "Name")]
pub name: String,
#[serde(alias = "Endpoint")]
pub endpoint: String,
#[serde(rename = "accessKey")]
#[serde(rename = "accessKey", alias = "AccessKey")]
pub access_key: String,
#[serde(rename = "secretKey")]
#[serde(rename = "secretKey", alias = "SecretKey")]
pub secret_key: String,
#[serde(alias = "Bucket")]
pub bucket: String,
#[serde(alias = "Prefix")]
pub prefix: String,
#[serde(alias = "Region")]
pub region: String,
#[serde(rename = "storageClass")]
#[serde(rename = "storageClass", alias = "StorageClass")]
pub storage_class: String,
#[serde(skip)]
#[serde(rename = "AWSRole", alias = "awsRole", skip_serializing)]
pub aws_role: bool,
#[serde(skip)]
#[serde(
rename = "AWSRoleWebIdentityTokenFile",
alias = "awsRoleWebIdentityTokenFile",
skip_serializing
)]
pub aws_role_web_identity_token_file: String,
#[serde(skip)]
#[serde(rename = "AWSRoleARN", alias = "awsRoleARN", alias = "awsRoleArn", skip_serializing)]
pub aws_role_arn: String,
#[serde(skip)]
#[serde(rename = "AWSRoleSessionName", alias = "awsRoleSessionName", skip_serializing)]
pub aws_role_session_name: String,
#[serde(skip)]
#[serde(rename = "AWSRoleDurationSeconds", alias = "awsRoleDurationSeconds", skip_serializing)]
pub aws_role_duration_seconds: i32,
}
@@ -623,8 +637,11 @@ pub struct TierHuaweicloud {
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct ServicePrincipalAuth {
#[serde(alias = "TenantID")]
pub tenant_id: String,
#[serde(alias = "ClientID")]
pub client_id: String,
#[serde(alias = "ClientSecret")]
pub client_secret: String,
}
@@ -640,9 +657,9 @@ pub struct TierAzure {
pub bucket: String,
pub prefix: String,
pub region: String,
#[serde(rename = "storageClass")]
#[serde(rename = "storageClass", alias = "StorageClass")]
pub storage_class: String,
#[serde(rename = "spAuth")]
#[serde(rename = "spAuth", alias = "SPAuth")]
pub sp_auth: ServicePrincipalAuth,
}
@@ -696,14 +713,19 @@ fn AzureStorageClass(sc string) func(az *TierAzure) error {
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
#[serde(default)]
pub struct TierGCS {
#[serde(alias = "Name")]
pub name: String,
#[serde(alias = "Endpoint")]
pub endpoint: String,
#[serde(rename = "creds")]
#[serde(rename = "creds", alias = "Creds")]
pub creds: String,
#[serde(alias = "Bucket")]
pub bucket: String,
#[serde(alias = "Prefix")]
pub prefix: String,
#[serde(alias = "Region")]
pub region: String,
#[serde(rename = "storageClass")]
#[serde(rename = "storageClass", alias = "StorageClass")]
pub storage_class: String,
}
@@ -725,6 +747,43 @@ pub struct TierR2 {
mod tests {
use super::*;
#[test]
fn s3_gcs_type_uppercase_aliases_preserve_lowercase_output() {
let s3: TierType = serde_json::from_str(r#""S3""#).expect("uppercase S3 wire value should decode");
let gcs: TierType = serde_json::from_str(r#""GCS""#).expect("uppercase GCS wire value should decode");
assert!(matches!(s3, TierType::S3));
assert!(matches!(gcs, TierType::GCS));
assert_eq!(serde_json::to_string(&s3).expect("S3 type should encode"), r#""s3""#);
assert_eq!(serde_json::to_string(&gcs).expect("GCS type should encode"), r#""gcs""#);
}
#[test]
fn azure_service_principal_accepts_canonical_madmin_field_names() {
for field in ["TenantID", "ClientID", "ClientSecret"] {
let mut sp_auth = serde_json::Map::new();
sp_auth.insert(field.to_string(), serde_json::Value::String("present".to_string()));
let config: TierConfig = serde_json::from_value(serde_json::json!({
"type": "azure",
"Name": "COLD-AZURE",
"azure": {
"name": "COLD-AZURE",
"endpoint": "https://azure.example.invalid",
"accessKey": "account",
"secretKey": "key",
"bucket": "archive",
"SPAuth": sp_auth
}
}))
.expect("mixed RustFS/madmin Azure payload should decode");
let sp_auth = &config.azure.expect("Azure payload should exist").sp_auth;
assert!(
!sp_auth.tenant_id.is_empty() || !sp_auth.client_id.is_empty() || !sp_auth.client_secret.is_empty(),
"canonical {field} must not be silently discarded"
);
}
}
fn wasabi_config() -> TierWasabi {
TierWasabi {
name: "COLD-WASABI".to_string(),
@@ -838,9 +897,14 @@ mod tests {
let config = TierConfig {
tier_type: TierType::Wasabi,
wasabi: Some(wasabi_config()),
rustfs: Some(TierRustFS {
access_key: "inactive-access".to_string(),
secret_key: "inactive-secret".to_string(),
..Default::default()
}),
..Default::default()
};
let redacted = config.clone();
let redacted = config.redacted();
assert_eq!(
redacted
.wasabi
@@ -849,21 +913,46 @@ mod tests {
.secret_key,
"REDACTED"
);
assert!(redacted.rustfs.is_none(), "the external view should retain only the active provider");
let cloned = config.clone();
assert_eq!(cloned.wasabi.expect("redacted Wasabi clone should remain").secret_key, "REDACTED");
assert!(cloned.rustfs.is_none(), "ordinary Clone must retain its redacted API semantics");
let preserved = config.clone_with_credentials();
assert_eq!(
config
.clone_with_credentials()
preserved
.wasabi
.as_ref()
.expect("credential-bearing Wasabi payload should remain")
.expect("credential-bearing Wasabi snapshot should remain")
.secret_key,
"secret"
);
assert_eq!(
preserved
.rustfs
.expect("credential-bearing snapshots should preserve inactive provider data")
.secret_key,
"inactive-secret"
);
let mut debug_config = wasabi_config();
debug_config.secret_key = "wasabi-debug-secret-value".to_string();
let debug = format!("{debug_config:?}");
assert!(debug.contains("REDACTED"));
assert!(!debug.contains("wasabi-debug-secret-value"));
let debug = format!(
"{:?}",
TierConfig {
tier_type: TierType::RustFS,
rustfs: Some(TierRustFS {
secret_key: "rustfs-debug-secret-value".to_string(),
..Default::default()
}),
..Default::default()
}
);
assert!(debug.contains("REDACTED"));
assert!(!debug.contains("rustfs-debug-secret-value"));
}
#[test]
@@ -894,7 +983,7 @@ mod tests {
assert_eq!(encoded, expected);
let decoded: TierConfig = serde_json::from_value(encoded).expect("Wasabi Admin JSON should decode");
assert!(matches!(decoded.tier_type, TierType::Wasabi));
let redacted = config.clone();
let redacted = config.redacted();
assert_eq!(
config
.wasabi
@@ -916,4 +1005,87 @@ mod tests {
"REDACTED"
);
}
#[test]
fn api_serialization_and_debug_redact_s3_gcs_and_azure_credentials() {
let cases = [
(
"s3",
TierConfig {
tier_type: TierType::S3,
s3: Some(TierS3 {
secret_key: "s3-secret-bytes".to_string(),
aws_role_web_identity_token_file: "/var/run/s3-private-token".to_string(),
..Default::default()
}),
..Default::default()
},
vec!["s3-secret-bytes", "/var/run/s3-private-token"],
),
(
"gcs",
TierConfig {
tier_type: TierType::GCS,
gcs: Some(TierGCS {
creds: r#"{"type":"service_account","private_key":"gcs-private-key-bytes"}"#.to_string(),
..Default::default()
}),
..Default::default()
},
vec!["gcs-private-key-bytes"],
),
(
"azure",
TierConfig {
tier_type: TierType::Azure,
azure: Some(TierAzure {
secret_key: "azure-account-secret-bytes".to_string(),
sp_auth: ServicePrincipalAuth {
client_secret: "azure-client-secret-bytes".to_string(),
..Default::default()
},
..Default::default()
}),
..Default::default()
},
vec!["azure-account-secret-bytes", "azure-client-secret-bytes"],
),
];
for (provider, config, secrets) in cases {
let api = serde_json::to_string(&config.redacted()).expect("redacted API config should serialize");
let debug = format!("{config:?}");
assert!(api.contains(TIER_CREDENTIAL_REDACTED), "{provider} API output should be visibly redacted");
assert!(
debug.contains(TIER_CREDENTIAL_REDACTED),
"{provider} Debug output should be visibly redacted"
);
for secret in secrets {
assert!(!api.contains(secret), "{provider} API output exposed credential bytes");
assert!(!debug.contains(secret), "{provider} Debug output exposed credential bytes");
}
}
}
#[test]
fn azure_static_account_redaction_preserves_an_empty_service_principal_secret() {
let config = TierConfig {
tier_type: TierType::Azure,
azure: Some(TierAzure {
secret_key: "azure-account-secret-bytes".to_string(),
sp_auth: ServicePrincipalAuth::default(),
..Default::default()
}),
..Default::default()
};
let api = serde_json::to_value(config.redacted()).expect("redacted Azure API config should serialize");
let debug = format!("{config:?}");
assert_eq!(api["azure"]["secretKey"], TIER_CREDENTIAL_REDACTED);
assert_eq!(api["azure"]["spAuth"]["client_secret"], "");
assert!(debug.contains("client_secret: \"\""));
assert!(!debug.contains("client_secret: \"REDACTED\""));
assert!(!debug.contains("azure-account-secret-bytes"));
}
}
crates/ecstore/src/services/tier/warm_backend.rs
+567 -34
View File
@@ -20,7 +20,7 @@
use crate::error::is_err_bucket_not_found;
use crate::services::tier::{
tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
tier_config::{TierConfig, TierType},
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
warm_backend_aliyun::WarmBackendAliyun,
@@ -55,18 +55,21 @@ use s3s::header::{
};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use time::OffsetDateTime;
use time::format_description::well_known::{Rfc2822, Rfc3339};
use tokio::io::AsyncReadExt;
use tracing::{info, warn};
pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
const PROBE_OBJECT: &str = "probeobject";
/// Largest object the S3-compatible warm backends accept for a multipart put.
pub(crate) const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
/// Part-count ceiling S3-compatible services impose on a multipart upload.
pub(crate) const MAX_PARTS_COUNT: i64 = 10000;
pub(crate) const WARM_BACKEND_PROBE_TIMEOUT: Duration = Duration::from_secs(30);
const WARM_BACKEND_PROBE_RECONCILE_INTERVAL: Duration = Duration::from_secs(1);
const WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT: Duration = Duration::from_secs(1);
#[derive(Default)]
pub struct WarmBackendGetOpts {
@@ -260,6 +263,23 @@ pub(crate) struct S3CompatibleWarmBackendParams<'a> {
pub validate_endpoint: fn(&url::Url) -> Result<(), rustfs_utils::egress::OutboundUrlError>,
}
/// Return the authority format accepted by `TransitionClient::new` while
/// retaining an explicitly configured port. `url::Url::host_str()` omits the
/// brackets needed when an IPv6 literal is combined with a port.
pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Error> {
let host = url
.host_str()
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
let port = url.port().unwrap_or(if url.scheme() == "https" { 443 } else { 80 });
if host.starts_with('[') && host.ends_with(']') {
Ok(format!("{host}:{port}"))
} else if host.contains(':') {
Ok(format!("[{host}]:{port}"))
} else {
Ok(format!("{host}:{port}"))
}
}
/// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
///
/// Credential, bucket, and endpoint validation run in this order because the
@@ -298,17 +318,11 @@ pub(crate) async fn new_s3_compatible_warm_backend(
bucket_lookup: params.bucket_lookup,
..Default::default()
};
let scheme = u.scheme();
let default_port = if scheme == "https" { 443 } else { 80 };
let host = u
.host_str()
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
// Runs after the host-presence check above (not immediately after Url::parse) so a
// host-less endpoint still reports this constructor's own "missing host" text instead of
// validate_endpoint's differently-worded rejection for the same input.
let endpoint = endpoint_authority(&u)?;
// Run the SSRF guard after the host-presence check so a host-less endpoint
// keeps this constructor's stable error text.
(params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
let client =
TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, params.provider_tag).await?;
let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
@@ -451,25 +465,187 @@ impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler {
}
}
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
w.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
let remote_version_id = w
.put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
.await
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
async fn remove_discovered_probe_candidate(
w: &WarmBackendImpl,
probe_object: &str,
candidate: TransitionCandidateProbe,
) -> Result<bool, std::io::Error> {
match candidate {
TransitionCandidateProbe::Missing => Ok(false),
TransitionCandidateProbe::VersionedPresent(remote_version_id) => {
w.remove_exact(probe_object, &remote_version_id).await?;
Ok(true)
}
TransitionCandidateProbe::UnversionedPresent => {
w.remove(probe_object, "").await?;
Ok(true)
}
TransitionCandidateProbe::Ambiguous => {
Err(std::io::Error::other("remote tier probe PUT produced multiple possible versions"))
}
TransitionCandidateProbe::Unsupported => Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"remote tier cannot discover the outcome of a probe PUT",
)),
}
}
if w.validate_remote_version_id(&remote_version_id).is_err() {
w.remove_exact(PROBE_OBJECT, &remote_version_id)
async fn compensate_uncertain_probe_put(
w: &WarmBackendImpl,
probe_object: &str,
settle_deadline: tokio::time::Instant,
) -> Result<(), std::io::Error> {
let final_deadline = settle_deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT;
let mut removed_any = false;
while tokio::time::Instant::now() < settle_deadline {
let candidate = match tokio::time::timeout_at(settle_deadline, w.probe_transition_candidate(probe_object)).await {
Ok(candidate) => candidate?,
Err(_) => break,
};
if matches!(candidate, TransitionCandidateProbe::Missing) && removed_any {
break;
}
removed_any |= tokio::time::timeout_at(settle_deadline, remove_discovered_probe_candidate(w, probe_object, candidate))
.await
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
return Err(ERR_TIER_INVALID_CONFIG.clone());
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out reconciling a remote tier probe PUT"))??;
let now = tokio::time::Instant::now();
if now >= settle_deadline {
break;
}
tokio::time::sleep_until(std::cmp::min(settle_deadline, now + WARM_BACKEND_PROBE_RECONCILE_INTERVAL)).await;
}
let read_result = w.get(PROBE_OBJECT, &remote_version_id, WarmBackendGetOpts::default()).await;
let remove_result = w.remove(PROBE_OBJECT, &remote_version_id).await;
//xhttp.DrainBody(r);
if read_result.is_err() || remove_result.is_err() {
let candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object))
.await
.map_err(|_| {
std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming the final remote tier probe state")
})??;
if !tokio::time::timeout_at(final_deadline, remove_discovered_probe_candidate(w, probe_object, candidate))
.await
.map_err(|_| {
std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out removing the final remote tier probe candidate")
})??
{
return Ok(());
}
let final_candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object))
.await
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming remote tier probe cleanup"))??;
match final_candidate {
TransitionCandidateProbe::Missing => Ok(()),
_ => Err(std::io::Error::other("remote tier probe cleanup could not be confirmed")),
}
}
fn probe_cleanup_incomplete_error() -> AdminError {
let mut err = ERR_TIER_PERM_ERR.clone();
err.message = "Remote tier probe outcome is uncertain; cleanup is incomplete".to_string();
err
}
async fn check_warm_backend_with_deadlines(
w: Option<&WarmBackendImpl>,
deadline: tokio::time::Instant,
cleanup_deadline: tokio::time::Instant,
) -> Result<(), AdminError> {
let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
let probe_object = format!("rustfs-tier-probe-{}", uuid::Uuid::new_v4());
let timeout_error = || {
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
err.message = "Timed out validating the remote tier mutation".to_string();
err
};
tokio::time::timeout_at(deadline, w.validate())
.await
.map_err(|_| timeout_error())?
.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
let put_result =
tokio::time::timeout_at(deadline, w.put(&probe_object, ReaderImpl::Body(Bytes::from_static(b"RustFS")), 6)).await;
let remote_version_id = match put_result {
Ok(Ok(remote_version_id)) => remote_version_id,
Ok(Err(_)) => {
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
Ok(()) => ERR_TIER_PERM_ERR.clone(),
Err(_) => probe_cleanup_incomplete_error(),
});
}
Err(_) => {
let err = timeout_error();
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
Ok(()) => err,
Err(_) => probe_cleanup_incomplete_error(),
});
}
};
// S3-family backends do not replay a failed request before returning `Ok`,
// while GCS discovers every matching generation. The authoritative probe
// below therefore closes the acknowledged-PUT path; only an error or
// timeout needs the longer visibility reconciliation above.
let authoritative_candidate = match tokio::time::timeout_at(deadline, w.probe_transition_candidate(&probe_object)).await {
Ok(Ok(candidate)) => candidate,
Ok(Err(_)) | Err(_) => {
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
Ok(()) => ERR_TIER_INVALID_CONFIG.clone(),
Err(_) => probe_cleanup_incomplete_error(),
});
}
};
let response_version_is_valid = w.validate_remote_version_id(&remote_version_id).is_ok();
let response_matches_candidate = match &authoritative_candidate {
TransitionCandidateProbe::UnversionedPresent => remote_version_id.is_empty(),
TransitionCandidateProbe::VersionedPresent(candidate_version) => candidate_version == &remote_version_id,
TransitionCandidateProbe::Missing | TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => false,
};
if !response_version_is_valid || !response_matches_candidate {
return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
Ok(()) => ERR_TIER_INVALID_CONFIG.clone(),
Err(_) => probe_cleanup_incomplete_error(),
});
}
let read_result = tokio::time::timeout_at(deadline, async {
let mut reader = w
.get(
&probe_object,
&remote_version_id,
WarmBackendGetOpts {
start_offset: 0,
length: 7,
},
)
.await
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
let mut body = Vec::new();
reader
.take(7)
.read_to_end(&mut body)
.await
.map_err(|_| ERR_TIER_PERM_ERR.clone())?;
if body != b"RustFS" {
return Err(ERR_TIER_PERM_ERR.clone());
}
Ok(())
})
.await
.map_err(|_| timeout_error())
.and_then(|result| result);
let cleanup_result = tokio::time::timeout_at(cleanup_deadline, async {
if !remove_discovered_probe_candidate(w, &probe_object, authoritative_candidate).await? {
return Err(std::io::Error::other("remote tier probe disappeared before cleanup"));
}
match w.probe_transition_candidate(&probe_object).await? {
TransitionCandidateProbe::Missing => Ok(()),
_ => Err(std::io::Error::other("remote tier probe remained after cleanup")),
}
})
.await;
if !matches!(cleanup_result, Ok(Ok(()))) {
return Err(probe_cleanup_incomplete_error());
}
if let Err(err) = read_result {
//if is_err_bucket_not_found(&err) {
// return Err(ERR_TIER_BUCKET_NOT_FOUND);
//}
@@ -477,12 +653,28 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin
return Err(ERR_TIER_MISSING_CREDENTIALS);
}*/
//else {
return Err(ERR_TIER_PERM_ERR.clone());
return Err(err);
//}
}
Ok(())
}
/// Validate a backend using a caller-owned deadline while retaining a bounded
/// reconciliation window for an uncertain probe PUT. The validation future is
/// kept alive through cleanup so an outer timeout cannot abandon the remote
/// probe object.
pub(crate) async fn check_warm_backend_until(
w: Option<&WarmBackendImpl>,
deadline: tokio::time::Instant,
) -> Result<(), AdminError> {
check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT).await
}
pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
let deadline = tokio::time::Instant::now() + WARM_BACKEND_PROBE_TIMEOUT;
check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_TIMEOUT).await
}
pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBackendImpl, AdminError> {
let mut d: Option<WarmBackendImpl> = None;
match tier.tier_type {
@@ -701,7 +893,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
let d: WarmBackendImpl = Box::new(MeteredWarmBackend { inner: d });
if probe {
d.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
check_warm_backend(Some(&d)).await?;
}
Ok(d)
}
@@ -754,6 +946,7 @@ pub(crate) async fn new_transition_candidate_reconciler(
#[cfg(test)]
mod tests {
use super::*;
use crate::services::tier::test_util::{MockWarmBackend, MockWarmOp};
use crate::services::tier::tier_config::TierWasabi;
use std::sync::{
Arc,
@@ -920,13 +1113,38 @@ mod tests {
struct RejectingProbeVersionBackend {
gets: Arc<AtomicUsize>,
present: Arc<std::sync::atomic::AtomicBool>,
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
returned_version: String,
}
struct RecordingProbeBackend {
get_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
present: Arc<std::sync::atomic::AtomicBool>,
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
remove_clears_candidate: bool,
fail_get: bool,
body: ProbeBody,
}
struct HangingProbePutBackend {
put_started: Arc<tokio::sync::Notify>,
present: Arc<std::sync::atomic::AtomicBool>,
probes: Arc<AtomicUsize>,
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
}
struct LateVisibleProbeBackend {
visible_at: tokio::time::Instant,
removed: Arc<std::sync::atomic::AtomicBool>,
probes: Arc<AtomicUsize>,
removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
}
#[derive(Clone, Copy)]
enum ProbeBody {
Exact,
Mismatch,
}
#[async_trait::async_trait]
@@ -976,7 +1194,7 @@ mod tests {
}
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
Ok(uuid::Uuid::nil().to_string())
Ok(self.returned_version.clone())
}
async fn put_with_meta(
@@ -999,10 +1217,19 @@ mod tests {
}
async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
self.present.store(false, Ordering::SeqCst);
self.removed_versions.lock().await.push(rv.to_string());
Ok(())
}
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
if self.present.load(Ordering::SeqCst) {
Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
} else {
Ok(TransitionCandidateProbe::Missing)
}
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
Ok(false)
}
@@ -1029,15 +1256,121 @@ mod tests {
if self.fail_get {
Err(std::io::Error::other("probe GET failed"))
} else {
Ok(ReadCloser::new(std::io::Cursor::new(Vec::new())))
match self.body {
ProbeBody::Exact => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFS".to_vec()))),
ProbeBody::Mismatch => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFT".to_vec()))),
}
}
}
async fn remove(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
if self.remove_clears_candidate {
self.present.store(false, Ordering::SeqCst);
}
self.removed_versions.lock().await.push(rv.to_string());
Ok(())
}
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
if self.present.load(Ordering::SeqCst) {
Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
} else {
Ok(TransitionCandidateProbe::Missing)
}
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
Ok(false)
}
}
#[async_trait::async_trait]
impl WarmBackend for HangingProbePutBackend {
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
self.put_started.notify_one();
std::future::pending().await
}
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
_meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
self.put(object, r, length).await
}
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
Err(std::io::Error::other("GET must not run after a timed out probe PUT"))
}
async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
Err(std::io::Error::other("generic remove must not replace exact probe cleanup"))
}
async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
self.present.store(false, Ordering::SeqCst);
self.removed_versions.lock().await.push(rv.to_string());
Ok(())
}
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.probes.fetch_add(1, Ordering::SeqCst);
if self.present.load(Ordering::SeqCst) {
Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
} else {
Ok(TransitionCandidateProbe::Missing)
}
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
Ok(false)
}
}
#[async_trait::async_trait]
impl WarmBackend for LateVisibleProbeBackend {
async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
Err(std::io::Error::new(
std::io::ErrorKind::ConnectionReset,
"probe PUT response was lost before the object became visible",
))
}
async fn put_with_meta(
&self,
object: &str,
r: ReaderImpl,
length: i64,
_meta: HashMap<String, String>,
) -> Result<String, std::io::Error> {
self.put(object, r, length).await
}
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
Err(std::io::Error::other("GET must not run after a lost probe PUT response"))
}
async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
Err(std::io::Error::other("generic remove must not replace exact probe cleanup"))
}
async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
self.removed.store(true, Ordering::SeqCst);
self.removed_versions.lock().await.push(rv.to_string());
Ok(())
}
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.probes.fetch_add(1, Ordering::SeqCst);
if tokio::time::Instant::now() >= self.visible_at && !self.removed.load(Ordering::SeqCst) {
Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
} else {
Ok(TransitionCandidateProbe::Missing)
}
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
Ok(false)
}
@@ -1098,13 +1431,15 @@ mod tests {
assert_eq!(probe, TransitionCandidateProbe::Unsupported);
}
#[tokio::test]
#[tokio::test(start_paused = true)]
async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() {
let gets = Arc::new(AtomicUsize::new(0));
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
gets: gets.clone(),
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
removed_versions: removed_versions.clone(),
returned_version: uuid::Uuid::nil().to_string(),
});
let err = check_warm_backend(Some(&backend))
@@ -1113,7 +1448,27 @@ mod tests {
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
assert_eq!(gets.load(Ordering::SeqCst), 0);
assert_eq!(removed_versions.lock().await.as_slice(), [uuid::Uuid::nil().to_string()]);
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test(start_paused = true)]
async fn check_warm_backend_rejects_empty_put_version_for_a_versioned_candidate() {
let gets = Arc::new(AtomicUsize::new(0));
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
gets: gets.clone(),
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
removed_versions: removed_versions.clone(),
returned_version: String::new(),
});
let err = check_warm_backend(Some(&backend))
.await
.expect_err("an empty PUT version must not read or generically delete a versioned object");
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
assert_eq!(gets.load(Ordering::SeqCst), 0);
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test]
@@ -1122,8 +1477,11 @@ mod tests {
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
get_versions: get_versions.clone(),
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
removed_versions: removed_versions.clone(),
remove_clears_candidate: true,
fail_get: false,
body: ProbeBody::Exact,
});
check_warm_backend(Some(&backend))
@@ -1140,8 +1498,11 @@ mod tests {
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
get_versions: get_versions.clone(),
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
removed_versions: removed_versions.clone(),
remove_clears_candidate: true,
fail_get: true,
body: ProbeBody::Exact,
});
let err = check_warm_backend(Some(&backend))
@@ -1153,6 +1514,169 @@ mod tests {
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test]
async fn check_warm_backend_removes_probe_after_body_mismatch() {
let get_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
get_versions,
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
removed_versions: removed_versions.clone(),
remove_clears_candidate: true,
fail_get: false,
body: ProbeBody::Mismatch,
});
let err = check_warm_backend(Some(&backend))
.await
.expect_err("a mismatched body should fail after cleanup");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test]
async fn check_warm_backend_rejects_a_stale_candidate_after_successful_delete() {
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
get_versions: Arc::new(tokio::sync::Mutex::new(Vec::new())),
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
removed_versions: removed_versions.clone(),
remove_clears_candidate: false,
fail_get: false,
body: ProbeBody::Exact,
});
let err = check_warm_backend(Some(&backend))
.await
.expect_err("cleanup must not succeed while the deleted candidate remains visible");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert!(err.message.contains("cleanup is incomplete"));
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test(start_paused = true)]
async fn check_warm_backend_reconciles_a_lost_put_response() {
let backend = MockWarmBackend::new();
backend.lose_next_put_response();
let driver: WarmBackendImpl = Box::new(backend.clone());
let err = check_warm_backend(Some(&driver))
.await
.expect_err("a lost probe PUT response must fail after compensation");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert_eq!(backend.object_count().await, 0);
assert_eq!(backend.exact_remove_count(), 1);
let operations = backend.op_log().await;
let put = operations.iter().find_map(|operation| match operation {
MockWarmOp::Put { object } => Some(object),
_ => None,
});
let probe = operations.iter().find_map(|operation| match operation {
MockWarmOp::Probe { object } => Some(object),
_ => None,
});
let remove = operations.iter().find_map(|operation| match operation {
MockWarmOp::Remove { object } => Some(object),
_ => None,
});
let (Some(put), Some(probe), Some(remove)) = (put, probe, remove) else {
panic!("lost-response compensation should PUT, probe, and remove");
};
assert_eq!(put, probe);
assert_eq!(probe, remove);
}
#[tokio::test(start_paused = true)]
async fn check_warm_backend_retries_until_a_late_put_becomes_visible() {
let probes = Arc::new(AtomicUsize::new(0));
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let driver: WarmBackendImpl = Box::new(LateVisibleProbeBackend {
visible_at: tokio::time::Instant::now() + Duration::from_secs(5),
removed: Arc::new(std::sync::atomic::AtomicBool::new(false)),
probes: probes.clone(),
removed_versions: removed_versions.clone(),
});
let err = check_warm_backend(Some(&driver))
.await
.expect_err("a late-visible probe PUT must still report the lost response");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert!(
probes.load(Ordering::SeqCst) > 5,
"reconciliation must not stop at the first Missing result"
);
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test]
async fn check_warm_backend_reports_incomplete_cleanup_without_guessing() {
for candidate in [TransitionCandidateProbe::Unsupported, TransitionCandidateProbe::Ambiguous] {
let backend = MockWarmBackend::new();
backend.set_transition_candidate_probe_override(Some(candidate)).await;
backend.lose_next_put_response();
let driver: WarmBackendImpl = Box::new(backend.clone());
let err = check_warm_backend(Some(&driver))
.await
.expect_err("an uncertain candidate must fail without a guessed delete");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert!(err.message.contains("cleanup is incomplete"));
assert_eq!(backend.remove_count().await, 0);
assert_eq!(backend.object_count().await, 1);
}
}
#[tokio::test]
async fn check_warm_backend_reports_an_exact_cleanup_failure() {
let backend = MockWarmBackend::new();
backend.set_remove_failure(true);
backend.lose_next_put_response();
let driver: WarmBackendImpl = Box::new(backend.clone());
let err = check_warm_backend(Some(&driver))
.await
.expect_err("an exact cleanup failure must replace the ambiguous PUT error");
assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
assert!(err.message.contains("cleanup is incomplete"));
assert_eq!(backend.exact_remove_count(), 1);
assert_eq!(backend.object_count().await, 1);
}
#[tokio::test(start_paused = true)]
async fn check_warm_backend_reconciles_a_timed_out_put() {
let put_started = Arc::new(tokio::sync::Notify::new());
let probes = Arc::new(AtomicUsize::new(0));
let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let driver: WarmBackendImpl = Box::new(HangingProbePutBackend {
put_started: put_started.clone(),
present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
probes: probes.clone(),
removed_versions: removed_versions.clone(),
});
let check = check_warm_backend(Some(&driver));
tokio::pin!(check);
tokio::select! {
_ = put_started.notified() => {}
result = &mut check => panic!("probe completed before the PUT timeout: {result:?}"),
}
tokio::time::advance(WARM_BACKEND_PROBE_TIMEOUT + Duration::from_millis(1)).await;
let err = check.await.expect_err("a timed out probe PUT must fail after compensation");
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
assert!(
probes.load(Ordering::SeqCst) > 1,
"timed-out PUT reconciliation must keep checking through the visibility window"
);
assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
}
#[tokio::test]
async fn new_wasabi_backend_honors_probe_flag() {
let tier = TierConfig {
@@ -1298,6 +1822,15 @@ mod tests {
assert_eq!(insecure.client.endpoint_url.port_or_known_default(), Some(80));
}
#[test]
fn endpoint_authority_preserves_ipv6_brackets_and_explicit_port() {
let url = url::Url::parse("https://[2001:db8::1]:9443").expect("the IPv6 endpoint should parse");
assert_eq!(
endpoint_authority(&url).expect("the endpoint should have an authority"),
"[2001:db8::1]:9443"
);
}
#[tokio::test]
async fn s3_compatible_backend_strips_only_a_trailing_prefix_separator() {
let mut params = s3_compatible_params("http://tier.example.com:9000");
crates/ecstore/src/services/tier/warm_backend_aliyun.rs
+5 -1
View File
@@ -23,7 +23,7 @@ use std::collections::HashMap;
use crate::services::tier::{
tier_config::TierAliyun,
warm_backend::{
S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
new_s3_compatible_warm_backend, optimal_part_size,
},
warm_backend_s3::WarmBackendS3,
@@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendAliyun {
self.0.remove(object, rv).await
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.0.probe_transition_candidate(object).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
crates/ecstore/src/services/tier/warm_backend_azure.rs
+109 -1
View File
@@ -23,7 +23,7 @@ use std::collections::HashMap;
use crate::services::tier::{
tier_config::TierAzure,
warm_backend::{
S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
new_s3_compatible_warm_backend, optimal_part_size,
},
warm_backend_s3::WarmBackendS3,
@@ -89,6 +89,16 @@ impl WarmBackend for WarmBackendAzure {
self.0.remove(object, rv).await
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
// Azure currently uses the shared S3/SigV4 transport, but its normal
// object path cannot persist exact remote versions across mixed
// RustFS releases. The mutation probe may still detect and precisely
// remove a versioned test object before rejecting that configuration.
self.0
.probe_transition_candidate_with_raw_version_header(object, "x-amz-version-id")
.await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
@@ -98,6 +108,23 @@ impl WarmBackend for WarmBackendAzure {
mod tests {
use super::*;
use crate::services::tier::tier_config::TierAzure;
use rustfs_s3_client::{
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{Options, TransitionClient, TransitionCore},
};
use std::sync::Arc;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
async fn read_request_head(stream: &mut tokio::net::TcpStream) -> String {
let mut request = Vec::new();
let mut buffer = [0_u8; 1024];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
let read = stream.read(&mut buffer).await.expect("fixture request should be readable");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
}
String::from_utf8_lossy(&request).into_owned()
}
/// The SSRF guard itself is exercised once, generically, in
/// `warm_backend::tests` (see backlog#2040/backlog#2041 and
@@ -119,4 +146,85 @@ mod tests {
Err(err) => assert!(err.to_string().contains("not allowed")),
}
}
#[tokio::test]
async fn versioned_candidate_cleanup_uses_the_exact_s3_version_without_enabling_data_versions() {
let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut get_stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
let get_request = read_request_head(&mut get_stream).await;
get_stream
.write_all(
b"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: azure-version\r\nConnection: close\r\n\r\nx",
)
.await
.expect("fixture should write candidate GET response");
let (mut delete_stream, _) = listener.accept().await.expect("fixture should accept exact DELETE");
let delete_request = read_request_head(&mut delete_stream).await;
delete_stream
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.await
.expect("fixture should write exact DELETE response");
(get_request, delete_request)
});
let client = Arc::new(
TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"azure",
)
.await
.expect("fixture client should build"),
);
let backend = WarmBackendAzure(WarmBackendS3 {
core: TransitionCore(Arc::clone(&client)),
client,
bucket: "bucket".to_string(),
prefix: String::new(),
storage_class: String::new(),
});
assert!(
!backend.0.client.provider_version_capabilities().exact_get_delete,
"probe-only version discovery must not change Azure's persisted data-path contract"
);
let candidate = backend
.probe_transition_candidate("probe")
.await
.expect("Azure candidate should be discovered");
assert_eq!(candidate, TransitionCandidateProbe::VersionedPresent("azure-version".to_string()));
backend
.remove_exact("probe", "azure-version")
.await
.expect("Azure candidate should be deleted by exact version");
let (get_request, delete_request) = fixture.await.expect("fixture should join");
assert!(get_request.to_ascii_lowercase().contains("\r\nrange: bytes=0-0\r\n"));
assert!(
delete_request
.lines()
.next()
.is_some_and(|line| line.contains("DELETE /bucket/probe?versionId=azure-version "))
);
}
}
crates/ecstore/src/services/tier/warm_backend_gcs.rs
+492 -13
View File
@@ -18,13 +18,13 @@
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::future::Future;
use std::io::{Error, ErrorKind};
use std::sync::Arc;
use bytes::Bytes;
use google_cloud_auth::credentials::Credentials;
use google_cloud_auth::credentials::user_account::Builder;
use google_cloud_auth::credentials::service_account::Builder;
use google_cloud_storage as gcs;
use google_cloud_storage::client::Storage;
use google_cloud_storage::client::StorageControl;
@@ -32,7 +32,7 @@ use std::convert::TryFrom;
use crate::services::tier::{
tier_config::TierGCS,
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts},
};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
@@ -43,6 +43,7 @@ use rustfs_utils::egress::validate_outbound_url;
use tracing::warn;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MAX_GCS_CANDIDATE_PAGES: usize = 64;
fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
if remote_version.is_empty() {
@@ -57,6 +58,85 @@ fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
Ok(Some(generation))
}
fn append_gcs_chunk<E: std::fmt::Display>(
contents: &mut Vec<u8>,
chunk: Result<Bytes, E>,
max_response_bytes: Option<usize>,
) -> std::io::Result<()> {
let chunk = chunk.map_err(|err| std::io::Error::other(err.to_string()))?;
if max_response_bytes.is_some_and(|limit| contents.len().saturating_add(chunk.len()) > limit) {
return Err(std::io::Error::new(
ErrorKind::InvalidData,
"GCS object response exceeded the configured byte limit",
));
}
contents.extend_from_slice(&chunk);
Ok(())
}
fn gcs_bucket_resource_name(bucket: &str) -> String {
format!("projects/_/buckets/{bucket}")
}
struct GcsCandidateObject {
name: String,
generation: i64,
}
struct GcsCandidatePage {
objects: Vec<GcsCandidateObject>,
next_page_token: String,
}
async fn probe_exact_gcs_candidate<F, Fut>(
remote_object: &str,
mut fetch_page: F,
) -> Result<TransitionCandidateProbe, std::io::Error>
where
F: FnMut(String) -> Fut,
Fut: Future<Output = Result<GcsCandidatePage, std::io::Error>>,
{
let mut page_token = String::new();
let mut seen_page_tokens = HashSet::new();
let mut generation = None;
let mut pages_seen = 0_usize;
loop {
pages_seen += 1;
if pages_seen > MAX_GCS_CANDIDATE_PAGES {
return Err(std::io::Error::new(
ErrorKind::InvalidData,
"GCS candidate listing exceeded the page limit",
));
}
let response = fetch_page(page_token.clone()).await?;
for candidate in response.objects.iter().filter(|candidate| candidate.name == remote_object) {
if candidate.generation <= 0 {
return Err(std::io::Error::new(
ErrorKind::InvalidData,
"GCS candidate listing returned a non-positive generation",
));
}
if generation.replace(candidate.generation).is_some() {
return Ok(TransitionCandidateProbe::Ambiguous);
}
}
if response.next_page_token.is_empty() {
break;
}
if !seen_page_tokens.insert(response.next_page_token.clone()) {
return Err(std::io::Error::new(ErrorKind::InvalidData, "GCS candidate listing repeated a page token"));
}
page_token = response.next_page_token;
}
Ok(match generation {
Some(generation) => TransitionCandidateProbe::VersionedPresent(generation.to_string()),
None => TransitionCandidateProbe::Missing,
})
}
pub struct WarmBackendGCS {
pub client: Arc<Storage>,
pub control: Arc<StorageControl>,
@@ -80,8 +160,8 @@ impl WarmBackendGCS {
.map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
}
let authorized_user = serde_json::from_str(&conf.creds)?;
let credentials = Builder::new(authorized_user)
let service_account = serde_json::from_str(&conf.creds)?;
let credentials = Builder::new(service_account)
//.with_retry_policy(AlwaysRetry.with_attempt_limit(3))
//.with_backoff_policy(backoff)
.build()
@@ -98,7 +178,11 @@ impl WarmBackendGCS {
let client = Arc::new(client);
// Control-plane client: the data-plane `Storage` client cannot delete or list objects;
// delete_object/list_objects live on StorageControl.
let Ok(control) = StorageControl::builder().with_credentials(credentials).build().await else {
let mut control_builder = StorageControl::builder().with_credentials(credentials);
if !conf.endpoint.is_empty() {
control_builder = control_builder.with_endpoint(conf.endpoint.clone());
}
let Ok(control) = control_builder.build().await else {
return Err(std::io::Error::other("StorageControl::builder error"));
};
let control = Arc::new(control);
@@ -136,9 +220,10 @@ impl WarmBackend for WarmBackendGCS {
ReaderImpl::Body(content_body) => content_body.to_vec(),
ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
};
let bucket = gcs_bucket_resource_name(&self.bucket);
let Ok(res) = Box::pin(
self.client
.write_object(&self.bucket, &self.get_dest(object), Bytes::from(d))
.write_object(&bucket, &self.get_dest(object), Bytes::from(d))
.send_buffered(),
)
.await
@@ -154,7 +239,9 @@ impl WarmBackend for WarmBackendGCS {
}
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
let bucket = gcs_bucket_resource_name(&self.bucket);
let mut req = self.client.read_object(&bucket, &self.get_dest(object));
let mut max_response_bytes = None;
if let Some(generation) = parse_generation(rv)? {
req = req.set_generation(generation);
}
@@ -170,6 +257,11 @@ impl WarmBackend for WarmBackendGCS {
.length
.try_into()
.map_err(|_| std::io::Error::other("invalid range: negative length"))?;
max_response_bytes = Some(
opts.length
.try_into()
.map_err(|_| std::io::Error::other("invalid range: length does not fit in memory"))?,
);
req = req.set_read_range(google_cloud_storage::model_ext::ReadRange::segment(offset, count));
}
@@ -177,8 +269,8 @@ impl WarmBackend for WarmBackendGCS {
return Err(std::io::Error::other("read_object error"));
};
let mut contents = Vec::new();
while let Ok(Some(chunk)) = reader.next().await.transpose() {
contents.extend_from_slice(&chunk);
while let Some(chunk) = reader.next().await {
append_gcs_chunk(&mut contents, chunk, max_response_bytes)?;
}
Ok(ReadCloser::new(std::io::Cursor::new(contents)))
}
@@ -190,7 +282,7 @@ impl WarmBackend for WarmBackendGCS {
let mut req = self
.control
.delete_object()
.set_bucket(format!("projects/_/buckets/{}", self.bucket))
.set_bucket(gcs_bucket_resource_name(&self.bucket))
.set_object(self.get_dest(object));
if let Some(generation) = parse_generation(rv)? {
req = req.set_generation(generation);
@@ -199,13 +291,47 @@ impl WarmBackend for WarmBackendGCS {
Ok(())
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
let remote_object = self.get_dest(object);
let parent = gcs_bucket_resource_name(&self.bucket);
probe_exact_gcs_candidate(&remote_object, |page_token| {
let control = self.control.clone();
let parent = parent.clone();
let prefix = remote_object.clone();
async move {
let response = control
.list_objects()
.set_parent(parent)
.set_prefix(prefix)
.set_versions(true)
.set_page_size(2)
.set_page_token(page_token)
.send()
.await
.map_err(|err| std::io::Error::other(err.to_string()))?;
Ok(GcsCandidatePage {
objects: response
.objects
.into_iter()
.map(|candidate| GcsCandidateObject {
name: candidate.name,
generation: candidate.generation,
})
.collect(),
next_page_token: response.next_page_token,
})
}
})
.await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
// Scope the listing to this tier's prefix (matching the other warm backends) and only
// need to know whether a single object exists.
let resp = self
.control
.list_objects()
.set_parent(format!("projects/_/buckets/{}", self.bucket))
.set_parent(gcs_bucket_resource_name(&self.bucket))
.set_prefix(self.prefix.clone())
.set_page_size(1)
.send()
@@ -218,10 +344,126 @@ impl WarmBackend for WarmBackendGCS {
#[cfg(test)]
mod tests {
use super::GcsCandidateObject;
use super::GcsCandidatePage;
use super::MAX_GCS_CANDIDATE_PAGES;
use super::WarmBackendGCS;
use super::append_gcs_chunk;
use super::gcs_bucket_resource_name;
use super::parse_generation;
use super::probe_exact_gcs_candidate;
use crate::services::tier::tier_config::TierGCS;
use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts};
use google_cloud_auth::credentials::anonymous::Builder as Anonymous;
use google_cloud_storage::client::{Storage, StorageControl};
use std::io::ErrorKind;
use std::sync::Arc;
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
async fn serve_data_plane_fixture(listener: TcpListener) -> Vec<String> {
let upload_body = r#"{"name":"probe","bucket":"tier-bucket","generation":"123"}"#;
let responses = [
format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{upload_body}",
upload_body.len()
),
"HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-6/7\r\nx-goog-generation: 123\r\ncontent-length: 7\r\nconnection: close\r\n\r\nRustFS!"
.to_string(),
"HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-7/8\r\nx-goog-generation: 123\r\ncontent-length: 8\r\nconnection: close\r\n\r\nRustFS!!"
.to_string(),
];
let mut requests = Vec::new();
for response in responses {
let (mut stream, _) = listener.accept().await.expect("the GCS fixture should accept a request");
let mut request = Vec::new();
loop {
let mut chunk = [0_u8; 1024];
let count = stream
.read(&mut chunk)
.await
.expect("the GCS fixture should read request headers");
if count == 0 {
break;
}
request.extend_from_slice(&chunk[..count]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
let header_end = request
.windows(4)
.position(|window| window == b"\r\n\r\n")
.map(|position| position + 4)
.expect("the GCS fixture should receive complete request headers");
let headers = String::from_utf8_lossy(&request[..header_end]);
if headers.lines().any(|line| line.eq_ignore_ascii_case("expect: 100-continue")) {
stream
.write_all(b"HTTP/1.1 100 Continue\r\n\r\n")
.await
.expect("the GCS fixture should acknowledge 100-continue");
}
let content_length = headers
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse::<usize>().expect("content-length should be numeric"))
})
.unwrap_or_default();
while request.len() < header_end.saturating_add(content_length) {
let mut chunk = [0_u8; 1024];
let count = stream
.read(&mut chunk)
.await
.expect("the GCS fixture should read the request body");
if count == 0 {
break;
}
request.extend_from_slice(&chunk[..count]);
}
requests.push(String::from_utf8_lossy(&request).into_owned());
stream
.write_all(response.as_bytes())
.await
.expect("the GCS fixture should write its response");
}
requests
}
fn candidate_page(objects: &[(&str, i64)], next_page_token: &str) -> GcsCandidatePage {
GcsCandidatePage {
objects: objects
.iter()
.map(|(name, generation)| GcsCandidateObject {
name: (*name).to_string(),
generation: *generation,
})
.collect(),
next_page_token: next_page_token.to_string(),
}
}
async fn probe_candidate_pages(
remote_object: &str,
pages: Vec<GcsCandidatePage>,
) -> (Result<TransitionCandidateProbe, std::io::Error>, Vec<String>) {
let mut pages = pages.into_iter();
let mut requested_tokens = Vec::new();
let result = probe_exact_gcs_candidate(remote_object, |page_token| {
requested_tokens.push(page_token);
std::future::ready(
pages
.next()
.ok_or_else(|| std::io::Error::new(ErrorKind::UnexpectedEof, "test fixture ran out of GCS pages")),
)
})
.await;
(result, requested_tokens)
}
#[test]
fn generation_parser_preserves_exact_numeric_versions() {
@@ -241,6 +483,243 @@ mod tests {
}
}
#[test]
fn body_collection_propagates_an_error_after_a_complete_prefix() {
let mut contents = Vec::new();
append_gcs_chunk::<std::io::Error>(&mut contents, Ok(bytes::Bytes::from_static(b"RustFS")), Some(7))
.expect("the prefix chunk should be collected");
let err = append_gcs_chunk(&mut contents, Err(std::io::Error::other("trailing stream failure")), Some(7))
.expect_err("a trailing stream error must not be mistaken for EOF");
assert_eq!(contents, b"RustFS");
assert!(err.to_string().contains("trailing stream failure"));
}
#[test]
fn body_collection_rejects_a_chunk_that_exceeds_the_probe_limit() {
let mut contents = Vec::new();
let err = append_gcs_chunk::<std::io::Error>(&mut contents, Ok(bytes::Bytes::from_static(b"RustFSxx")), Some(7))
.expect_err("the GCS collection layer must reject an oversized probe response");
assert!(contents.is_empty());
assert_eq!(err.kind(), ErrorKind::InvalidData);
}
#[tokio::test]
async fn candidate_probe_finds_exact_name_on_first_or_later_page() {
let (first, first_tokens) =
probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", 7)], "")]).await;
assert_eq!(
first.expect("an exact first-page object should be discovered"),
TransitionCandidateProbe::VersionedPresent("7".to_string())
);
assert_eq!(first_tokens, [""]);
let (later, later_tokens) = probe_candidate_pages(
"prefix/object",
vec![
candidate_page(&[("prefix/object-shadow", 8)], "next"),
candidate_page(&[("prefix/object", 9)], ""),
],
)
.await;
assert_eq!(
later.expect("an exact later-page object should be discovered"),
TransitionCandidateProbe::VersionedPresent("9".to_string())
);
assert_eq!(later_tokens, ["", "next"]);
}
#[tokio::test]
async fn candidate_probe_ignores_non_exact_prefix_matches() {
let (probe, _) = probe_candidate_pages(
"prefix/object",
vec![candidate_page(
&[("prefix/object-shadow", 8), ("prefix/object/child", 9), ("prefix/object", 7)],
"",
)],
)
.await;
assert_eq!(
probe.expect("prefix-only matches should not hide the exact object"),
TransitionCandidateProbe::VersionedPresent("7".to_string())
);
}
#[tokio::test]
async fn candidate_probe_reports_duplicate_exact_names_as_ambiguous() {
let (probe, _) = probe_candidate_pages(
"prefix/object",
vec![
candidate_page(&[("prefix/object", 7)], "next"),
candidate_page(&[("prefix/object", 8)], ""),
],
)
.await;
assert_eq!(
probe.expect("multiple exact generations should produce a conservative result"),
TransitionCandidateProbe::Ambiguous
);
}
#[tokio::test]
async fn candidate_probe_reports_missing_without_an_exact_name() {
let (probe, _) = probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object-shadow", 8)], "")]).await;
assert_eq!(
probe.expect("a complete listing without an exact name should be definitive"),
TransitionCandidateProbe::Missing
);
}
#[tokio::test]
async fn candidate_probe_rejects_non_positive_generations() {
for generation in [0, -1] {
let (probe, _) =
probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", generation)], "")]).await;
let err = probe.expect_err("a non-positive GCS generation must fail closed");
assert_eq!(err.kind(), ErrorKind::InvalidData, "generation {generation}");
}
}
#[tokio::test]
async fn candidate_probe_rejects_a_page_token_that_does_not_advance() {
let (probe, requested_tokens) =
probe_candidate_pages("prefix/object", vec![candidate_page(&[], "next"), candidate_page(&[], "next")]).await;
let err = probe.expect_err("a repeated GCS page token must fail closed");
assert_eq!(err.kind(), ErrorKind::InvalidData);
assert_eq!(requested_tokens, ["", "next"]);
}
#[tokio::test]
async fn candidate_probe_rejects_a_non_adjacent_page_token_cycle() {
let (probe, requested_tokens) = probe_candidate_pages(
"prefix/object",
vec![candidate_page(&[], "a"), candidate_page(&[], "b"), candidate_page(&[], "a")],
)
.await;
let err = probe.expect_err("a non-adjacent GCS page token cycle must fail closed");
assert_eq!(err.kind(), ErrorKind::InvalidData);
assert_eq!(requested_tokens, ["", "a", "b"]);
}
#[tokio::test]
async fn candidate_probe_rejects_an_unbounded_unique_token_chain() {
let pages = (0..MAX_GCS_CANDIDATE_PAGES)
.map(|index| candidate_page(&[], &format!("token-{index}")))
.collect();
let (probe, requested_tokens) = probe_candidate_pages("prefix/object", pages).await;
let err = probe.expect_err("an unbounded unique page-token chain must fail closed");
assert_eq!(err.kind(), ErrorKind::InvalidData);
assert_eq!(requested_tokens.len(), MAX_GCS_CANDIDATE_PAGES);
}
#[tokio::test]
async fn plain_bucket_reaches_gcs_put_and_get_transport_with_resource_name() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("the GCS fixture should bind a loopback port");
let endpoint = format!("http://{}", listener.local_addr().expect("the GCS fixture should have a local address"));
let fixture = tokio::spawn(serve_data_plane_fixture(listener));
let credentials = Anonymous::new().build();
let client = Storage::builder()
.with_endpoint(endpoint.clone())
.with_credentials(credentials.clone())
.build()
.await
.expect("the GCS data client should build");
let control = StorageControl::builder()
.with_endpoint(endpoint)
.with_credentials(credentials)
.build()
.await
.expect("the GCS control client should build");
let backend = WarmBackendGCS {
client: Arc::new(client),
control: Arc::new(control),
bucket: "tier-bucket".to_string(),
prefix: String::new(),
};
let (version, body, oversized_error_kind, requests) = tokio::time::timeout(Duration::from_secs(5), async {
let version = backend
.put(
"probe",
rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"RustFS")),
6,
)
.await
.expect("a plain configured bucket should reach the GCS upload transport");
let mut reader = backend
.get(
"probe",
&version,
WarmBackendGetOpts {
start_offset: 0,
length: 7,
},
)
.await
.expect("a plain configured bucket should reach the GCS read transport");
let mut body = Vec::new();
reader
.read_to_end(&mut body)
.await
.expect("the fixture body should be readable");
let oversized_error = match backend
.get(
"probe",
&version,
WarmBackendGetOpts {
start_offset: 0,
length: 7,
},
)
.await
{
Ok(_) => panic!("an eight-byte response must not pass a seven-byte collection limit"),
Err(err) => err,
};
let requests = fixture.await.expect("the GCS fixture task should finish");
(version, body, oversized_error.kind(), requests)
})
.await
.expect("the GCS data-plane requests should not be rejected before transport");
assert_eq!(gcs_bucket_resource_name("tier-bucket"), "projects/_/buckets/tier-bucket");
assert_eq!(version, "123");
assert_eq!(body, b"RustFS!");
assert_eq!(oversized_error_kind, ErrorKind::InvalidData);
assert!(
requests[0].starts_with("POST /upload/storage/v1/b/tier-bucket/o?"),
"unexpected upload request line: {}",
requests[0].lines().next().unwrap_or_default()
);
assert!(
requests[1].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"),
"unexpected read request line: {}",
requests[1].lines().next().unwrap_or_default()
);
assert!(
requests[1].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"),
"the GCS probe read must preserve its seven-byte range"
);
assert!(
requests[2].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"),
"unexpected oversized read request line: {}",
requests[2].lines().next().unwrap_or_default()
);
assert!(
requests[2].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"),
"the oversized response must be fetched under the same seven-byte request boundary"
);
}
#[tokio::test]
async fn new_rejects_loopback_endpoint_before_credential_setup() {
let conf = TierGCS {
crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs
+5 -1
View File
@@ -23,7 +23,7 @@ use std::collections::HashMap;
use crate::services::tier::{
tier_config::TierHuaweicloud,
warm_backend::{
S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
new_s3_compatible_warm_backend, optimal_part_size,
},
warm_backend_s3::WarmBackendS3,
@@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendHuaweicloud {
self.0.remove(object, rv).await
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.0.probe_transition_candidate(object).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
crates/ecstore/src/services/tier/warm_backend_s3.rs
+272 -38
View File
@@ -26,11 +26,12 @@ use crate::services::tier::{
tier_config::TierS3,
warm_backend::{
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
build_transition_put_options,
build_transition_put_options, endpoint_authority,
},
};
use http::HeaderMap;
use rustfs_s3_client::{
api_error_response::to_error_response,
api_get_options::GetObjectOptions,
api_list::ListObjectsOptions,
api_put_object::PutObjectOptions,
@@ -43,7 +44,7 @@ use rustfs_s3_client::{
};
use rustfs_utils::egress::validate_outbound_url;
use rustfs_utils::path::SLASH_SEPARATOR;
use s3s::dto::BucketVersioningStatus;
use s3s::{S3ErrorCode, dto::BucketVersioningStatus};
pub struct WarmBackendS3 {
pub client: Arc<TransitionClient>,
@@ -74,6 +75,19 @@ fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result<RemoteBu
})
}
fn bounded_get_range(opts: &WarmBackendGetOpts) -> Result<Option<(i64, i64)>, std::io::Error> {
if opts.start_offset < 0 || opts.length <= 0 {
return Ok(None);
}
usize::try_from(opts.length)
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: length does not fit in memory"))?;
let end_offset = opts
.start_offset
.checked_add(opts.length - 1)
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: end offset overflow"))?;
Ok(Some((opts.start_offset, end_offset)))
}
impl WarmBackendS3 {
pub async fn new(conf: &TierS3, _tier: &str) -> Result<Self, std::io::Error> {
Self::new_with_bucket_lookup(conf, BucketLookupType::BucketLookupAuto, "s3").await
@@ -132,10 +146,8 @@ impl WarmBackendS3 {
bucket_lookup,
..Default::default()
};
let host = u
.host()
.ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
let client = TransitionClient::new(&host.to_string(), opts, tier_type).await?;
let endpoint = endpoint_authority(&u)?;
let client = TransitionClient::new(&endpoint, opts, tier_type).await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
@@ -177,10 +189,8 @@ impl WarmBackendS3 {
if !rv.is_empty() {
gopts.version_id = rv.to_string();
}
if opts.start_offset >= 0 && opts.length > 0 {
gopts
.set_range(opts.start_offset, opts.start_offset + opts.length - 1)
.map_err(std::io::Error::other)?;
if let Some((start_offset, end_offset)) = bounded_get_range(&opts)? {
gopts.set_range(start_offset, end_offset)?;
}
let (_, headers, reader) = self.core.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
Ok((headers, reader))
@@ -191,36 +201,64 @@ impl WarmBackendS3 {
remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
}
async fn probe_transition_candidate_versions(
async fn probe_current_transition_candidate_with_header(
&self,
object: &str,
bucket_versioning: RemoteBucketVersioning,
raw_version_header: Option<&'static str>,
) -> Result<TransitionCandidateProbe, std::io::Error> {
let remote_object = self.get_dest(object);
let mut opts = ListObjectsOptions::default();
opts.set("prefix", &remote_object);
opts.set("max-keys", "1000");
let mut key_marker = String::new();
let mut version_id_marker = String::new();
let mut candidates = TransitionCandidateVersions::default();
loop {
let versions = self
.client
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
.await?;
candidates.extend(&remote_object, &versions);
if candidates.is_ambiguous() {
return Ok(TransitionCandidateProbe::Ambiguous);
match self
.get_with_headers(
object,
"",
WarmBackendGetOpts {
start_offset: 0,
length: 1,
},
)
.await
{
Ok((headers, _)) => {
let version_id = match raw_version_header {
Some(header_name) => match headers.get(header_name) {
Some(value) => {
let version_id = value.to_str().map_err(|_| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"remote object version id is not valid ASCII",
)
})?;
validate_remote_version_id(version_id)?;
Some(version_id)
}
None => None,
},
None => self.client.raw_version_id(&headers)?,
};
Ok(match version_id {
Some(version_id) => TransitionCandidateProbe::VersionedPresent(version_id.to_string()),
None => TransitionCandidateProbe::UnversionedPresent,
})
}
if !versions.is_truncated {
return classify_transition_candidates(candidates, bucket_versioning);
Err(err) => {
let response = to_error_response(&err);
if response.code == S3ErrorCode::NoSuchKey {
Ok(TransitionCandidateProbe::Missing)
} else {
Err(err)
}
}
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
}
}
pub(crate) async fn probe_transition_candidate_with_raw_version_header(
&self,
object: &str,
raw_version_header: &'static str,
) -> Result<TransitionCandidateProbe, std::io::Error> {
self.probe_current_transition_candidate_with_header(object, Some(raw_version_header))
.await
}
async fn probe_transition_candidate_identity(
&self,
object: &str,
@@ -343,6 +381,7 @@ struct TransitionCandidateVersions {
}
impl TransitionCandidateVersions {
#[cfg(test)]
fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
if self.version_id.is_some() {
@@ -353,10 +392,6 @@ impl TransitionCandidateVersions {
}
}
fn is_ambiguous(&self) -> bool {
self.ambiguous
}
fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
if self.ambiguous {
return TransitionCandidateProbe::Ambiguous;
@@ -380,6 +415,8 @@ impl TransitionCandidateVersions {
mod tests {
use super::*;
use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
#[tokio::test]
async fn new_rejects_loopback_endpoint_before_network_setup() {
@@ -398,6 +435,204 @@ mod tests {
}
}
#[tokio::test]
async fn new_preserves_an_explicit_endpoint_port() {
let conf = TierS3 {
endpoint: "https://tier.example.com:9443".to_string(),
bucket: "tier-bucket".to_string(),
access_key: "access".to_string(),
secret_key: "secret".to_string(),
region: "us-east-1".to_string(),
..Default::default()
};
let backend = WarmBackendS3::new(&conf, "tier")
.await
.expect("a well-formed S3 endpoint should initialize without network I/O");
assert_eq!(backend.client.endpoint_url.host_str(), Some("tier.example.com"));
assert_eq!(backend.client.endpoint_url.port(), Some(9443));
}
#[tokio::test]
async fn overflowing_get_range_is_rejected_before_network_io() {
let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let client = Arc::new(
TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"s3",
)
.await
.expect("fixture client should build"),
);
let backend = WarmBackendS3 {
core: TransitionCore(Arc::clone(&client)),
client,
bucket: "bucket".to_string(),
prefix: String::new(),
storage_class: String::new(),
};
let err = backend
.get_with_headers(
"probe",
"",
WarmBackendGetOpts {
start_offset: i64::MAX,
length: 2,
},
)
.await
.expect_err("an overflowing range must fail before issuing a GET");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert!(
tokio::time::timeout(Duration::from_millis(100), listener.accept())
.await
.is_err()
);
}
async fn candidate_probe_fixture() -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let responses = [
"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx",
"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx",
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
"HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
];
let mut requests = Vec::new();
for response in responses {
let (mut stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
requests.push(String::from_utf8_lossy(&request).into_owned());
stream
.write_all(response.as_bytes())
.await
.expect("fixture should write candidate response");
}
requests
});
let client = Arc::new(
TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"s3",
)
.await
.expect("fixture client should build"),
);
Some((
WarmBackendS3 {
core: TransitionCore(Arc::clone(&client)),
client,
bucket: "bucket".to_string(),
prefix: String::new(),
storage_class: String::new(),
},
fixture,
))
}
#[tokio::test]
async fn candidate_probe_uses_only_exact_bounded_get_permissions() {
let Some((backend, fixture)) = candidate_probe_fixture().await else {
return;
};
assert_eq!(
backend
.probe_transition_candidate("versioned-probe")
.await
.expect("versioned candidate should be discovered"),
TransitionCandidateProbe::VersionedPresent("opaque-version".to_string())
);
assert_eq!(
backend
.probe_transition_candidate("unversioned-probe")
.await
.expect("unversioned candidate should be discovered"),
TransitionCandidateProbe::UnversionedPresent
);
assert_eq!(
backend
.probe_transition_candidate("missing-probe")
.await
.expect("a missing key should be classified"),
TransitionCandidateProbe::Missing
);
assert_eq!(
backend
.probe_transition_candidate("provider-missing-probe")
.await
.expect("a provider-specific missing code should be classified"),
TransitionCandidateProbe::Missing
);
let err = backend
.probe_transition_candidate("forbidden-probe")
.await
.expect_err("an authorization failure must not be mistaken for a missing key");
assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
let requests = fixture.await.expect("candidate fixture should join");
for request in requests {
let request = request.to_ascii_lowercase();
assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET");
assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
assert!(!request.contains("?versioning"));
assert!(!request.contains("?versions"));
}
}
fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
ListVersionsResult {
versions: versions
@@ -631,8 +866,7 @@ impl WarmBackend for WarmBackendS3 {
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
let bucket_versioning = self.remote_bucket_versioning().await?;
self.probe_transition_candidate_versions(object, bucket_versioning).await
self.probe_current_transition_candidate_with_header(object, None).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
crates/ecstore/src/services/tier/warm_backend_tencent.rs
+5 -1
View File
@@ -23,7 +23,7 @@ use std::collections::HashMap;
use crate::services::tier::{
tier_config::TierTencent,
warm_backend::{
S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
new_s3_compatible_warm_backend, optimal_part_size,
},
warm_backend_s3::WarmBackendS3,
@@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendTencent {
self.0.remove(object, rv).await
}
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
self.0.probe_transition_candidate(object).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
self.0.in_use().await
}
crates/ecstore/src/services/tier/warm_backend_wasabi.rs
+5 -1
View File
@@ -23,7 +23,7 @@ use uuid::Uuid;
use crate::services::tier::{
tier_config::{TierS3, TierWasabi},
warm_backend::{WarmBackend, WarmBackendGetOpts},
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts},
warm_backend_s3::WarmBackendS3,
};
use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl};
@@ -169,6 +169,10 @@ impl WarmBackend for WarmBackendWasabi {
self.s3.remove(object, rv).await
}
async fn probe_transition_candidate(&self, object: &str) -> io::Result<TransitionCandidateProbe> {
self.s3.probe_transition_candidate(object).await
}
async fn in_use(&self) -> io::Result<bool> {
self.check_remote_bucket_unversioned().await?;
let in_use = self.s3.in_use().await?;
crates/ecstore/src/set_disk/ops/object.rs
+2
View File
@@ -7935,6 +7935,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
delete_marker_mtime: vr.mod_time.or(goi.mod_time),
object_name: vr.name.clone(),
replication_state: vr.replication_state_internal.clone(),
source_missing,
..Default::default()
}
} else {
@@ -7946,6 +7947,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
vr.version_id
},
replication_state: vr.replication_state_internal.clone(),
source_missing,
..Default::default()
};
accounting[i] = Some(DeleteAccounting {
crates/ecstore/src/storage_api_contracts/mod.rs
+3 -2
View File
@@ -32,8 +32,9 @@ pub(crate) mod internode {
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY,
PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse,
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
WALK_DIR_STREAM_COMPLETION_V1,
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
};
}
crates/ecstore/src/store/init.rs
+943 -7
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/store/object.rs
+392 -87
View File
@@ -18,7 +18,9 @@ use crate::bucket::lifecycle::{
get_expiry_configs,
tier_delete_journal::{
ActiveTierDeleteDispatch, EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_LIFECYCLE,
complete_tier_delete_dispatch, prepare_tier_delete_dispatch, record_tier_delete_journal_backend_identity,
TierDeleteChunkParentInspection, complete_tier_delete_chunk_parent, complete_tier_delete_dispatch,
inspect_tier_delete_chunk_parent, prepare_tier_delete_chunk_dispatch, prepare_tier_delete_dispatch,
record_tier_delete_journal_backend_identity, resume_tier_delete_dispatch, tier_delete_dispatch_batch_limit,
tier_delete_journal_object_name, tier_delete_source_matches_dispatch_scope,
},
tier_sweeper::{
@@ -41,7 +43,10 @@ use crate::object_api::{
NamespaceLockFence, ObjectLockConfigSnapshot, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState,
TierFreeVersionReceiptSink,
};
use crate::services::notification_sys::acquire_tier_delete_journal_fleet_proof;
use crate::services::notification_sys::{
TierDeleteJournalFleetProofToken, acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches,
tier_delete_journal_topology_generation,
};
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
use crate::set_disk::{
SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
@@ -52,6 +57,7 @@ use crate::storage_api_contracts::{
namespace::NamespaceLocking as _,
object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _},
};
use futures::StreamExt as _;
use parking_lot::Mutex as ParkingMutex;
use rustfs_filemeta::ObjectPartInfo;
use rustfs_io_metrics::{
@@ -73,6 +79,7 @@ const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000;
const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2;
const RESTORE_WORKER_LOCK_PREFIX: &str = "ilm/restore-worker-locks";
const RESTORE_WORKER_LOCK_PROBE_TIMEOUT: Duration = Duration::from_millis(50);
const TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY: usize = 16;
fn install_tier_free_version_receipt_sink(opts: &mut ObjectOptions) -> Option<TierFreeVersionReceiptSink> {
if opts.tier_free_version_receipt_sink.is_some() || opts.skip_free_version || opts.delete_prefix {
@@ -147,14 +154,96 @@ async fn prepare_prefix_tier_delete_journal_entries(
Box::pin(prepare_prefix_tier_delete_journal_entries_inner(api, bucket, prefix, opts)).await
}
type TierDeleteLeaseReference = (String, Option<TierDestinationId>);
fn tier_delete_walk_cancellation_is_expected(truncated: bool, limit_cancellation: bool, error: &Error) -> bool {
truncated && limit_cancellation && matches!(error, Error::OperationCanceled)
}
fn combine_tier_delete_walk_results(results: impl IntoIterator<Item = Result<()>>) -> Result<()> {
let mut cancelled = false;
for result in results {
match result {
Ok(()) => {}
Err(Error::OperationCanceled) => cancelled = true,
Err(err) => return Err(err),
}
}
if cancelled { Err(Error::OperationCanceled) } else { Ok(()) }
}
async fn acquire_prefix_tier_delete_reference_leases(
api: &Arc<ECStore>,
tier_references: &std::collections::HashSet<TierDeleteLeaseReference>,
) -> Result<Vec<TierOperationLease>> {
let mut tier_references = tier_references.iter().cloned().collect::<Vec<_>>();
tier_references.sort_unstable();
let mut leases = Vec::with_capacity(tier_references.len());
for (tier_name, backend_identity) in tier_references {
let lease = match backend_identity {
Some(backend_identity) => {
TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity)
.await
}
None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await,
}
.map_err(Error::other)?;
leases.push(lease);
}
Ok(leases)
}
async fn acquire_prefix_tier_delete_leases(api: &Arc<ECStore>, entries: &[Jentry]) -> Result<Vec<TierOperationLease>> {
let tier_references = entries
.iter()
.map(|entry| (entry.tier_name.clone(), entry.backend_identity))
.collect::<std::collections::HashSet<_>>();
acquire_prefix_tier_delete_reference_leases(api, &tier_references).await
}
async fn prepare_prefix_tier_delete_journal_entries_inner(
api: &Arc<ECStore>,
bucket: &str,
prefix: &str,
opts: &ObjectOptions,
) -> Result<PreparedPrefixTierDelete> {
let (chunk_parent_active, legacy_manifest_active, chunk_parent_topology_generation) = if is_meta_bucketname(bucket) {
(false, false, None)
} else {
let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
let bucket_fence = opts
.bucket_lifecycle_lock_fence
.as_ref()
.ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
match Box::pin(inspect_tier_delete_chunk_parent(
Arc::clone(api),
bucket,
bucket_incarnation,
prefix,
bucket_fence,
))
.await?
{
TierDeleteChunkParentInspection::NoParent => (false, false, None),
TierDeleteChunkParentInspection::LegacyManifest => (false, true, None),
TierDeleteChunkParentInspection::Ready(topology_generation) => (true, false, Some(topology_generation)),
TierDeleteChunkParentInspection::Resume(dispatch) => {
let leases = acquire_prefix_tier_delete_leases(api, dispatch.entries()?).await?;
return Ok(PreparedPrefixTierDelete {
dispatch: Some(*dispatch),
chunk_parent_active: true,
chunk_parent_fleet_proof: None,
_leases: leases,
});
}
TierDeleteChunkParentInspection::RetryRequired => {
return Err(Error::other("tier delete chunk parent made durable progress; retry the next batch"));
}
}
};
let mut tier_references = std::collections::HashSet::<(String, Option<TierDestinationId>)>::new();
let mut entries_by_name = std::collections::BTreeMap::new();
let batch_limit = tier_delete_dispatch_batch_limit();
let logical_prefix = decode_dir_object(prefix);
let exact_object = opts.delete_prefix_object.then(|| logical_prefix.clone());
let physical_sets = api
@@ -164,12 +253,11 @@ async fn prepare_prefix_tier_delete_journal_entries_inner(
.collect::<Vec<_>>();
let (tx, mut rx) = tokio::sync::mpsc::channel::<ObjectInfoOrErr>(100);
let cancellation = tokio_util::sync::CancellationToken::new();
let limit_cancellation = Arc::new(AtomicBool::new(false));
let walk_cancel = cancellation.clone();
let bucket_owned = bucket.to_string();
let prefix_owned = prefix.to_string();
let walk = async move {
use futures::StreamExt as _;
let results = futures::stream::iter(physical_sets.into_iter().map(|set| {
let tx = tx.clone();
let cancellation = walk_cancel.clone();
@@ -199,14 +287,28 @@ async fn prepare_prefix_tier_delete_journal_entries_inner(
.collect::<Vec<_>>()
.await;
drop(tx);
results.into_iter().collect::<Result<Vec<_>>>().map(|_| ())
combine_tier_delete_walk_results(results)
};
let collect_limit_cancellation = limit_cancellation.clone();
let collect = async {
let mut truncated = false;
while let Some(result) = rx.recv().await {
if let Some(err) = result.err {
// Once limit + 1 has been observed this request can authorize
// only the exact retained batch; it cannot infer prefix
// absence or run the raw delete. Drain only the explicit
// cancellation fallout; a real walker error must still fail
// the request even when another set reached the limit first.
if tier_delete_walk_cancellation_is_expected(truncated, collect_limit_cancellation.load(Ordering::Acquire), &err)
{
continue;
}
cancellation.cancel();
return Err(err);
}
if truncated {
continue;
}
let Some(source) = result.item else {
continue;
};
@@ -223,62 +325,125 @@ async fn prepare_prefix_tier_delete_journal_entries_inner(
"recursive prefix delete cannot discard an existing tier free-version cleanup obligation",
));
}
if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE {
let tier_reference = if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE {
let backend_identity = tier_destination_id_from_metadata(&source.user_defined).map_err(Error::other)?;
tier_references.insert((source.transitioned_object.tier.clone(), backend_identity));
}
Some((source.transitioned_object.tier.clone(), backend_identity))
} else {
None
};
if let Some(entry) = build_tier_delete_journal_entry(bucket, &object, opts, &source)? {
entries_by_name
.entry(tier_delete_journal_object_name(&entry))
.or_insert(entry);
let name = tier_delete_journal_object_name(&entry);
let at_limit = entries_by_name.len() == batch_limit;
match entries_by_name.entry(name) {
std::collections::btree_map::Entry::Occupied(_) => {}
std::collections::btree_map::Entry::Vacant(_) if at_limit => {
truncated = true;
collect_limit_cancellation.store(true, Ordering::Release);
cancellation.cancel();
}
std::collections::btree_map::Entry::Vacant(slot) => {
if let Some(tier_reference) = tier_reference {
tier_references.insert(tier_reference);
}
slot.insert(entry);
}
}
} else if let Some(tier_reference) = tier_reference {
tier_references.insert(tier_reference);
}
}
Ok(())
Ok(truncated)
};
let (walk_result, collect_result) = tokio::join!(walk, collect);
collect_result?;
walk_result?;
let entries = entries_by_name.into_values().collect::<Vec<_>>();
let mut tier_references = tier_references.into_iter().collect::<Vec<_>>();
tier_references.sort_unstable();
let mut leases = Vec::with_capacity(tier_references.len());
for (tier_name, backend_identity) in tier_references {
let lease = match backend_identity {
Some(backend_identity) => {
TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity)
.await
}
None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await,
}
.map_err(Error::other)?;
leases.push(lease);
let truncated = collect_result?;
// A truncated walk normally reports OperationCanceled from the physical
// walkers. That cancellation is expected; any other result is a genuine
// scan failure and cannot be hidden by the bounded batch.
if let Err(err) = walk_result
&& !tier_delete_walk_cancellation_is_expected(truncated, limit_cancellation.load(Ordering::Acquire), &err)
{
return Err(err);
}
let entries = entries_by_name.into_values().collect::<Vec<_>>();
let mut leased_tier_references = tier_references;
let mut leases = acquire_prefix_tier_delete_reference_leases(api, &leased_tier_references).await?;
if entries.is_empty() {
let chunk_parent_fleet_proof = if let Some(expected_topology) = chunk_parent_topology_generation.as_deref() {
let fleet_proof = acquire_tier_delete_journal_fleet_proof()
.ok_or_else(|| Error::other("tier delete chunk parent fleet capability is unavailable"))?;
if tier_delete_journal_topology_generation(&fleet_proof) != expected_topology {
return Err(Error::other("tier delete chunk parent topology changed during final source scan"));
}
Some(fleet_proof)
} else {
None
};
return Ok(PreparedPrefixTierDelete {
dispatch: None,
chunk_parent_active,
chunk_parent_fleet_proof,
_leases: leases,
});
}
let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
let fleet_proof = acquire_tier_delete_journal_fleet_proof()
.ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?;
let bucket_fence = opts
.bucket_lifecycle_lock_fence
.as_ref()
.ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
let dispatch =
let fleet_proof = acquire_tier_delete_journal_fleet_proof()
.ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?;
if chunk_parent_topology_generation
.as_deref()
.is_some_and(|expected| tier_delete_journal_topology_generation(&fleet_proof) != expected)
{
return Err(Error::other("tier delete chunk parent topology changed during source scan"));
}
let mut dispatch = if !legacy_manifest_active && (chunk_parent_active || truncated) {
Box::pin(prepare_tier_delete_chunk_dispatch(
Arc::clone(api),
bucket,
bucket_incarnation,
prefix,
entries,
truncated && !chunk_parent_active,
fleet_proof,
bucket_fence,
))
.await?
} else if legacy_manifest_active && truncated {
resume_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence)
.await?
} else {
prepare_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence)
.await?;
.await?
};
// A resumed legacy authorization may own predecessors that are absent
// from this bounded scan. Pin every backend generation in the actual
// permit before any local mutation, while avoiding duplicate leases for
// entries already covered by the scan.
let additional_tier_references = dispatch
.entries()?
.iter()
.map(|entry| (entry.tier_name.clone(), entry.backend_identity))
.filter(|reference| leased_tier_references.insert(reference.clone()))
.collect::<std::collections::HashSet<_>>();
leases.extend(acquire_prefix_tier_delete_reference_leases(api, &additional_tier_references).await?);
if legacy_manifest_active && truncated {
dispatch.require_exact_predecessor_replay();
}
Ok(PreparedPrefixTierDelete {
dispatch: Some(dispatch),
chunk_parent_active: chunk_parent_active || truncated,
chunk_parent_fleet_proof: None,
_leases: leases,
})
}
struct PreparedPrefixTierDelete {
dispatch: Option<crate::bucket::lifecycle::tier_delete_journal::PreparedTierDeleteDispatch>,
chunk_parent_active: bool,
chunk_parent_fleet_proof: Option<TierDeleteJournalFleetProofToken>,
_leases: Vec<TierOperationLease>,
}
@@ -365,12 +530,16 @@ async fn delete_prefix_with_tier_delete_journal(
let Some(api) = tier_journal_api else {
return store.delete_prefix(bucket, object, opts).await;
};
let PreparedPrefixTierDelete { dispatch, _leases } =
prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?;
let PreparedPrefixTierDelete {
dispatch,
chunk_parent_active,
chunk_parent_fleet_proof,
_leases,
} = prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?;
let Some(dispatch) = dispatch else {
// There is no remote-cleanup candidate, so no v6 manifest or fleet
// proof is required. Keep any compatibility-path tier leases alive
// until the local delete has committed.
// There is no new remote-cleanup candidate. Keep compatibility-path
// tier leases and, for a chunked final pass, the matching parent fleet
// proof alive until local deletion and parent completion both commit.
let _tier_leases = _leases;
let mut operation_opts = opts.clone();
// `tier_delete_journal_api` means a v6 dispatch authorization must be
@@ -379,7 +548,42 @@ async fn delete_prefix_with_tier_delete_journal(
// transitioned metadata retains its FreeVersion fallback.
operation_opts.tier_delete_journal_api = None;
operation_opts.tier_delete_dispatch_authorization = None;
return store.delete_prefix(bucket, object, &operation_opts).await;
let parent_fleet_proof = if chunk_parent_active {
Some(
chunk_parent_fleet_proof
.as_ref()
.filter(|proof| tier_delete_journal_fleet_proof_matches(proof))
.ok_or_else(|| Error::other("tier delete chunk parent fleet proof changed before final deletion"))?,
)
} else {
None
};
store.delete_prefix(bucket, object, &operation_opts).await?;
if let Some(parent_fleet_proof) = parent_fleet_proof {
#[cfg(all(test, feature = "test-util"))]
crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause(
crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::FinalLocalDeletionCompleted,
)
.await;
let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
let bucket_fence = opts
.bucket_lifecycle_lock_fence
.as_ref()
.ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
if !Box::pin(complete_tier_delete_chunk_parent(
Arc::clone(api),
bucket,
bucket_incarnation,
object,
bucket_fence,
parent_fleet_proof,
))
.await?
{
return Err(Error::other("tier delete chunk parent disappeared after final local deletion"));
}
}
return Ok(());
};
let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
let bucket_fence = opts
@@ -396,9 +600,19 @@ async fn delete_prefix_with_tier_delete_journal(
// Keep every backend generation lease until the whole local operation has
// either committed its journal set or returned an ambiguous mutation.
let _tier_leases = _leases;
if active.predecessor_replay_required() {
if active.predecessor_replay_required() || active.is_chunked() {
replay_authorized_tier_delete_sources(store, bucket, object, &active, &operation_opts).await?;
#[cfg(all(test, feature = "test-util"))]
if active.is_chunked() {
crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause(
crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::LocalReplayCompleted,
)
.await;
}
complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await?;
if active.is_chunked() {
return Err(Error::other("tier delete chunk completed; retry the next durable batch"));
}
return Err(Error::other("authorized tier delete predecessor completed; retry the successor dispatch"));
}
let result = store.delete_prefix(bucket, object, &operation_opts).await;
@@ -443,7 +657,7 @@ async fn replay_authorized_tier_delete_sources(
let authorization = active.authorization();
authorization.mark_mutation_started(bucket, bucket_incarnation, prefix)?;
let mut source_objects = std::collections::BTreeSet::new();
let mut source_objects = std::collections::HashSet::with_capacity(active.entries().len());
for entry in active.entries() {
let source = entry
.source
@@ -453,61 +667,130 @@ async fn replay_authorized_tier_delete_sources(
if !tier_delete_source_matches_replay_scope(source, bucket, prefix, opts.delete_prefix_object) {
return Err(Error::other("authorized tier delete predecessor source escaped its prefix scope"));
}
source_objects.insert(source.object.clone());
source_objects.insert(source.object.as_str());
}
let mut deleted = 0;
for object in source_objects {
if bucket_fence.is_lock_lost() {
return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
if let Some(scope) = publication_scope {
if scope.state() == ScannerPublicationCommitState::Admitted {
scope
.try_begin()
.map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?;
}
let encoded_object = encode_dir_object(&object);
let guards = if opts.delete_prefix_object {
store
.acquire_remaining_physical_object_write_locks("tier_delete_dispatch_predecessor_replay", bucket, &encoded_object)
.await?
} else {
store
.acquire_all_physical_object_write_locks("tier_delete_dispatch_predecessor_replay", bucket, &encoded_object)
.await?
};
authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
if let Some(scope) = publication_scope {
if scope.state() == ScannerPublicationCommitState::Admitted {
scope
.try_begin()
.map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?;
}
if !scope.can_commit() {
let _ = scope.mark_indeterminate();
return Err(StorageError::OperationCanceled);
}
}
let mut replay_opts = opts.clone();
replay_opts.no_lock = true;
replay_opts.delete_prefix = false;
replay_opts.delete_prefix_object = false;
for guard in &guards {
guard.add_namespace_lock_fence(&mut replay_opts);
}
for pool in &store.pools {
for set in &pool.disk_set {
authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
deleted += set
.replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts)
.await?;
}
}
if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
if !scope.can_commit() {
let _ = scope.mark_indeterminate();
return Err(StorageError::OperationCanceled);
}
}
if let Some(scope) = publication_scope {
let _ = scope.mark_committed();
let stopped = Arc::new(AtomicBool::new(false));
// The caller holds bucket lifecycle WRITE. Each bounded future acquires
// only one logical object's physical lock set and releases it before
// completion; no future nests locks for two object keys.
let make_replay = |object: String| {
let stopped = stopped.clone();
let authorization = authorization.clone();
async move {
if stopped.load(Ordering::Acquire) {
return Ok::<_, Error>(0usize);
}
let result = async {
if bucket_fence.is_lock_lost() {
return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
}
let encoded_object = encode_dir_object(&object);
let guards = if opts.delete_prefix_object {
store
.acquire_remaining_physical_object_write_locks(
"tier_delete_dispatch_predecessor_replay",
bucket,
&encoded_object,
)
.await?
} else {
store
.acquire_all_physical_object_write_locks(
"tier_delete_dispatch_predecessor_replay",
bucket,
&encoded_object,
)
.await?
};
authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
if publication_scope.is_some_and(|scope| !scope.can_commit()) {
return Err(StorageError::OperationCanceled);
}
let mut replay_opts = opts.clone();
replay_opts.no_lock = true;
replay_opts.delete_prefix = false;
replay_opts.delete_prefix_object = false;
for guard in &guards {
guard.add_namespace_lock_fence(&mut replay_opts);
}
let mut deleted = 0usize;
for pool in &store.pools {
for set in &pool.disk_set {
authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
deleted = deleted
.checked_add(
set.replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts)
.await?,
)
.ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?;
}
}
if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
}
Ok(deleted)
}
.await;
if result.is_err() {
stopped.store(true, Ordering::Release);
}
result
}
};
let mut objects = source_objects.into_iter();
let mut replays = futures::stream::FuturesUnordered::new();
for _ in 0..TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY {
let Some(object) = objects.next().map(ToOwned::to_owned) else {
break;
};
replays.push(make_replay(object));
}
let mut deleted = 0usize;
let mut first_error = None;
while let Some(result) = replays.next().await {
match result {
Ok(count) => {
deleted = deleted
.checked_add(count)
.ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?;
}
Err(err) if first_error.is_none() => first_error = Some(err),
Err(_) => {}
}
if first_error.is_none()
&& !stopped.load(Ordering::Acquire)
&& let Some(object) = objects.next().map(ToOwned::to_owned)
{
replays.push(make_replay(object));
}
}
if deleted > 0 {
super::list_objects::observe_list_objects_mutation(store, bucket).await;
}
if let Some(err) = first_error {
if publication_scope.is_some_and(|scope| !scope.can_commit())
&& let Some(scope) = publication_scope
{
let _ = scope.mark_indeterminate();
}
return Err(err);
}
if let Some(scope) = publication_scope {
let _ = scope.mark_committed();
}
Ok(())
}
@@ -6518,6 +6801,28 @@ mod tests {
);
}
#[test]
fn tier_delete_walk_only_accepts_explicit_limit_cancellation() {
let cancelled = Error::OperationCanceled;
assert!(tier_delete_walk_cancellation_is_expected(true, true, &cancelled));
assert!(!tier_delete_walk_cancellation_is_expected(false, true, &cancelled));
assert!(!tier_delete_walk_cancellation_is_expected(true, false, &cancelled));
assert!(!tier_delete_walk_cancellation_is_expected(true, true, &Error::other("scan failed")));
}
#[test]
fn tier_delete_walk_results_prioritize_real_errors_over_cancellation() {
let err = combine_tier_delete_walk_results([Err(Error::OperationCanceled), Ok(()), Err(StorageError::FileAccessDenied)])
.expect_err("a real walk error must not be hidden by earlier cancellation");
assert_eq!(err, StorageError::FileAccessDenied);
assert_eq!(
combine_tier_delete_walk_results([Ok(()), Err(Error::OperationCanceled)])
.expect_err("cancellation must remain visible when there is no real error"),
Error::OperationCanceled
);
}
impl Drop for BodyCacheHookGuard {
fn drop(&mut self) {
clear_get_object_body_cache_hook();
crates/filemeta/src/fileinfo.rs
+94 -3
View File
@@ -28,7 +28,7 @@ use serde::de::{self, MapAccess, SeqAccess, Visitor, value::MapAccessDeserialize
use serde::ser::SerializeMap;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use time::{OffsetDateTime, PrimitiveDateTime, format_description::well_known::Rfc3339};
use time::{format_description::FormatItem, macros::format_description};
use uuid::Uuid;
@@ -1417,6 +1417,31 @@ impl RestoreStatusOps for RestoreStatus {
}
}
/// Parse the `expiry-date` value carried by a persisted `x-amz-restore` header.
///
/// RustFS has always serialized this field as RFC3339, while MinIO (and any
/// object migrated in place from a MinIO deployment) writes Go's
/// `http.TimeFormat` — RFC1123 with a literal `GMT` zone. Accept both shapes
/// so a migrated "restored" object stays readable.
///
/// This is not only a HEAD 500: `is_restored_object_on_disk` fails open when
/// the header cannot be parsed, and `MetaObject::uses_data_dir` falls back to
/// it, so an unparseable restore header makes a live restored data dir look
/// unused (backlog#1342).
///
/// The write side deliberately stays on RFC3339 for now: an older node only
/// parses RFC3339, so switching the persisted format before this parser is
/// deployed everywhere would break rolling upgrades and rollbacks in the
/// opposite direction.
fn parse_restore_expiry_date(value: &str) -> Result<OffsetDateTime> {
if let Ok(expiry) = OffsetDateTime::parse(value, &Rfc3339) {
return Ok(expiry);
}
PrimitiveDateTime::parse(value, &RFC1123)
.map(PrimitiveDateTime::assume_utc)
.map_err(|_| Error::other(ERR_RESTORE_HDR_MALFORMED))
}
pub fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
let tokens: Vec<&str> = restore_hdr.splitn(2, ",").collect();
let progress_tokens: Vec<&str> = tokens[0].splitn(2, "=").collect();
@@ -1445,8 +1470,7 @@ pub fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
if expiry_tokens[0].trim() != "expiry-date" {
return Err(Error::other(ERR_RESTORE_HDR_MALFORMED));
}
let expiry = OffsetDateTime::parse(expiry_tokens[1].trim_matches('"'), &Rfc3339)
.map_err(|_| Error::other(ERR_RESTORE_HDR_MALFORMED))?;
let expiry = parse_restore_expiry_date(expiry_tokens[1].trim_matches('"'))?;
return Ok(RestoreStatus {
is_restore_in_progress: Some(false),
restore_expiry_date: Some(Timestamp::from(expiry)),
@@ -1471,6 +1495,7 @@ mod tests {
use super::*;
use proptest::collection::{hash_map, vec};
use proptest::prelude::*;
use time::macros::datetime;
// backlog#959 / ECA-18: the interleaved per-block bitrot subsystem in
// rustfs-ecstore (BitrotWriter / bitrot_verify / bitrot_shard_file_size) is
@@ -2685,4 +2710,70 @@ mod tests {
let empty = FileInfo::default();
assert!(format!("{empty:?}").contains("data: None"));
}
/// backlog#1342: MinIO writes the persisted `x-amz-restore` completion
/// header with Go's `http.TimeFormat` (RFC1123/GMT). Before this parser
/// accepted that shape, every object migrated in place from MinIO failed
/// `parse_restore_obj_status`.
#[test]
fn parses_minio_rfc1123_restore_expiry() {
let status = parse_restore_obj_status("ongoing-request=\"false\", expiry-date=\"Wed, 01 Jan 2025 10:20:30 GMT\"")
.expect("RFC1123 expiry-date must parse");
assert_eq!(status.is_restore_in_progress, Some(false));
let expiry = status.expiry().expect("completed restore carries an expiry");
assert_eq!(expiry, datetime!(2025-01-01 10:20:30 UTC));
}
/// The RustFS-written shape must keep parsing unchanged: the persisted
/// format stays RFC3339 until every node can read both.
#[test]
fn parses_rustfs_rfc3339_restore_expiry() {
let status = parse_restore_obj_status("ongoing-request=\"false\", expiry-date=\"2025-01-01T10:20:30Z\"")
.expect("RFC3339 expiry-date must parse");
assert_eq!(status.expiry(), Some(datetime!(2025-01-01 10:20:30 UTC)));
}
/// Round-trip both directions of the migration: what RustFS emits on the
/// wire (`to_string2`, RFC1123) and what it persists (`to_string`,
/// RFC3339) must both be readable by this parser.
#[test]
fn restore_status_round_trips_through_both_formats() {
let status = RestoreStatus {
is_restore_in_progress: Some(false),
restore_expiry_date: Some(Timestamp::from(datetime!(2030-06-15 07:08:09 UTC))),
};
for rendered in [RestoreStatusOps::to_string(&status), status.to_string2()] {
let parsed = parse_restore_obj_status(&rendered).unwrap_or_else(|e| panic!("{rendered} must parse: {e}"));
assert_eq!(parsed.expiry(), Some(datetime!(2030-06-15 07:08:09 UTC)), "{rendered}");
}
}
/// A restored object migrated from MinIO must still be recognised as
/// on-disk: `is_restored_object_on_disk` fails open, and
/// `MetaObject::uses_data_dir` uses it to decide whether a data dir is
/// live, so a parse failure here can make a live data dir look reclaimable.
#[test]
fn minio_restored_object_is_recognised_as_on_disk() {
let mut meta = HashMap::new();
meta.insert(
X_AMZ_RESTORE.as_str().to_string(),
"ongoing-request=\"false\", expiry-date=\"Fri, 01 Jan 9999 00:00:00 GMT\"".to_string(),
);
assert!(is_restored_object_on_disk(&meta));
}
/// Widening the accepted formats must not weaken malformed-input
/// rejection: an unparseable or non-GMT-shaped expiry still fails closed.
#[test]
fn rejects_malformed_restore_expiry() {
for header in [
"ongoing-request=\"false\", expiry-date=\"not-a-date\"",
"ongoing-request=\"false\", expiry-date=\"Wed, 01 Jan 2025 10:20:30\"",
"ongoing-request=\"false\", expiry-date=\"01 Jan 2025 10:20:30 GMT\"",
"ongoing-request=\"false\"",
"ongoing-request=\"false\", expires=\"2025-01-01T10:20:30Z\"",
] {
assert!(parse_restore_obj_status(header).is_err(), "{header} must be rejected");
}
}
}
crates/protos/src/generated/proto_gen/node_service.rs
+79
View File
@@ -1251,6 +1251,38 @@ pub struct ScannerActivityResponse {
#[prost(bool, optional, tag = "11")]
pub publication_blocked: ::core::option::Option<bool>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerDirtyUsageBucket {
#[prost(string, tag = "1")]
pub bucket: ::prost::alloc::string::String,
#[prost(uint64, tag = "2")]
pub generation: u64,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerDirtyUsageSnapshotRequest {
#[prost(bytes = "bytes", tag = "1")]
pub challenge: ::prost::bytes::Bytes,
#[prost(uint32, tag = "2")]
pub protocol_version: u32,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ScannerDirtyUsageSnapshotResponse {
#[prost(string, tag = "1")]
pub instance_id: ::prost::alloc::string::String,
#[prost(uint64, tag = "2")]
pub generation: u64,
#[prost(uint64, tag = "3")]
pub pending_bucket_count: u64,
#[prost(uint32, tag = "4")]
pub protocol_version: u32,
/// Incomplete snapshots are all-or-nothing and carry no bucket entries.
#[prost(bool, tag = "5")]
pub complete: bool,
#[prost(message, repeated, tag = "6")]
pub buckets: ::prost::alloc::vec::Vec<ScannerDirtyUsageBucket>,
#[prost(bytes = "bytes", tag = "7")]
pub response_proof: ::prost::bytes::Bytes,
}
/// A short-lived storage-owned read admission used only around a final
/// scanner metadata publication. It is intentionally separate from the
/// ScannerActivity observation wire so v6/v7 rolling compatibility remains
@@ -2899,6 +2931,21 @@ pub mod node_service_client {
.insert(GrpcMethod::new("node_service.NodeService", "ScannerActivity"));
self.inner.unary(req, path, codec).await
}
pub async fn scanner_dirty_usage_snapshot(
&mut self,
request: impl tonic::IntoRequest<super::ScannerDirtyUsageSnapshotRequest>,
) -> std::result::Result<tonic::Response<super::ScannerDirtyUsageSnapshotResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ScannerDirtyUsageSnapshot");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.NodeService", "ScannerDirtyUsageSnapshot"));
self.inner.unary(req, path, codec).await
}
pub async fn acquire_scanner_publication_lease(
&mut self,
request: impl tonic::IntoRequest<super::ScannerPublicationLeaseRequest>,
@@ -3457,6 +3504,10 @@ pub mod node_service_server {
&self,
request: tonic::Request<super::ScannerActivityRequest>,
) -> std::result::Result<tonic::Response<super::ScannerActivityResponse>, tonic::Status>;
async fn scanner_dirty_usage_snapshot(
&self,
request: tonic::Request<super::ScannerDirtyUsageSnapshotRequest>,
) -> std::result::Result<tonic::Response<super::ScannerDirtyUsageSnapshotResponse>, tonic::Status>;
async fn acquire_scanner_publication_lease(
&self,
request: tonic::Request<super::ScannerPublicationLeaseRequest>,
@@ -5749,6 +5800,34 @@ pub mod node_service_server {
};
Box::pin(fut)
}
"/node_service.NodeService/ScannerDirtyUsageSnapshot" => {
#[allow(non_camel_case_types)]
struct ScannerDirtyUsageSnapshotSvc<T: NodeService>(pub Arc<T>);
impl<T: NodeService> tonic::server::UnaryService<super::ScannerDirtyUsageSnapshotRequest> for ScannerDirtyUsageSnapshotSvc<T> {
type Response = super::ScannerDirtyUsageSnapshotResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::ScannerDirtyUsageSnapshotRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { <T as NodeService>::scanner_dirty_usage_snapshot(&inner, request).await };
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = ScannerDirtyUsageSnapshotSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/node_service.NodeService/AcquireScannerPublicationLease" => {
#[allow(non_camel_case_types)]
struct AcquireScannerPublicationLeaseSvc<T: NodeService>(pub Arc<T>);
crates/protos/src/lib.rs
+128 -1
View File
@@ -541,6 +541,34 @@ pub fn canonical_scanner_activity_v7_response_body(
Ok(body)
}
pub fn canonical_scanner_dirty_usage_snapshot_request_body(
request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-scanner-dirty-usage-snapshot-request-v1\0");
body.push_u32(request.protocol_version);
body.push_bytes(request.challenge.as_ref())?;
Ok(body.finish())
}
pub fn canonical_scanner_dirty_usage_snapshot_response_body(
challenge: &[u8],
response: &proto_gen::node_service::ScannerDirtyUsageSnapshotResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-scanner-dirty-usage-snapshot-response-v1\0");
body.push_bytes(challenge)?;
body.push_str(&response.instance_id)?;
body.push_u64(response.generation);
body.push_u64(response.pending_bucket_count);
body.push_u32(response.protocol_version);
body.push_bool(response.complete);
body.push_count(response.buckets.len())?;
for bucket in &response.buckets {
body.push_str(&bucket.bucket)?;
body.push_u64(bucket.generation);
}
Ok(body.finish())
}
/// Builds the body authenticated by the short-lived remote scanner publication
/// lease request. This is a separate domain from ScannerActivity so v6/v7
/// observation proofs remain byte-for-byte compatible.
@@ -1751,13 +1779,112 @@ mod scanner_activity_tests {
use super::{
canonical_scanner_activity_request_body, canonical_scanner_activity_response_body,
canonical_scanner_activity_v4_response_body, canonical_scanner_activity_v7_response_body,
canonical_scanner_dirty_usage_snapshot_request_body, canonical_scanner_dirty_usage_snapshot_response_body,
canonical_scanner_publication_lease_release_request_body, canonical_scanner_publication_lease_request_body,
canonical_scanner_publication_lease_response_body,
proto_gen::node_service::{
ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshotRequest,
ScannerDirtyUsageSnapshotResponse, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
},
};
#[test]
fn canonical_scanner_dirty_usage_snapshot_request_binds_every_field() {
let request = ScannerDirtyUsageSnapshotRequest {
challenge: vec![1; 16].into(),
protocol_version: 1,
};
let baseline = canonical_scanner_dirty_usage_snapshot_request_body(&request)
.expect("scanner dirty usage snapshot request should encode");
for variant in [
ScannerDirtyUsageSnapshotRequest {
challenge: vec![2; 16].into(),
..request.clone()
},
ScannerDirtyUsageSnapshotRequest {
protocol_version: 2,
..request
},
] {
assert_ne!(
baseline,
canonical_scanner_dirty_usage_snapshot_request_body(&variant)
.expect("scanner dirty usage snapshot request variant should encode")
);
}
}
#[test]
fn canonical_scanner_dirty_usage_snapshot_response_binds_every_field() {
let response = ScannerDirtyUsageSnapshotResponse {
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
generation: 7,
pending_bucket_count: 2,
protocol_version: 1,
complete: true,
buckets: vec![
ScannerDirtyUsageBucket {
bucket: "archive".to_string(),
generation: 3,
},
ScannerDirtyUsageBucket {
bucket: "photos".to_string(),
generation: 7,
},
],
response_proof: vec![9; 32].into(),
};
let baseline = canonical_scanner_dirty_usage_snapshot_response_body(&[1; 16], &response)
.expect("scanner dirty usage snapshot response should encode");
let mut variants = Vec::new();
let mut instance = response.clone();
instance.instance_id = "fedcba9876543210fedcba9876543210".to_string();
variants.push(instance);
let mut generation = response.clone();
generation.generation = 8;
variants.push(generation);
let mut count = response.clone();
count.pending_bucket_count = 3;
variants.push(count);
let mut protocol = response.clone();
protocol.protocol_version = 2;
variants.push(protocol);
let mut complete = response.clone();
complete.complete = false;
variants.push(complete);
let mut bucket_name = response.clone();
bucket_name.buckets[0].bucket = "backups".to_string();
variants.push(bucket_name);
let mut bucket_generation = response.clone();
bucket_generation.buckets[0].generation = 4;
variants.push(bucket_generation);
let mut bucket_order = response.clone();
bucket_order.buckets.reverse();
variants.push(bucket_order);
for variant in variants {
assert_ne!(
baseline,
canonical_scanner_dirty_usage_snapshot_response_body(&[1; 16], &variant)
.expect("scanner dirty usage snapshot response variant should encode")
);
}
assert_ne!(
baseline,
canonical_scanner_dirty_usage_snapshot_response_body(&[2; 16], &response)
.expect("scanner dirty usage snapshot response challenge variant should encode")
);
let mut proof_only = response;
proof_only.response_proof = vec![8; 32].into();
assert_eq!(
baseline,
canonical_scanner_dirty_usage_snapshot_response_body(&[1; 16], &proof_only)
.expect("response proof must not authenticate itself")
);
}
#[test]
fn canonical_scanner_activity_request_binds_every_field() {
let request = ScannerActivityRequest {
crates/protos/src/node.proto
+22
View File
@@ -882,6 +882,27 @@ message ScannerActivityResponse {
optional bool publication_blocked = 11;
}
message ScannerDirtyUsageBucket {
string bucket = 1;
uint64 generation = 2;
}
message ScannerDirtyUsageSnapshotRequest {
bytes challenge = 1;
uint32 protocol_version = 2;
}
message ScannerDirtyUsageSnapshotResponse {
string instance_id = 1;
uint64 generation = 2;
uint64 pending_bucket_count = 3;
uint32 protocol_version = 4;
// Incomplete snapshots are all-or-nothing and carry no bucket entries.
bool complete = 5;
repeated ScannerDirtyUsageBucket buckets = 6;
bytes response_proof = 7;
}
// A short-lived storage-owned read admission used only around a final
// scanner metadata publication. It is intentionally separate from the
// ScannerActivity observation wire so v6/v7 rolling compatibility remains
@@ -1206,6 +1227,7 @@ service NodeService {
// rpc CommitBinary() returns () {};
rpc SignalService(SignalServiceRequest) returns (SignalServiceResponse) {}; // auth-policy: body-bound
rpc ScannerActivity(ScannerActivityRequest) returns (ScannerActivityResponse) {}; // auth-policy: body-bound
rpc ScannerDirtyUsageSnapshot(ScannerDirtyUsageSnapshotRequest) returns (ScannerDirtyUsageSnapshotResponse) {}; // auth-policy: body-bound
rpc AcquireScannerPublicationLease(ScannerPublicationLeaseRequest) returns (ScannerPublicationLeaseResponse) {}; // auth-policy: body-bound
rpc ReleaseScannerPublicationLease(ScannerPublicationLeaseReleaseRequest) returns (ScannerPublicationLeaseReleaseResponse) {}; // auth-policy: body-bound
rpc BackgroundHealStatus(BackgroundHealStatusRequest) returns (BackgroundHealStatusResponse) {}; // auth-policy: read-only
crates/s3-client/src/api_error_response.rs
+22 -1
View File
@@ -57,7 +57,11 @@ fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
where
D: Deserializer<'de>,
{
Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
let code = String::deserialize(d)?;
if code == "NoSuchObject" {
return Ok(S3ErrorCode::NoSuchKey);
}
Ok(S3ErrorCode::from_bytes(code.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
}
impl Default for ErrorResponse {
@@ -325,4 +329,21 @@ mod tests {
assert_eq!(response.code, S3ErrorCode::NoSuchVersion);
assert_eq!(response.status_code, StatusCode::NOT_FOUND);
}
#[test]
fn normalizes_provider_specific_missing_object_code() {
let mut headers = HeaderMap::new();
headers.insert("x-amz-request-id", "request-id".parse().expect("request ID header should parse"));
let response = http_resp_to_error_response(
StatusCode::NOT_FOUND,
&headers,
b"<Error><Code>NoSuchObject</Code><Message>remote detail</Message></Error>".to_vec(),
"bucket",
"object",
);
assert_eq!(response.code, S3ErrorCode::NoSuchKey);
assert_eq!(response.status_code, StatusCode::NOT_FOUND);
}
}
crates/s3-client/src/api_get_object.rs
+242 -7
View File
@@ -30,7 +30,9 @@ use tokio_util::io::StreamReader;
use crate::{
api_error_response::err_invalid_argument,
api_get_options::GetObjectOptions,
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider},
transition_api::{
ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, collect_response_body, to_object_info_for_provider,
},
};
use futures_util::StreamExt;
use http_body_util::BodyExt;
@@ -39,6 +41,42 @@ use hyper::body::Bytes;
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use tokio_util::io::ReaderStream;
fn response_limit_from_range(opts: &GetObjectOptions) -> Result<Option<usize>, std::io::Error> {
let Some(range) = opts
.headers
.iter()
.find_map(|(name, value)| name.eq_ignore_ascii_case("range").then_some(value.as_str()))
else {
return Ok(None);
};
let Some((unit, bounds)) = range.split_once('=') else {
return Ok(None);
};
if !unit.eq_ignore_ascii_case("bytes") {
return Ok(None);
}
let Some((start, end)) = bounds.split_once('-') else {
return Ok(None);
};
if start.is_empty() || end.is_empty() || end.contains(',') {
return Ok(None);
}
let start = start
.parse::<u64>()
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range start is invalid"))?;
let end = end
.parse::<u64>()
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range end is invalid"))?;
let length = end
.checked_sub(start)
.and_then(|length| length.checked_add(1))
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length overflows"))?;
let limit = usize::try_from(length).map_err(|_| {
std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length does not fit in memory")
})?;
Ok(Some(limit))
}
impl TransitionClient {
pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
let _ = opts;
@@ -54,6 +92,7 @@ impl TransitionClient {
object_name: &str,
opts: &GetObjectOptions,
) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
let max_response_bytes = response_limit_from_range(opts)?;
let resp = self
.execute_method(
http::Method::GET,
@@ -81,18 +120,214 @@ impl TransitionClient {
let h = resp.headers().clone();
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
let body_vec = if let Some(limit) = max_response_bytes {
collect_response_body(body, limit).await?
} else {
let mut body_vec = Vec::new();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
}
body_vec
};
Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
}
}
#[cfg(test)]
mod bounded_response_tests {
use super::response_limit_from_range;
use crate::{
api_get_options::GetObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, TransitionClient, collect_response_body},
};
use http_body_util::Full;
use hyper::body::Bytes;
use std::time::Duration;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpListener,
};
#[test]
fn closed_range_derives_a_collection_limit_without_new_public_options() {
let mut opts = GetObjectOptions::default();
opts.set_range(5, 11).expect("the closed range should be valid");
assert_eq!(response_limit_from_range(&opts).expect("the range should parse"), Some(7));
}
#[tokio::test]
async fn response_collection_rejects_the_body_that_exceeds_its_range_limit() {
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let max_response_bytes = response_limit_from_range(&opts)
.expect("the range should parse")
.expect("the closed range should have a limit");
let err = collect_response_body(Full::new(Bytes::from_static(b"RustFSxx")), max_response_bytes)
.await
.expect_err("the collection layer must reject a response larger than its limit");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let request = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
let request = String::from_utf8_lossy(&request).into_owned();
let response = format!(
"HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
stream
.write_all(response.as_bytes())
.await
.expect("fixture should write response headers");
stream.write_all(body).await.expect("fixture should write response body");
request
});
let client = TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
)
.await
.expect("fixture client should build");
Some((client, request))
}
#[tokio::test]
async fn real_transport_accepts_the_exact_closed_range_length() {
let Some((client, request)) = bounded_get_fixture(b"RustFS!").await else {
return;
};
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let (_, _, mut reader) = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect("a seven-byte response should fit the requested range");
let mut body = Vec::new();
reader
.read_to_end(&mut body)
.await
.expect("bounded response should be readable");
assert_eq!(body, b"RustFS!");
assert!(
request
.await
.expect("fixture should join")
.to_ascii_lowercase()
.contains("\r\nrange: bytes=0-6\r\n")
);
}
#[tokio::test]
async fn real_transport_rejects_a_body_larger_than_the_closed_range() {
let Some((client, request)) = bounded_get_fixture(b"RustFS!!").await else {
return;
};
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let err = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect_err("an eight-byte response must exceed the seven-byte range limit");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(
request
.await
.expect("fixture should join")
.to_ascii_lowercase()
.contains("\r\nrange: bytes=0-6\r\n")
);
}
#[tokio::test]
async fn overflowing_closed_range_is_rejected_before_network_io() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let client = TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
)
.await
.expect("fixture client should build");
let mut opts = GetObjectOptions::default();
opts.headers
.insert("range".to_string(), "bytes=0-18446744073709551615".to_string());
let err = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect_err("an overflowing closed range must be rejected locally");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert!(
tokio::time::timeout(Duration::from_millis(100), listener.accept())
.await
.is_err()
);
}
}
#[derive(Default)]
pub struct GetRequest {
pub buffer: Vec<u8>,
crates/scanner/src/lib.rs
+7 -4
View File
@@ -89,14 +89,17 @@ pub use scanner::{
scanner_cycle_schedule_status, scanner_pause_backlog_status, scanner_topology_digest,
};
pub use scanner_io::{
ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,
record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state,
scanner_maintenance_generation,
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change,
scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation,
};
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
use std::sync::atomic::{AtomicU64, Ordering};
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
pub use storage_api::scan::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
pub use storage_api::scan::{
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
};
pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
crates/scanner/src/scanner_io.rs
+3 -3
View File
@@ -748,9 +748,9 @@ pub(crate) use cache::{
current_cache_root_or_prepare_with_generation,
};
pub use dirty_usage::{
ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,
record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state,
scanner_maintenance_generation,
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change,
scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation,
};
#[cfg(test)]
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
crates/scanner/src/scanner_io/dirty_usage.rs
+48
View File
@@ -27,6 +27,25 @@ pub struct ScannerDirtyUsageState {
pub pending: bool,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ScannerDirtyUsageBucket {
pub bucket: String,
pub generation: u64,
}
/// A point-in-time view of the local dirty bucket generations.
///
/// `complete == false` is an all-or-nothing overflow signal: `buckets` is
/// empty and callers must fall back to the global dirty generation rather than
/// treating a bounded prefix as authoritative.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ScannerDirtyUsageSnapshot {
pub generation: u64,
pub pending_bucket_count: u64,
pub complete: bool,
pub buckets: Vec<ScannerDirtyUsageBucket>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
pub enum ScannerDirtyUsageAckError {
#[error("scanner process instance changed before dirty usage acknowledgement")]
@@ -98,6 +117,35 @@ pub fn scanner_dirty_usage_state() -> ScannerDirtyUsageState {
}
}
pub fn scanner_dirty_usage_snapshot(max_entries: usize) -> ScannerDirtyUsageSnapshot {
let (generation, pending_bucket_count, complete, mut buckets) = {
let dirty_buckets = dirty_usage_buckets();
let generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire);
let pending_bucket_count = usize_to_u64_saturated(dirty_buckets.len());
let complete = dirty_buckets.len() <= max_entries;
let buckets = if complete {
dirty_buckets
.iter()
.map(|(bucket, generation)| ScannerDirtyUsageBucket {
bucket: bucket.clone(),
generation: *generation,
})
.collect::<Vec<_>>()
} else {
Vec::new()
};
(generation, pending_bucket_count, complete, buckets)
};
buckets.sort_unstable_by(|left, right| left.bucket.cmp(&right.bucket));
ScannerDirtyUsageSnapshot {
generation,
pending_bucket_count,
complete,
buckets,
}
}
pub fn acknowledge_dirty_usage_generation(
instance_id: &str,
generation: u64,
crates/scanner/src/scanner_io/tests.rs
+45
View File
@@ -500,6 +500,51 @@ fn dirty_usage_generation_acknowledgement_preserves_newer_mutations() {
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn dirty_usage_snapshot_is_sorted_and_reports_its_cutoff() {
clear_dirty_usage_buckets_for_tests();
let empty = scanner_dirty_usage_snapshot(0);
assert_eq!(empty.pending_bucket_count, 0);
assert!(empty.complete);
assert!(empty.buckets.is_empty());
record_dirty_usage_bucket("videos");
record_dirty_usage_bucket("photos");
let expected_generation = scanner_dirty_usage_state().generation;
let snapshot = scanner_dirty_usage_snapshot(2);
assert_eq!(snapshot.generation, expected_generation);
assert_eq!(snapshot.pending_bucket_count, 2);
assert!(snapshot.complete);
assert_eq!(
snapshot
.buckets
.iter()
.map(|bucket| bucket.bucket.as_str())
.collect::<Vec<_>>(),
vec!["photos", "videos"]
);
assert!(snapshot.buckets.iter().all(|bucket| bucket.generation <= snapshot.generation));
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn dirty_usage_snapshot_marks_truncated_results_incomplete() {
clear_dirty_usage_buckets_for_tests();
record_dirty_usage_bucket("archive");
record_dirty_usage_bucket("photos");
let snapshot = scanner_dirty_usage_snapshot(1);
assert_eq!(snapshot.pending_bucket_count, 2);
assert!(!snapshot.complete);
assert!(snapshot.buckets.is_empty(), "incomplete snapshots must not expose a partial bucket list");
clear_dirty_usage_buckets_for_tests();
}
#[test]
#[serial]
fn dirty_usage_generation_acknowledgement_rejects_stale_process_and_future_generation() {
crates/scanner/src/storage_api.rs
+4 -1
View File
@@ -304,7 +304,10 @@ pub(crate) mod scan {
};
#[cfg(test)]
pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions, ObjectIO};
pub use super::storage_contracts::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
pub use super::storage_contracts::{
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
};
}
pub(crate) mod scanner_io {
crates/storage-api/src/lib.rs
+3
View File
@@ -56,6 +56,9 @@ pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 5;
/// state is not authenticated by that version.
pub const SCANNER_ACTIVITY_V6_PROTOCOL_VERSION: u32 = 6;
pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 7;
pub const SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION: u32 = 1;
pub const SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES: usize = 4096;
pub const SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE: usize = 512 * 1024;
#[derive(Debug, serde::Deserialize, serde::Serialize)]
#[serde(deny_unknown_fields)]
crates/storage-api/src/object.rs
+10
View File
@@ -243,6 +243,16 @@ impl ObjectToDelete {
#[derive(Debug, Default, Clone)]
pub struct DeletedObject {
pub delete_marker: bool,
/// True when the delete plan looked the target up and found no such
/// object or version.
///
/// The lookup only runs when the plan needs the source (Object Lock
/// check, replication decision, tier journal, or an expected identity),
/// so this proves absence and never proves presence: it stays false when
/// no lookup ran. Callers that must not announce a delete that removed
/// nothing need this, because the disk layer treats an absent version as
/// an idempotent success and reports `found` regardless.
pub source_missing: bool,
pub delete_marker_version_id: Option<Uuid>,
pub object_name: String,
pub version_id: Option<Uuid>,
docs/architecture/README.md
+1
View File
@@ -48,6 +48,7 @@ Required headings and strings in these files are asserted by `scripts/check_arch
| [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration |
| [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced |
| [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch |
| [remote-credential-sealing-adr.md](remote-credential-sealing-adr.md) | adding, reading, or persisting a stored remote credential (replication target, remote tier, on-demand migration source), or needing the sealed-envelope format and its mixed-version rules |
| [tier-stats-contract.md](tier-stats-contract.md) | changing what `GET /rustfs/admin/v3/tier-stats` returns, adding a tier accounting source, or wiring a metric to a remote tier request |
## Support and compatibility matrices (release-facing, keep current)
docs/architecture/compat-cleanup-register.md
+1
View File
@@ -12,6 +12,7 @@
## Open Items
- `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation.
- `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains.
- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release.
- `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources.
- `backlog-2122` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources.
docs/architecture/ilm-tiering-persistence-contracts.md
+119 -13
View File
@@ -30,7 +30,7 @@ These are approved-target invariants. A protocol's explicitly labeled current ex
| Remote PUT is in flight or its response is unknown | Transition transaction | Only cleanup of its own canonical candidate, subject to the transaction recovery predicate | Durable transaction identity plus a known remote-version state; the approved target also requires expiry and durable takeover of the creator fence |
| Local transition commit is complete | Exact transitioned version in `xl.meta` | No | Current recovery finds the transaction's logical bucket/object/version and checks `TRANSITION_COMPLETE` plus the same remote object, tier, and remote version. It does not compare the recorded data directory, modification time, size, or ETag; the approved target adds that full source comparison |
| An ordinary delete removes that transitioned version | Hidden `xl.meta` free-version | Yes | Metadata quorum atomically removes the visible version and preserves its exact tier tuple in the free-version |
| A recursive prefix/delete-all operation cannot preserve per-object markers | v6 journal bound to an immutable dispatch manifest | Yes, but only after manifest completion and all-pool absence proof | `DispatchAuthorized`, local destructive mutation, every journal `Committed`, then manifest `Completed` |
| A recursive prefix/delete-all operation cannot preserve per-object markers | v6 journal bound to an immutable single dispatch manifest or a chunk-parent-bound child manifest | Yes, but only after child/manifest completion and all-pool absence proof | `DispatchAuthorized`, exact local destructive mutation, every journal `Committed`, then child/manifest `Completed`; a chunk parent advances only after that child completion |
| Tier configuration mutation, manual job, or decommission receipt | Intent/admission/copy proof only | No | These records gate configuration, scheduling, or migration; they never become remote-object cleanup owners |
An old journal and a free-version can coexist during compatibility recovery. That coexistence is evidence of multiple possible owners, not permission to choose one: the journal path must retain its record until the version-specific recovery rule proves which owner is authoritative.
@@ -50,11 +50,12 @@ All keys below are objects in the internal metadata bucket. The table gives the
| Manual worker result | `rustfs-manual-transition-worker-result-v1` | `ilm/manual-transition/results/<job shards>/<job-id>/<task-key>.json` | The worker persists it after an actual result; no current GC owner | Immutable job/task key and outcome/reason | Append-only create with `If-None-Match: *` and maximum parity |
| Legacy tier-delete journal | Versions 1 through 5 | `ilm/tier-delete-journal/<identity-digest>.json` | The deleting path creates it; version-specific journal recovery cleans it | Remote tuple; v2 adds backend identity, v3 exact version, v4 version state, v5 stable source and transaction state | v5 state changes use ETag CAS; v3/v4 recovery rereads and conditionally cleans. Initial legacy-compatible writes can still be unconditional |
| Sole-owner tier-delete journal | Version 6 | `ilm/tier-delete-journal-v6/<operation-id>/<identity-digest>.json` | The manifest coordinator creates/dispatches it; the journal worker deletes the remote object and cleans the record | Exact remote/source/backend identity plus manifest/operation/topology binding; mutable state `Prepared`/`Dispatched`/`Committed` | Create-only and fenced ETag CAS. Record cleanup writes a terminal receipt first only while a decommission run is active; ordinary recovery without one conditionally deletes the exact ETag directly |
| Tier-delete dispatch manifest | Version 1 | `ilm/tier-delete-dispatch-manifests/<scope-digest>.json` | The prefix-delete coordinator creates it; manifest recovery is its only rollback/completion owner | Immutable operation, bucket/incarnation/prefix, sorted journal set/count/digest, topology generation; mutable manifest state | Create-only and fenced ETag CAS; lost authorization response requires exact strong readback |
| Tier-delete dispatch manifest | Version 1 | Single dispatch: `ilm/tier-delete-dispatch-manifests/<scope-digest>.json`; chunk child: `ilm/tier-delete-dispatch-manifests/chunks/<scope-digest>/<operation-id>.json` | The prefix-delete coordinator creates it; manifest recovery is its only rollback/completion owner | Immutable operation, bucket/incarnation/prefix, sorted journal set/count/digest, topology generation; mutable manifest state | Create-only and fenced ETag CAS; lost authorization response requires exact strong readback. A child cannot authorize local mutation without the exact active parent binding |
| Tier-delete chunk parent | Version 1 with `record_type = "chunked_parent"` | `ilm/tier-delete-dispatch-manifests/<scope-digest>.json` | The over-limit prefix-delete coordinator creates and advances it; parent recovery advances completed children and removes the terminal parent | Immutable operation, bucket/incarnation/prefix/topology; mutable monotonic revision, next child sequence, completed journal count, one optional exact child binding, and `Active`/`Completed` state | Create-only and fenced ETag CAS. The parent binds a `Preparing` child before it can become `DispatchAuthorized`; final `Completed` follows an error-free, non-truncated empty-candidate rescan and local prefix deletion |
| Decommission durable-namespace receipt | `v2` | `decommission/ilm-receipts/<run-token>/<source-path>/<id-kind>/<id>.json` | The decommission coordinator writes target/source proof and is the only cleanup owner for that run | Source path, namespace and record identity, monotonic checkpoint, optional terminal checkpoint, optional v6 topology generation | Create-only then ETag CAS merge; checksum envelope; maximum parity |
| Decommission expected-receipt manifest | `v1` | `decommission/ilm-manifests/<run-token>.json` | The source-pool decommission coordinator creates and cleans it | Run token plus exact sorted receipt-path count/digest | Create-only, exact readback, and verification before pool removal |
`durable_namespace.rs` registers exactly the two tier-journal namespaces, dispatch manifest, transaction, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object.
`durable_namespace.rs` registers exactly the two tier-journal namespaces, the dispatch-record namespace shared by single manifests, chunk children, and chunk parents, the transaction namespace, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object.
## Durable fences and write primitives
@@ -89,8 +90,8 @@ Lock ordering is part of the recovery contract. Callers acquire only the locks n
| Path | Current acquisition order | Operations allowed while held | Operations forbidden while held |
|---|---|---|---|
| Tier edit/remove/clear | Tier-config namespace WRITE lock; dedicated owned `admin_updates` serialization mutex; short `TierConfigMgr` state locks only while accessing manager/runtime state | The dedicated `admin_updates` guard intentionally spans awaited backend validation/probes, peer Prepare/Commit/Abort RPC, reference scans, config CAS, and candidate publication in the current protocol | Ordinary manager `RwLock` and runtime-state `Mutex` guards must not cross awaited network I/O; that rule does not prohibit the dedicated `admin_updates` guard from spanning those awaits. Remote object DELETE is never part of mutation |
| v6 manifest prepare | Caller already holds the bucket-lifecycle WRITE fence; caller acquires a bucket-metadata transaction READ guard covering the Object Lock and bucket-incarnation snapshot and keeps it through local mutation; exact tier-generation leases; fleet/topology proof; synthetic manifest-operation WRITE lock | Build and write the immutable journal set and manifest, validate exact set/digest, then authorize local dispatch while both caller-held bucket guards and all leases remain current | Remote tier DELETE; per-object worker cleanup; releasing the metadata guard or a required lease before the authorized local mutation completes |
| v6 manifest recovery | Fleet/topology proof; bucket-lifecycle WRITE lock; synthetic manifest-operation WRITE lock | Read/write manifest and journal metadata, verify exact set/digest, authorize, converge, or roll back local records | Remote tier DELETE; per-object worker cleanup; rollback after authorization |
| v6 manifest prepare | Caller already holds the bucket-lifecycle WRITE fence; caller acquires a bucket-metadata transaction READ guard covering the Object Lock and bucket-incarnation snapshot and keeps it through local mutation; exact tier-generation leases; fleet/topology proof; for a single dispatch, synthetic manifest-operation WRITE; for a child, parent-operation WRITE then child-operation WRITE | Build and write one immutable bounded journal set and manifest, validate exact set/digest, then authorize local dispatch while both caller-held bucket guards and all leases remain current. A parent binding is durable before child authorization | Remote tier DELETE; per-object worker cleanup; releasing the metadata guard or a required lease before the authorized local mutation completes; child-to-parent nested lock acquisition |
| v6 manifest/parent recovery | Fleet/topology proof; bucket-lifecycle WRITE lock; then exactly one synthetic manifest- or parent-operation WRITE lock | Read/write manifest, parent, and journal metadata; verify exact set/digest/binding; converge or roll back child records; advance a parent only after child completion | Remote tier DELETE; per-object worker cleanup; rollback after authorization; taking a child lock while holding a parent lock in background recovery |
| v5 journal destructive recovery | Synthetic per-journal recovery lock; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Authoritative source/free-version scan; fenced state CAS; for an eligible terminal state, one bounded remote DELETE; conditional record cleanup | Any delete when a lock or lease is lost; publishing local metadata; selecting an arbitrary backend/version |
| v6 journal destructive recovery | Synthetic per-journal recovery lock; fleet/topology proof; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Immutable manifest/topology validation, authoritative source/free-version scan, fenced state CAS, and, for an eligible terminal state, one bounded remote DELETE followed by record cleanup | Any delete when a lock, lease, or fleet proof is lost; publishing local metadata; selecting an arbitrary backend/version |
| Free-version cleanup | Bucket-lifecycle READ lock; exact tier-generation lease; all physical object WRITE locks in stable pool/set order | Exact all-pool scan; bounded remote DELETE; local marker removal; post-delete rescan | Deleting before the free-version is the sole owner or after any fence changes |
@@ -242,7 +243,9 @@ The lease interval is 60 seconds. CAS behavior is phase-specific: cancellation t
| v5 | Adds stable source identity and transaction state. Recovery proves source/free-version presence across physical sets before deciding abort, retain, or commit |
| v6 | Sole-owner record bound to immutable operation/manifest/topology. It is the only new journal format for destructive prefix dispatch |
The v1 manifest binds a bucket incarnation and prefix to an operation UUID, topology generation, and sorted journal names/count/digest. Its legal edges are:
For a complete source set at or below 200,000 journals, the byte-compatible v1 single manifest remains at the deterministic scope-digest root. For a larger source set, that same root instead contains a strict `chunked_parent` sentinel, and each bounded child uses the unchanged v1 manifest payload at `chunks/<scope-digest>/<operation-id>.json`. A pre-chunking reader rejects the parent schema and the non-root child path, so it cannot start a competing single dispatch while chunking is active.
A v1 single/child manifest binds a bucket incarnation and prefix to an operation UUID, topology generation, and sorted journal names/count/digest. Its legal edges are:
```text
Preparing -> DispatchAuthorized -> Completed
@@ -251,6 +254,16 @@ Preparing -> DispatchAuthorized -> Completed
The journal edge is `Prepared -> Dispatched -> Committed`. A manifest coordinator owns the whole `Prepared` set and is the only actor that may roll it back or complete the manifest. A per-journal worker cannot remove one prepared member.
The chunk parent stores only monotonic O(1) progress and binds at most one child:
```text
Active(no child) -> Active(bound Preparing child)
Active(bound Completed child) -> Active(no child, next sequence/count)
Active(no child, final empty rescan and local delete complete) -> Completed
```
Child creation is ordered `Preparing` child create, parent binding CAS, journal preparation/dispatch, then child `DispatchAuthorized`. One request exactly replays one bounded child under source-object locks, commits every child journal, marks the child `Completed`, advances the parent, and returns retry-required. A successor request rescans from the prefix start; no listing cursor crosses bucket-lock lifetimes. New or changed source identities are therefore admitted only by a fresh child. Final success requires an error-free, non-truncated scan with no v6 candidate, local prefix deletion under the same bucket fence, and the parent `Completed` CAS.
### Journal recovery decisions
| Record/state and evidence | Unique current owner | Current recovery decision | Remote DELETE admission |
@@ -278,11 +291,24 @@ The journal edge is `Prepared -> Dispatched -> Committed`. A manifest coordinato
| `Completed` | Journal workers own member cleanup; coordinator owns final manifest cleanup | Wait for all member records to disappear, then conditionally delete manifest | Journal workers are the remote-delete owners |
| Missing member, set/digest mismatch, wrong incarnation/topology, corrupt state, scan ambiguity, or cancellation | No actor acquires new destructive authority | Retain | Fail closed; an authorized operation never rolls back |
Manifest preparation is bounded by 200,000 journals and a 32 MiB record. Journal recovery scans bounded batches with per-entry timeouts and limited concurrency. Those are work bounds, not retention bounds: v1/v2 quarantine and unresolved v6 operations can remain indefinitely.
### Chunk-parent recovery decisions
| Parent/child state and evidence | Unique current owner | Current recovery decision | Authority |
|---|---|---|---|
| Active parent with no child | A later prefix-delete retry under bucket WRITE | Retain the parent and rescan from the prefix start | No local or remote deletion |
| Active parent with exact bound `Preparing`/`Aborting`/`Aborted` child | Child manifest coordinator | Retain parent while child recovery rolls back and removes the child | Never authorize or advance that child |
| Active parent with exact bound `DispatchAuthorized` child | The bound child permit or journal recovery | Resume exact-source replay on request; otherwise retain until all journals become `Committed` and the child becomes `Completed` | Only the exact parent-bound child may authorize local replay |
| Active parent with exact bound `Completed` child | Parent coordinator | CAS the next sequence/count and clear the binding | Parent progress only; remote DELETE remains owned by committed child journals |
| Bound child missing and its exact operation journal namespace is non-empty or unreadable | No actor can prove safe abandonment | Retain and fail closed | Never clear the binding |
| Bound child missing and its exact operation journal namespace is proven empty | Parent coordinator | CAS-clear the stale binding and retry from the prefix start | No deletion; the fresh scan reconstructs any remaining source work |
| Completed parent with no active child | Parent recovery | Record terminal decommission evidence when applicable, then conditionally delete the exact parent ETag | Metadata cleanup only |
| Parent identity, child binding, topology, incarnation, sequence/count, CAS generation, or fence mismatches | No actor acquires progress authority | Retain | Fail closed |
Single and child manifest preparation is bounded by 200,000 journals and a 32 MiB record. On the first unique candidate beyond the bound, the physical walks are cancelled and only the retained exact batch can proceed; cancellation fallout is not absence proof. The parent never accumulates child names, and exact local replay uses bounded concurrency. Journal and manifest recovery retain their existing bounded pages, per-entry timeouts, and concurrency. These are work bounds, not retention bounds: v1/v2 quarantine and unresolved v6 operations can remain indefinitely.
### Approved target and open design
- New destructive prefix paths use only v6 plus a manifest. No new v1-v5 sole-owner records may be created.
- New destructive prefix paths use only v6 plus either one byte-compatible manifest or one parent-bound sequence of byte-compatible child manifests. No new v1-v5 sole-owner records may be created.
- Preserve the two-phase authorization barrier: all prepared records, durable barrier, all dispatched records, durable `DispatchAuthorized`, local mutation, journals committed, durable `Completed`, then remote DELETE.
- Do not downgrade every v6-aware recovery worker while v6 records remain. v5-and-older readers reject and retain v6 records; older nodes may continue producing fallback free-versions until the fleet is homogeneous.
- **Open:** bounded age/count policy and operator disposition for quarantined v1/v2, incomplete manifests, and repeatedly failing exact deletes. Capacity rejection and recovery throughput must not be “fixed” by weakening ownership proof.
@@ -301,17 +327,97 @@ Recursive prefix/delete-all is the exception because physical directory removal
- Every code path that removes or overwrites transitioned metadata either atomically leaves an exact free-version owner or enters an already-authorized v6 dispatch.
- If both a journal and free-version are observable, recovery preserves the remote object until version-specific authority and all-pool absence prove a single owner.
- Missing `transition-version-state` remains unknown. Compatibility work must not synthesize known-disabled or exact semantics merely from an empty stored version ID.
- Missing `transitioned-version-state` remains unknown. Compatibility work must not synthesize known-disabled or exact semantics merely from an empty stored version ID.
How historical objects without RustFS transition-version-state can be upgraded safely is an **open design**. It requires a read/repair rule with rollback behavior before destructive cleanup may consume those records.
The following single-record protocol approves how historical objects without RustFS `transitioned-version-state` can be upgraded. It does not approve a bulk scanner or allow destructive cleanup to consume an unproven record.
## Legacy transitioned-version-state reconciliation
### Current
An absent `transitioned-version-state` key decodes as `TransitionVersionState::Unknown`. The current GET and free-version cleanup paths reject that state rather than interpreting an empty remote version as unversioned. There is no admin route that repairs this field in `xl.meta`. The existing transition-transaction reconcile route operates on expired `UploadOutcomeUnknown` transaction records and can exact-delete their canonical candidates; it is a separate protocol and must not be reused for metadata reconciliation.
An explicitly persisted `unknown`, a malformed state, conflicting RustFS/MinIO compatibility keys, an invalid or nil version identifier, and a partial transition tuple are not legacy absence. They remain invalid or ambiguous and fail closed.
### Approved single-record control surface
The approved target is one synchronous, exact logical-version operation. It does not list a bucket or prefix and does not create a durable job:
```text
GET /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=<bucket>&object=<object>&versionId=<local-version-id>
POST /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=<bucket>&object=<object>&versionId=<local-version-id>
```
`versionId` is required; the literal `null` is the explicit selector for a locally unversioned object, while an omitted or empty selector is invalid. GET requires `admin:ListTier`. It performs an authoritative all-pool metadata read and a bounded server-side live backend probe, but it never writes metadata or mutates the remote tier. Its response includes the canonical immutable source tuple, every original per-set missing-state representation, the proposed target tuple when one is provable, an opaque reconciliation digest over all three, fleet/topology and tier-generation readiness, and whether POST is ready to attempt migration. GET never labels an unmodified legacy record `migrated`.
POST requires `admin:SetTier`. Its body contains `confirm: true`, the complete source and target tuples, every original per-set representation, and their reconciliation digest returned by a recent GET. The server does not trust a client-supplied state or external assertion: it rereads the object, requires every immutable source field to match exactly after canonicalization, repeats the bounded live probe, and derives the target state itself. For the mutable state/version/destination fields, each set must equal either its original missing-state representation bound by the digest or the exact newly proven target; this is the only accepted partial-retry shape. A missing confirmation, any other stale tuple or digest, or a widened selector is rejected before any write.
The expected tuple binds all evidence whose change could redirect the repair:
- bucket name and incarnation; exact object name, local version ID, data directory, modification time, size, and ETag;
- transition completion status, tier name, canonical remote object name, raw remote-version key presence/value, and raw state-key presence/value under both compatibility prefixes;
- tier-config generation and the `transition-tier-destination-id` binding, including backend type, endpoint/bucket/prefix identity, and the credential-independent backend fingerprint;
- topology generation and the generation/digest of every authoritative `xl.meta` copy found across pools and sets.
The only allowed state derivations are:
| Live proof | Persisted state | Persisted remote version | Remote request meaning |
|---|---|---|---|
| Provider proves versioning disabled and the candidate is present | `KnownDisabled` | Absent/empty | Send no `versionId` |
| Provider proves suspended-version null semantics and the exact candidate is present | `SuspendedNull` | Literal `null` | Send the provider's null-version form |
| Provider proves one exact, nonempty, non-`null` opaque version | `Exact` | That exact opaque identifier | Send that exact `versionId` |
Missing, multiple, changing, or unsupported probe results do not select a state. In particular, a preexisting empty remote-version field is not evidence for `KnownDisabled`, and a client may not nominate `Exact` or `SuspendedNull`.
The POST may write only the derived `transitioned-version-state`, its corresponding `transitioned-versionID` value when the proven model requires one, and the exact `transition-tier-destination-id` binding under both compatibility prefixes in the matching `xl.meta` version. It does not issue remote GET beyond the proof probe, remote PUT, remote DELETE, local object DELETE, free-version cleanup, transaction/journal cleanup, tier-config mutation, restore, or source-payload rewrite. Reconciliation establishes metadata meaning; a later ordinary owner may perform cleanup under its own destructive protocol.
### Outcome contract
POST returns exactly one of the following outcomes and whether it changed bytes. GET uses the same diagnostic names for non-applicable cases, returns `ready-to-migrate` when a missing state is provable, and returns `migrated` only when strong readback shows the record was already explicit and converged:
| Outcome | Meaning and permitted effect |
|---|---|
| `migrated` | All authoritative copies already contain, or were monotonically advanced to, the same proven state and destination identity. Only this outcome makes the record eligible for later ordinary read/delete semantics. |
| `retained-ambiguous` | The tuple is structurally legacy-compatible, but the live probe is missing, multiple, changing, unsupported, or otherwise cannot prove exactly one state. No metadata or remote object is changed. |
| `corrupt` | Explicit `Unknown`, malformed/contradictory compatibility keys, nil/invalid identifiers, partial transition metadata, or authoritative copies outside the one allowed `{original missing representation, exact proven target}` retry subset were observed. No backend probe is required after corruption is established, and nothing is changed. |
| `backend-unavailable` | The bound tier generation/destination cannot be acquired, the bounded probe fails, or a metadata quorum/strong readback needed to complete the operation is unavailable. Any already-persisted monotonic subset is retained for an idempotent retry; it is never rolled back. |
HTTP failure detail may distinguish a stale expected tuple, lost fence, timeout, or unavailable quorum, but it must preserve one of these machine-readable outcomes. Logs and audit events include request identity, object identity, tier, generations, outcome, and whether bytes changed; they never include credentials or raw credential-derived configuration.
### Fence, write, and retry order
The approved POST executes the following order. A step that cannot be proven stops the operation without remote mutation:
1. Authenticate `admin:SetTier`, validate the exact single-record selector, `confirm: true`, expected tuple, and digest.
2. Prove every required node advertises the reconciliation format and destination-identity capability; capture the fleet and topology generation. Unknown or unsupported nodes block the writer.
3. Acquire the bucket-lifecycle WRITE fence and validate the bucket incarnation.
4. Acquire the exact tier-config generation lease bound to the expected destination identity.
5. Acquire exact object-version WRITE locks for every owning physical pool/set in stable pool/set order.
6. Perform an authoritative all-pool read, reject duplicate/conflicting ownership, validate every compatibility key, and require each set to match the immutable source tuple plus either its digest-bound original missing-state representation or the exact proposed target.
7. Run one bounded, cancellation-aware live probe through the leased backend. No client or cached probe result is authority.
8. Before writing, revalidate the fleet/topology generation, bucket incarnation and lifecycle fence, tier lease/destination identity, physical owner set, and complete metadata tuple.
9. Write the same derived state and destination identity to each authoritative set with that set's metadata quorum and conditional generation. A timeout or response loss is resolved only by a strong read of that exact set.
10. Strongly reread every authoritative set and revalidate the full tuple, state, destination identity, and topology before returning `migrated`. Release locks and leases in reverse order.
Cross-pool and cross-set partial success is monotonic. The only legal repair edge is `missing state -> one proven {state, remote version, destination identity}`. A retry may accept an already-written subset only when every known copy equals the newly proven target, every remaining copy equals its original missing-state representation captured by the reconciliation digest, and all immutable source fields still match; it then fills only the missing copies. This exact target-plus-original subset is neither stale nor corrupt. The retry never clears a known state, rewrites it to another state, changes destination identity, or rolls a successful set back to missing/`Unknown`. Any other divergent value produces `corrupt`; an unavailable set/readback produces `backend-unavailable`, and destructive cleanup remains blocked until a later strong all-pool read proves complete convergence.
GET takes the same fleet/topology snapshot and authoritative all-pool read but no write locks that imply mutation authority. Because GET is advisory, POST always repeats every fence, read, and live proof rather than promoting the GET result.
### Mixed-version and future batch work
The writer gate requires every node that can serve, rewrite, heal, decommission, or recover the affected `xl.meta` to preserve the explicit state and destination identity. A rolling fleet with an unknown/unsupported node is inspect-only. Downgrade is blocked while reconciled records could be rewritten by readers that erase or misinterpret those fields. Cross-pool movement must either copy the proven tuple unchanged or block reconciliation; a first-match lookup is never sufficient.
Explicit `Unknown`, corruption, and ambiguity remain fail closed for reads that cannot prove non-destructive semantics and for every destructive path. A migrated record becomes ordinary explicit metadata, but reconciliation itself never transfers remote DELETE ownership.
A bucket/prefix/fleet batch reconcile is still an **open design**. It requires a separate durable job identity, create-only admission, lease/CAS checkpoint, bounded pages, per-record expected tuples and outcomes, cancellation/restart semantics, retention, fleet rollout negotiation, and status counters. Implementations must not approximate that protocol by adding a list selector or background loop to the synchronous route.
## Durable namespace receipts during decommission
### Current contract
Decommission cannot treat durable ILM objects as ordinary configuration blobs. `validate_durable_ilm_record` validates namespace, size, schema/checksum, identity, and a protocol-specific checkpoint, and most protocol branches recompute the canonical path. Its transition-transaction branch currently inherits the weaker final-component parser: mismatched shard directories, extra components, and uppercase hex can pass when the final UUID and record contents agree. Exact transition-path validation is therefore an approved target, not a current decommission guarantee. Checkpoint successors enforce journal/manifest legal states, transition identity and revision progression, monotonic manual-job progress, scope ownership, and immutable task/result payloads.
Decommission cannot treat durable ILM objects as ordinary configuration blobs. `validate_durable_ilm_record` validates namespace, size, schema/checksum, identity, and a protocol-specific checkpoint, and most protocol branches recompute the canonical path. Its transition-transaction branch currently inherits the weaker final-component parser: mismatched shard directories, extra components, and uppercase hex can pass when the final UUID and record contents agree. Exact transition-path validation is therefore an approved target, not a current decommission guarantee. Checkpoint successors enforce journal/manifest legal states, chunk-parent revision/sequence/count/binding progression, transition identity and revision progression, monotonic manual-job progress, scope ownership, and immutable task/result payloads.
The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current delete primitive: v6 journal/manifest cleanup uses the exact ETag, while transition-transaction cleanup remains unconditional as documented above. Completion verifies every expected receipt and target checkpoint before the source pool can be removed.
The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current delete primitive: v6 journal/manifest/parent cleanup uses the exact ETag, while transition-transaction cleanup remains unconditional as documented above. Completion verifies every expected receipt and target checkpoint before the source pool can be removed.
A terminal receipt is proof that an exact target copy reached a terminal checkpoint. It may authorize conditional removal of the matching source record when every active target copy is covered; it never authorizes remote DELETE. A terminal receipt on one target cannot hide a later nonterminal receipt on another target.
@@ -371,7 +477,7 @@ Transition transaction v1, manual job/task/result v1, and receipt v2 do not curr
| Journal v1/v2 | Readers decode but quarantine because remote-version authority is missing; compatibility writers can preserve these forms | Retain indefinitely unless a separately approved, authoritative repair protocol resolves them; never translate empty version ID to known-disabled |
| Journal v3/v4 | Readers recover supported committed records according to exact or explicit version-state semantics; current compatible writes use v4 for known state | Unknown/inconsistent state is retained. These legacy paths are not evidence that a new sole-owner operation may omit v5/v6 source proof |
| Journal v5 | Readers use stable source/all-pool proof; decoded v5 can be checkpointed, while new online sole-owner transactions are not emitted as v5 | Retain and recover conservatively during upgrade. Do not manufacture v5 from older records or use it to bypass v6 manifest authorization |
| Journal v6 and dispatch manifest v1 | v6-aware writers/readers require immutable manifest membership and topology; v5-and-older readers reject and retain v6 | Gate v6 writers on fleet capability. Drain v6 before removing all v6-aware workers; do not downgrade by rewriting a live v6 operation |
| Journal v6, dispatch manifest v1, and chunk parent v1 | v6-aware writers/readers require immutable manifest membership and topology. Complete sets at or below 200,000 retain the legacy root manifest bytes; larger sets install a strict parent at that root and operation-scoped v1 child payloads. Pre-chunking v6 readers reject the parent schema and child paths, while v5-and-older readers reject and retain v6 journals | Gate writers on the current fleet capability and retain the root parent for the entire active chunk sequence. Drain v6 before removing all v6-aware workers; do not downgrade by rewriting a live v6 operation |
| Decommission receipt v2 and expected manifest v1 | Current decommission readers validate exact schema/checksum/path/checkpoint and fail completion on unknown input | No ignore path. Mixed-version decommission must not complete unless every participant preserves the registered durable namespace; broader downgrade negotiation is open |
## Reconcile, observability, and retention
docs/architecture/remote-credential-sealing-adr.md
+52
View File
@@ -0,0 +1,52 @@
# Remote Credential Sealing ADR
**Use this when:** you add, read, or persist a stored remote credential — a replication target, a remote tier, or an on-demand migration source — or you need the sealed-envelope format, its fail-closed rules, and the mixed-version compatibility matrix.
**Source of truth:** the three stores that hold remote credentials today — `BUCKET_TARGETS_FILE` and `BUCKET_ON_DEMAND_MIGRATION_CONFIG` in `crates/ecstore/src/bucket/metadata.rs`, and `TIER_CONFIG_FILE` in `crates/ecstore/src/services/tier/tier.rs` — plus the consumers `crates/ecstore/src/bucket/bucket_target_sys.rs`, `crates/ecstore/src/services/tier/tier.rs`, and `crates/ecstore/src/bucket/on_demand_migration/config.rs`.
## Decision
Remote credentials are sealed **per field, into an added field, behind one shared seam**, and ECStore reaches KMS through an installed hook rather than a crate dependency.
1. **One seam, three consumers.** `BucketTargetSys`, `TierConfigMgr`, and `OnDemandMigrationSys` seal and unseal through a single ECStore-owned envelope type. No consumer talks to KMS, and no consumer defines its own ciphertext layout.
2. **Only secret material is sealed.** `secret_key` and `session_token` are sealed. Endpoint, region, ARN, bucket, prefixes, path style, TLS flags, and the custom CA bundle stay in clear text: they are needed for validation, listing, and support diagnosis, and none of them is a secret.
3. **Sealed material lives in an added field, never in place of the plaintext field.** A record carries either the plaintext field or the sealed field. A reader that does not understand the sealed field therefore finds the credential *absent* rather than finding a ciphertext string it would sign requests with.
4. **Unsealing happens at client construction, not at parse time.** `build_remote_s3_client` in `crates/ecstore/src/bucket/remote_s3_client.rs` is the single point that needs plaintext, so admin reads, listings, validation, and status paths never call KMS.
## Envelope format
A versioned, self-describing record: envelope version, KMS key id, KMS key version, algorithm, nonce, and ciphertext. It is stored base64 in the two JSON stores and as raw bytes inside the msgpack payload of the tier blob; the tier blob's own `TIER_CONFIG_FORMAT` / `TIER_CONFIG_VERSION` header constants are unchanged, because the envelope carries its own version.
The KMS encryption context binds each ciphertext to the record that owns it — store kind, owning bucket or tier name, and field name — so a ciphertext copied into another bucket, another tier, or another field fails to decrypt instead of silently authorizing a different remote.
## Why a hook instead of a dependency
`crates/ecstore/Cargo.toml` has no `rustfs-kms` dependency, and adding one would invert the crate layering. The established shape is an `OnceLock` hook that ECStore defines and the binary installs at startup: `EVENT_DISPATCH_HOOK` in `crates/ecstore/src/services/event_notification.rs`, installed by `install_ecstore_event_dispatch_hook` in `rustfs/src/server/event.rs`, and `ON_DEMAND_MIGRATION_CONFIG_HOOK` in `crates/ecstore/src/bucket/on_demand_migration/config.rs`. Sealing uses the same shape, with the binary supplying an implementation backed by `get_global_kms_service_manager` in `crates/kms/src/service_manager.rs`.
## Compatibility matrix
| Stored form | Reader | Behavior |
|---|---|---|
| Plaintext (today's format) | Old node | Unchanged. |
| Plaintext | New node | Read as plaintext, no KMS call. Carries a `RUSTFS_COMPAT_TODO` marker per [compat-cleanup-register.md](compat-cleanup-register.md). |
| Sealed | New node, hook installed | Unsealed at client construction. |
| Sealed | New node, no hook or decrypt failure | Typed error; the target, tier, or source is unusable and reports why. Never a default, an empty credential, or the ciphertext bytes. |
| Sealed | Old node | The credential field is absent, so the old node fails closed on its existing "missing credentials" path. This is the migration hazard the rollout gate exists for. |
## Rollout gate
Sealing is written only when KMS is configured **and** a module switch in `rustfs/src/module_switches.rs` is on, defaulting off in the release that introduces it. Reading sealed records is always supported; writing them is what waits. Operators enable the switch after every node in the cluster can read the format, and existing plaintext records are sealed by re-submitting the configuration through its admin API — this task ships no in-place migration sweep.
## Rotation
The envelope records the key id and key version it was wrapped under. Re-wrapping is the KMS side's job and follows [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md); nothing in this design rotates, re-wraps, or expires a key on its own.
## Fail-closed rules
- A missing hook, a malformed envelope, an unknown envelope version, a failed decrypt, or an encryption-context mismatch is a typed error, per the AGENTS.md rule that required values return a typed error when absent or corrupt.
- A seal failure fails the admin write. A configuration is never persisted with the secret dropped or left in clear text after the operator asked for sealing.
- Redaction is unchanged and independent: admin responses keep returning `REDACTED`, and `Debug` implementations keep hiding secret fields whether or not the stored form is sealed.
- Logs may carry the key id and envelope version. They never carry ciphertext, plaintext, or the encryption context's secret-adjacent values.
## Non-goals
Sealing the server config, IAM credentials, or object data keys; changing which principals may read a configuration; key material migration between KMS backends; and any at-rest protection when KMS is not configured — without KMS the stored form stays plaintext and the existing trust boundary (reserved bucket paths plus admin authorization) is unchanged.
docs/operations/tier-ilm-debugging.md
+45 -6
View File
@@ -21,6 +21,16 @@
| `FileMeta` / `FileInfo` / version metadata | `crates/filemeta/src/` |
| Dual-key internal metadata helpers (`insert_bytes` / `get_bytes`) | `crates/utils/src/http/metadata_compat.rs` |
## Free-version recovery controls
The dedicated free-version recovery loop is enabled by default and is independent of the data scanner and heal switches. Setting `RUSTFS_SCANNER_ENABLED=false` does not stop this repair loop. Set `RUSTFS_TIER_FREE_VERSION_RECOVERY_ENABLED=false` before process startup to disable only the dedicated persisted-marker walk. That setting does not disable lifecycle workers or prevent another scanner path from discovering a free version, and it can leave remote cleanup markers pending for longer, so use it as a break-glass pressure control rather than a cleanup mechanism.
Normal transitioned deletes pass a post-commit receipt directly to the lifecycle queue. The namespace walk is the crash, queue-pressure, mixed-version, and historical-record fallback. It cannot safely skip a bucket merely because that bucket has no current lifecycle rule or the referenced tier was removed: an older `xl.meta` free-version can still be the only owner of a required remote DELETE.
One background recovery page completes at most 10,000 logical objects across at most 100 buckets and enqueues at most 1,000 recoverable free versions. The scanner can decode one additional object to detect truncation; a continuation marker preserves the first unscanned bucket or the last completely scanned object. Every truncated page and follow-up sweep waits at least 60 seconds after the previous page completes; failed pages back off from 60 seconds to 10 minutes, and complete idle sweeps exponentially back off to 10 minutes with jitter. Individual walks have no fixed total timeout, but inherit the drive walk stall timeout so a large healthy bucket can make progress without recreating the old timeout/restart loop.
The structured `lifecycle_worker_state` recovery event reports `duration_ms`, `scanned_entries`, `buckets_scanned`, queue counts, truncation, and continuation markers. `rustfs_internal_stage_duration_ms{stage="lifecycle_free_version_recovery"}` records successful page duration; `stage="lifecycle_free_version_recovery_failed"` records failures.
## Metadata key conventions
Internal metadata is stored under both `x-rustfs-internal-<suffix>` and `x-minio-internal-<suffix>` for MinIO interoperability. `get_bytes` prefers the RustFS key and falls back to the MinIO key.
@@ -29,11 +39,11 @@ Internal metadata is stored under both `x-rustfs-internal-<suffix>` and `x-minio
|--------|---------|
| `transition-status` | `"complete"` when tiered |
| `transitioned-object` | tier key path (stored without the tier prefix; `get_dest` adds it) |
| `transitioned-versionID` | S3 version_id returned by tier PUT (16 raw UUID bytes, or absent) |
| `transitioned-versionID` | Provider version identifier: current exact UTF-8 text, legacy RustFS raw UUID bytes, MinIO's empty unversioned value, or absent for some historical unversioned records. Interpret it only with `transitioned-version-state` or a live compatibility probe. |
| `transition-tier` | tier name |
| `tier-free-versionID` | delete-marker version for free-version sweep |
Reading binary values must reject empty, malformed, and nil values (regression covered in `crates/filemeta/src/filemeta/version.rs` tests):
Legacy raw UUID values must reject empty, malformed, and nil UUIDs (regression covered in `crates/filemeta/src/filemeta/version.rs` tests):
```rust
get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)
@@ -42,7 +52,7 @@ get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)
// None for: absent key, wrong-length bytes, nil UUID
```
`transition_version_id == None` means the tier bucket is unversioned; the GET/DELETE against the tier must then send no `versionId` parameter. A nil UUID (`00000000-...`) sent as `?versionId=` causes `NoSuchVersion`. Do not use `Uuid::from_slice(..).unwrap_or_default()` here: it converts an empty metadata value into `Uuid::nil()`, which is exactly that failure.
`transition_version_id == None` means only that no usable legacy UUID projection exists; it does not prove the remote bucket's versioning model. Only an explicit `KnownDisabled` state authorizes ordinary GET/DELETE to omit `versionId`. A missing state with an absent or empty version key remains `Unknown` and requires the bounded compatibility probe or the approved reconcile workflow; it never directly authorizes cleanup. A nil UUID (`00000000-...`) sent as `?versionId=` causes `NoSuchVersion`. Do not use `Uuid::from_slice(..).unwrap_or_default()` here: it converts an empty metadata value into `Uuid::nil()`, which is exactly that failure.
## Inspect xl.meta directly
@@ -54,8 +64,8 @@ cargo build -p rustfs-filemeta --example dump_fileinfo
| Output | Meaning |
|---|---|
| `transition_ver_id: <none>` | No versionId will be sent to the tier (correct for a non-versioned tier bucket). |
| `transition_ver_id: <uuid>` | That UUID will be sent as `?versionId=<uuid>`. |
| `transition_ver_id: <none>` | No usable legacy UUID projection exists. Inspect `transitioned-version-state` and the raw compatibility keys; do not infer unversioned semantics. |
| `transition_ver_id: <uuid>` | A legacy UUID representation decoded successfully. It is not destructive authority unless the persisted state or reconcile proof establishes the exact remote model. |
There is one `xl.meta` per erasure shard disk (`{disk}/{bucket}/{object}/xl.meta`); all shards of a healthy object should be identical. `dump_versions` (same crate) lists every version in a file.
@@ -70,7 +80,7 @@ RUST_LOG=rustfs_ecstore::bucket::lifecycle=debug rustfs ...
| `fetching transitioned object from tier` | DEBUG | Emitted before the tier request. |
| `tier GET failed` | ERROR | Includes `tier_version_id`. |
If both `x-rustfs-internal-transitioned-versionID` and `x-minio-internal-transitioned-versionID` are the empty string, the object was transitioned to a non-versioned tier bucket and no versionId must be sent.
If the version keys are empty while `transitioned-version-state` is absent, the record has the historical MinIO unversioned shape but still remains `Unknown`; only the compatibility probe or reconcile protocol may prove `KnownDisabled`. If state is explicitly `KnownDisabled`, no `versionId` is sent.
## Manual transition run
@@ -144,6 +154,35 @@ Historical transition transactions in `upload_outcome_unknown` state can use an
`finalize_missing` re-runs the provider probe and fails closed for `unversioned_present`, `versioned_present`, `ambiguous`, `unsupported`, or probe errors. It never accepts an operator assertion in place of a live `missing` result. Providers without an authoritative probe or exact version deletion remain pending; the endpoint does not infer provider capabilities, accept external absence assertions, or select a candidate automatically.
## Reconcile legacy transition-version metadata
This section describes an **approved target that is not implemented yet**. The current server has no admin route that backfills a missing `transitioned-version-state` in `xl.meta`. Do not use the transaction reconcile route above for this purpose: that route owns an upload transaction candidate and may delete it, while legacy metadata reconciliation is non-destructive and may update only the exact local metadata version.
The approved interface is synchronous and accepts exactly one bucket/object/local-version tuple:
```text
GET /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=<bucket>&object=<object>&versionId=<local-version-id>
POST /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=<bucket>&object=<object>&versionId=<local-version-id>
```
`versionId` is required; use the literal `null` for a locally unversioned object. An omitted or empty selector is invalid. GET requires `admin:ListTier`. It reports the authoritative all-pool tuple, destination identity, fleet/topology readiness, live probe classification, opaque expected-tuple digest, and a machine-readable diagnosis. It returns `ready-to-migrate`, not `migrated`, when a missing state is provable because GET is read-only.
POST requires `admin:SetTier`, `confirm: true`, and the complete immutable source tuple, original per-set missing-state representations, proposed target, and reconciliation digest returned by GET. The server rereads every authoritative copy and repeats the bounded live backend probe; provider console output or an operator-supplied state is diagnostic evidence only, never write authority. A retry accepts only copies that still match their digest-bound original representation or already equal the exact proven target; any other divergence is stale or corrupt. The server may persist only one of these exact state/version pairs, together with the bound destination identity:
| Proven remote model | State | Version value |
|---|---|---|
| Versioning disabled | `KnownDisabled` | Empty/absent; later requests omit `versionId` |
| Versioning suspended null object | `SuspendedNull` | Literal `null` |
| One exact version | `Exact` | Exact nonempty, non-`null` opaque identifier |
The response outcome is `migrated`, `retained-ambiguous`, `corrupt`, or `backend-unavailable`. `migrated` means strong all-pool readback proved the same state and destination identity on every authoritative copy; it can be idempotent with `changed=false`. Ambiguous/missing/multiple probe results are retained, and explicit `Unknown`, malformed or conflicting dual keys, nil identifiers, partial tuples, or copies outside the exact `{original missing representation, proven target}` retry subset fail closed. An unavailable backend, tier generation, metadata quorum, or required strong readback reports `backend-unavailable`; a monotonic partial write is retained for retry and never rolled back.
The POST does not issue remote DELETE or PUT, remove local data, create a free-version, clean a transaction/journal, or change tier configuration. It holds the approved fleet/topology, bucket-lifecycle, exact tier-generation/destination, and stable all-pool object-version fences across authoritative reread and the bounded probe; it rechecks them before quorum writes and after strong readback. A fleet containing a node that cannot preserve the explicit state/destination binding is inspect-only, and a cross-pool first match is never enough.
There is intentionally no bucket, prefix, or fleet selector. Batch repair requires a separate durable, resumable job protocol and remains future work. Until the single-record route is implemented, retain affected metadata, use external inspection only for diagnosis, and never hand-edit `xl.meta` or enable remote cleanup by assuming that an empty version field means an unversioned tier.
The full approved fence, quorum, cross-set retry, destination-binding, and mixed-version contract is specified in [../architecture/ilm-tiering-persistence-contracts.md](../architecture/ilm-tiering-persistence-contracts.md#legacy-transitioned-version-state-reconciliation).
## Invariant: local-first expiry ordering
`expire_transitioned_object` deletes local metadata first (making the object unreachable) and leaves a persisted free-version for remote-tier cleanup. `tier_free_version_recovery.rs` scans and re-enqueues that record; the lifecycle worker's `cleanup_free_version_exact` in `bucket_lifecycle_ops.rs` performs the fenced remote delete, local-marker cleanup, and rescan. Never remove a remote tier version while live local metadata still points at it: doing so lets a concurrent GET read a stored version_id whose remote version is already gone and fail with `NoSuchVersion`.
rustfs/src/admin/handlers/tier.rs
+121 -69
View File
@@ -80,6 +80,82 @@ fn wasabi_payload_name(config: &TierConfig) -> S3Result<String> {
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Wasabi configuration"))
}
fn normalize_add_tier_payload_name(config: &mut TierConfig) -> S3Result<()> {
match config.tier_type {
TierType::S3 => {
let _ = config
.s3
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?;
}
TierType::Wasabi => config.name = wasabi_payload_name(config)?,
TierType::RustFS => {
config.name = config
.rustfs
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))?
.name
.clone();
}
TierType::MinIO => {
config.name = config
.minio
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))?
.name
.clone();
}
TierType::Aliyun => {
config.name = config
.aliyun
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))?
.name
.clone();
}
TierType::Tencent => {
config.name = config
.tencent
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))?
.name
.clone();
}
TierType::Huaweicloud => {
config.name = config
.huaweicloud
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))?
.name
.clone();
}
TierType::Azure => {
config.name = config
.azure
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))?
.name
.clone();
}
TierType::GCS => {
let _ = config
.gcs
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))?;
}
TierType::R2 => {
config.name = config
.r2
.as_ref()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))?
.name
.clone();
}
TierType::Unsupported => {}
}
Ok(())
}
fn spawn_transition_tier_config_propagation(action: &'static str) {
if let Some(notification_sys) = current_notification_system() {
debug!(
@@ -263,75 +339,7 @@ impl Operation for AddTier {
let mut args: TierConfig = serde_json::from_slice(&body)
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;
match args.tier_type {
TierType::S3 => {
args.name = args
.s3
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?
.name;
}
TierType::Wasabi => {
args.name = wasabi_payload_name(&args)?;
}
TierType::RustFS => {
args.name = args
.rustfs
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))?
.name;
}
TierType::MinIO => {
args.name = args
.minio
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))?
.name;
}
TierType::Aliyun => {
args.name = args
.aliyun
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))?
.name;
}
TierType::Tencent => {
args.name = args
.tencent
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))?
.name;
}
TierType::Huaweicloud => {
args.name = args
.huaweicloud
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))?
.name;
}
TierType::Azure => {
args.name = args
.azure
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))?
.name;
}
TierType::GCS => {
args.name = args
.gcs
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))?
.name;
}
TierType::R2 => {
args.name = args
.r2
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))?
.name;
}
_ => (),
}
normalize_add_tier_payload_name(&mut args)?;
debug!(
event = EVENT_ADMIN_TIER_STATE,
component = LOG_COMPONENT_ADMIN,
@@ -1150,6 +1158,44 @@ mod tests {
assert_eq!(err.message(), Some("missing Wasabi configuration"));
}
#[test]
fn add_tier_payload_preserves_canonical_madmin_s3_and_gcs_names() {
for (provider, wire) in [
(
"S3",
serde_json::json!({
"Type": "s3",
"Name": "COLD-S3",
"S3": {
"Endpoint": "https://s3.example.invalid",
"AccessKey": "access",
"SecretKey": "secret",
"Bucket": "archive"
}
}),
),
(
"GCS",
serde_json::json!({
"Type": "gcs",
"Name": "COLD-GCS",
"GCS": {
"Endpoint": "https://storage.googleapis.com",
"Creds": "e30=",
"Bucket": "archive"
}
}),
),
] {
let mut config: TierConfig = serde_json::from_value(wire).expect("canonical madmin payload should decode");
let expected = config.name.clone();
normalize_add_tier_payload_name(&mut config).expect("canonical madmin payload should pass the handler boundary");
assert_eq!(config.name, expected, "{provider} top-level Name must not be cleared");
}
}
#[test]
fn resolve_tier_name_prefers_path_parameter() {
let uri: Uri = "/rustfs/admin/v3/tier/HOT?tier=COLD".parse().expect("uri should parse");
@@ -1756,5 +1802,11 @@ mod tests {
}
assert!(!production.contains("check_key_valid(get_session_token"));
let add_tier = source_block(production, "impl Operation for AddTier");
assert!(
add_tier.contains("normalize_add_tier_payload_name(&mut args)?;"),
"AddTier must preserve canonical top-level provider names through the tested boundary helper"
);
}
}
rustfs/src/app/bucket_list_through.rs
+6 -4
View File
@@ -336,7 +336,7 @@ async fn fetch_source_page(
delimiter: params.delimiter.as_deref(),
// S3 ignores start-after once a continuation token is present, so
// the client's own start-after only applies to the first page.
start_after: token.is_none().then(|| params.start_after_for_query.as_deref()).flatten(),
start_after: token.is_none().then_some(params.start_after_for_query.as_deref()).flatten(),
continuation_token: token,
max_keys: params.max_keys,
},
@@ -363,9 +363,11 @@ async fn fetch_source_page(
SourceListPlan::Folded { common_prefix, .. } => {
let exists = !page.objects.is_empty() || !page.common_prefixes.is_empty();
Ok((
exists
.then(|| vec![SideEntry::Prefix(common_prefix.clone())])
.unwrap_or_default(),
if exists {
vec![SideEntry::Prefix(common_prefix.clone())]
} else {
Vec::new()
},
false,
None,
))
rustfs/src/app/object/mod.rs
+5 -5
View File
@@ -170,12 +170,12 @@ use s3s::dto::{
DeleteObjectsOutput, DeletedObject, ETag, GetObjectAttributesInput, GetObjectAttributesOutput, GetObjectAttributesParts,
GetObjectInput, GetObjectOutput, HeadObjectInput, HeadObjectOutput, MetadataDirective, ObjectAttributes, ObjectLockLegalHold,
ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, ObjectPart, PutObjectInput,
PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreStatus, SSECustomerAlgorithm,
SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput, ServerSideEncryption,
ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader, Timestamp, TimestampFormat,
WebsiteRedirectLocation,
PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreRequestType, RestoreStatus,
SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput,
ServerSideEncryption, ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader,
Timestamp, TimestampFormat, WebsiteRedirectLocation,
};
use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH};
use s3s::header::X_AMZ_RESTORE;
use s3s::stream::{ByteStream, DynByteStream, RemainingLength};
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
rustfs/src/app/object/restore.rs
+166 -57
View File
@@ -165,18 +165,49 @@ impl DefaultObjectUsecase {
validate_table_catalog_object_mutation(&bucket, &object).await?;
let rreq = rreq.ok_or_else(|| {
S3Error::with_message(S3ErrorCode::Custom("ErrValidRestoreObject".into()), "restore request is required")
})?;
// Typed S3 errors on every RestoreObject failure (backlog#2205): a
// `Custom` code serializes as a generic 500, which makes SDK clients
// retry client errors and conflicts alike.
let rreq = rreq.ok_or_else(|| S3Error::with_message(S3ErrorCode::MalformedXML, "restore request is required"))?;
// SELECT-type restore is not supported (backlog#1341). The restore
// path can only write the retrieved bytes back to the source key, so
// honouring a SELECT request overwrote the source object with
// SELECT-only metadata (dropping `x-amz-restore`, user metadata and
// tags on an unversioned bucket, or publishing a bogus latest version
// on a versioned one) while never writing anything to
// `OutputLocation.S3`. Reject before any guard, metadata write or
// fabricated `x-amz-restore-output-path` response header.
if rreq
.type_
.as_ref()
.is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
{
return Err(S3Error::with_message(
S3ErrorCode::NotImplemented,
"SELECT restore requests are not supported.",
));
}
let Some(store) = self.object_store() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
// Validate the request shape before taking any lock or reading the
// object: a malformed request or an illegal `Days` value is a client
// error, and the validator messages are static — they carry no
// backend or credential detail.
if let Err(e) = validate_restore_request(&rreq, store.clone()) {
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!("Restore object validation failed: {e}"),
));
}
let version_id_str = version_id.clone().unwrap_or_default();
let mut opts = post_restore_opts(&version_id_str, &bucket, &object)
.await
.map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrPostRestoreOpts".into()), "restore object failed."))?;
.map_err(ApiError::from)?;
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
// `apply_bucket_generation_guard` deliberately tolerates a missing guard
// (only the S3 access layer installs one), so this must not hard-require
@@ -192,11 +223,7 @@ impl DefaultObjectUsecase {
}
};
// SELECT-type restores skip both the ongoing check and the metadata
// write below, so the accept guard would protect nothing for them —
// they keep the plain (read-locked) accept path.
let is_select = rreq.type_.as_ref().is_some_and(|t| t.as_str() == "SELECT");
let restore_operation_id = (!is_select).then(Uuid::new_v4);
let restore_operation_id = Some(Uuid::new_v4());
let mut restore_worker_guard = if let Some(operation_id) = restore_operation_id {
Some(
store
@@ -210,10 +237,10 @@ impl DefaultObjectUsecase {
// Hold the restore-accept guard across the restore-status read, the
// ongoing/already-restored decision, and the metadata write below, so
// two concurrent (non-SELECT) POST ?restore cannot both observe
// ongoing=false and both start a copy-back (backlog#1304). Reads and
// writes inside this scope run with no_lock; the guard is dropped
// before the copy-back is spawned so it never blocks readers.
// two concurrent POST ?restore cannot both observe ongoing=false and
// both start a copy-back (backlog#1304). Reads and writes inside this
// scope run with no_lock; the guard is dropped before the copy-back is
// spawned so it never blocks readers.
// Contention on the accept guard (e.g. a concurrent accept or an
// in-flight commit on the same object) is transient — answer 503
// SlowDown so SDK clients back off and retry instead of treating it
@@ -222,9 +249,7 @@ impl DefaultObjectUsecase {
if store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)? != restore_bucket_incarnation_id {
return Err(ApiError::from(StorageError::BucketNotFound(bucket.clone())).into());
}
let mut accept_guard = if is_select {
None
} else {
let mut accept_guard = {
let guard = store
.acquire_restore_accept_guard(&bucket, &object)
.await
@@ -233,24 +258,17 @@ impl DefaultObjectUsecase {
Some(guard)
};
let mut obj_info = store
.get_object_info(&bucket, &object, &opts)
.await
.map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "restore object failed."))?;
// A missing key or version must stay NoSuchKey / NoSuchVersion, and an
// authorization or storage failure must keep its own identity, so map
// the storage error instead of flattening it (backlog#2205).
let mut obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?;
// Check if object is in a transitioned state
// Restoring an object that was never transitioned is the S3
// InvalidObjectState case, not an internal error.
if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
return Err(S3Error::with_message(
S3ErrorCode::Custom("ErrInvalidTransitionedState".into()),
"restore object failed.",
));
}
// Validate restore request
if let Err(e) = validate_restore_request(&rreq, store.clone()) {
return Err(S3Error::with_message(
S3ErrorCode::Custom("ErrValidRestoreObject".into()),
format!("Restore object validation failed: {}", e),
S3ErrorCode::InvalidObjectState,
"The operation is not valid for the object's storage class.",
));
}
@@ -260,7 +278,7 @@ impl DefaultObjectUsecase {
// would create an ABBA cycle. If the probe succeeds, reacquire and
// re-read the object before replacing the exact orphan generation.
let mut superseded_worker_guard = None;
if obj_info.restore_ongoing && !is_select {
if obj_info.restore_ongoing {
match classify_ongoing_restore(obj_info.user_defined.as_ref(), OffsetDateTime::now_utc()) {
OngoingRestoreRecovery::ActiveOrUnsafe => {
return Err(S3Error::with_message(
@@ -293,13 +311,11 @@ impl DefaultObjectUsecase {
.map_err(|_| S3Error::with_message(S3ErrorCode::SlowDown, "restore object failed."))?,
);
opts.no_lock = true;
obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(|_| {
S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "restore object failed.")
})?;
obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?;
if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
return Err(S3Error::with_message(
S3ErrorCode::Custom("ErrInvalidTransitionedState".into()),
"restore object failed.",
S3ErrorCode::InvalidObjectState,
"The operation is not valid for the object's storage class.",
));
}
if obj_info.restore_ongoing {
@@ -327,11 +343,11 @@ impl DefaultObjectUsecase {
remove_str(&mut metadata, SUFFIX_RESTORE_OPERATION_ID);
remove_str(&mut metadata, SUFFIX_RESTORE_WORKER_LOCK);
let mut header = HeaderMap::new();
let event_object_info = obj_info.clone();
let obj_info_ = obj_info.clone();
if !is_select {
// Scopes the accept-guarded metadata write: everything below runs
// inside the accept critical section, which is released right after.
{
obj_info.metadata_only = true;
metadata.insert(AMZ_RESTORE_EXPIRY_DAYS.to_string(), rreq.days.unwrap_or(1).to_string());
let request_date = OffsetDateTime::now_utc().format(&Rfc3339).map_err(|e| {
@@ -403,7 +419,7 @@ impl DefaultObjectUsecase {
&restore_dst_opts,
)
.await
.map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrCopyObject".into()), "restore object failed."))?;
.map_err(ApiError::from)?;
rustfs_scanner::record_dirty_usage_bucket(&bucket);
#[cfg(test)]
maybe_pause_after_restore_status_commit(&bucket, &object).await;
@@ -429,17 +445,6 @@ impl DefaultObjectUsecase {
drop(accept_guard);
drop(restore_bucket_lifecycle_guard);
// Handle output location for SELECT requests
if let Some(output_location) = &rreq.output_location
&& let Some(s3) = &output_location.s3
&& !s3.bucket_name.is_empty()
{
let restore_object = Uuid::new_v4().to_string();
if let Ok(header_value) = format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse() {
header.insert(X_AMZ_RESTORE_OUTPUT_PATH, header_value);
}
}
// Spawn restoration task in the background. Pin the copy-back to the
// version the accept resolved and flagged: with a versionless request
// on a versioned bucket, a PUT landing between the accept and the
@@ -499,7 +504,7 @@ impl DefaultObjectUsecase {
restore_output_path: None,
};
helper = helper.object(event_object_info).version_id(version_id_str);
let result = Ok(S3Response::with_headers(output, header));
let result = Ok(S3Response::new(output));
let _ = helper.complete(&result);
result
}
@@ -629,6 +634,28 @@ mod tests {
assert_eq!(classify_ongoing_restore(&conflicting_date, now), OngoingRestoreRecovery::ActiveOrUnsafe);
}
fn restore_request(days: Option<i32>) -> RestoreRequest {
RestoreRequest {
days,
description: None,
glacier_job_parameters: None,
output_location: None,
select_parameters: None,
tier: None,
type_: None,
}
}
fn restore_input(bucket: &str, key: &str, rreq: RestoreRequest) -> RestoreObjectInput {
RestoreObjectInput::builder()
.bucket(bucket.to_string())
.key(key.to_string())
.restore_request(Some(rreq))
.build()
.expect("restore input should build")
}
/// backlog#2205: a missing restore body is a client error, not a 500.
#[tokio::test]
async fn execute_restore_object_rejects_missing_restore_request() {
let input = RestoreObjectInput::builder()
@@ -641,10 +668,92 @@ mod tests {
let usecase = DefaultObjectUsecase::without_context();
let err = usecase.execute_restore_object(req).await.unwrap_err();
match err.code() {
S3ErrorCode::Custom(code) => assert_eq!(code, "ErrValidRestoreObject"),
code => panic!("unexpected error code: {:?}", code),
}
assert_eq!(err.code(), &S3ErrorCode::MalformedXML);
}
/// backlog#1341: a SELECT restore must be rejected outright — the restore
/// path can only write back to the source key, never to
/// `OutputLocation.S3`. Rejection happens before the store is resolved, so
/// an uninitialized usecase still answers NotImplemented rather than the
/// InternalError every request that gets past this point returns.
#[tokio::test]
async fn execute_restore_object_rejects_select_type() {
let mut rreq = restore_request(None);
rreq.type_ = Some(s3s::dto::RestoreRequestType::from_static(s3s::dto::RestoreRequestType::SELECT));
let req = build_request(restore_input("test-bucket", "test-key", rreq), Method::POST);
let usecase = DefaultObjectUsecase::without_context();
let err = usecase.execute_restore_object(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
/// backlog#2205: every RestoreObject failure that reaches storage must
/// keep its typed S3 identity. Before this, a missing key, a malformed
/// version-id, an illegal `Days` and an object that was never transitioned
/// all collapsed into `Custom(...)` codes, which serialize as a retryable
/// HTTP 500.
#[tokio::test]
#[serial_test::serial]
async fn execute_restore_object_maps_failures_to_typed_s3_errors() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let context = crate::app::gating_test_env::shared_gating_ambient().await;
let bucket = format!("restore-typed-errors-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create restore test bucket");
let mut reader = PutObjReader::from_vec(b"never transitioned".to_vec());
store
.put_object(&bucket, "local-object", &mut reader, &ObjectOptions::default())
.await
.expect("put untransitioned test object");
let usecase = DefaultObjectUsecase::with_context(Some(context));
// An illegal `Days` is a client error, rejected before any lock or
// object read.
let err = usecase
.execute_restore_object(build_request(
restore_input(&bucket, "local-object", restore_request(Some(0))),
Method::POST,
))
.await
.expect_err("days=0 must be rejected");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
// A malformed version-id keeps InvalidArgument instead of being
// flattened inside `post_restore_opts`.
let mut input = restore_input(&bucket, "local-object", restore_request(Some(1)));
input.version_id = Some("not-a-uuid".to_string());
let err = usecase
.execute_restore_object(build_request(input, Method::POST))
.await
.expect_err("malformed version-id must be rejected");
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
// A missing key stays NoSuchKey.
let err = usecase
.execute_restore_object(build_request(
restore_input(&bucket, "missing-object", restore_request(Some(1))),
Method::POST,
))
.await
.expect_err("missing key must be rejected");
assert_eq!(err.code(), &S3ErrorCode::NoSuchKey);
// Restoring an object that was never transitioned is the S3
// InvalidObjectState case, not an internal error.
let err = usecase
.execute_restore_object(build_request(
restore_input(&bucket, "local-object", restore_request(Some(1))),
Method::POST,
))
.await
.expect_err("untransitioned object must be rejected");
assert_eq!(err.code(), &S3ErrorCode::InvalidObjectState);
}
#[tokio::test]
rustfs/src/app/storage_api.rs
+2 -1
View File
@@ -368,7 +368,8 @@ pub(crate) mod bucket {
version_id: &str,
bucket: &str,
object: &str,
) -> Result<crate::storage::storage_api::StorageObjectOptions, std::io::Error> {
) -> Result<crate::storage::storage_api::StorageObjectOptions, crate::storage::storage_api::StorageError>
{
crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::post_restore_opts(
version_id, bucket, object,
)
rustfs/src/main.rs
+6 -2
View File
@@ -14,6 +14,8 @@
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
use std::alloc::{GlobalAlloc, Layout};
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
use std::ptr::NonNull;
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
#[derive(Default)]
@@ -35,8 +37,10 @@ unsafe impl GlobalAlloc for MiMallocAllocator {
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
// SAFETY: ptr came from this allocator and layout.size() is the original allocation size.
unsafe { rustfs_mimalloc::MiMalloc::free_csize(ptr, layout.size()) }
// SAFETY: ptr came from this allocator, is non-null by GlobalAlloc's
// dealloc contract, and layout.size() is the original allocation size.
let ptr = unsafe { NonNull::new_unchecked(ptr) };
unsafe { rustfs_mimalloc::MiMalloc::free_csize_nonnull(ptr, layout.size()) }
}
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
rustfs/src/storage/rpc/node_service.rs
+135 -6
View File
@@ -279,6 +279,27 @@ fn scanner_activity_response(
}
}
fn scanner_dirty_usage_snapshot_response(
snapshot: rustfs_scanner::ScannerDirtyUsageSnapshot,
) -> ScannerDirtyUsageSnapshotResponse {
ScannerDirtyUsageSnapshotResponse {
instance_id: rustfs_scanner::scanner_activity_epoch().to_string(),
generation: snapshot.generation,
pending_bucket_count: snapshot.pending_bucket_count,
protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
complete: snapshot.complete,
buckets: snapshot
.buckets
.into_iter()
.map(|bucket| ScannerDirtyUsageBucket {
bucket: bucket.bucket,
generation: bucket.generation,
})
.collect(),
response_proof: Bytes::new(),
}
}
fn scanner_activity_response_v7(
namespace_generation: u64,
topology_digest: [u8; 32],
@@ -2085,6 +2106,42 @@ impl Node for NodeService {
Ok(Response::new(response))
}
async fn scanner_dirty_usage_snapshot(
&self,
request: Request<ScannerDirtyUsageSnapshotRequest>,
) -> Result<Response<ScannerDirtyUsageSnapshotResponse>, Status> {
let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_request_body(request.get_ref())
.map_err(|_| Status::invalid_argument("scanner dirty usage snapshot request is too large to authenticate"))?;
verify_tonic_canonical_body_digest(&request, &canonical)
.map_err(|err| Status::permission_denied(format!("scanner dirty usage snapshot authentication failed: {err}")))?;
if request.get_ref().protocol_version != rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION {
return Err(Status::failed_precondition(format!(
"unsupported scanner dirty usage snapshot request protocol {}",
request.get_ref().protocol_version
)));
}
if request.get_ref().challenge.len() != 16 {
return Err(Status::invalid_argument("scanner dirty usage snapshot challenge must be 16 bytes"));
}
let challenge: [u8; 16] = request
.into_inner()
.challenge
.as_ref()
.try_into()
.map_err(|_| Status::invalid_argument("scanner dirty usage snapshot challenge must be 16 bytes"))?;
let snapshot = rustfs_scanner::scanner_dirty_usage_snapshot(rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES);
if snapshot.generation == u64::MAX {
return Err(Status::resource_exhausted("scanner dirty usage generation is exhausted"));
}
let mut response = scanner_dirty_usage_snapshot_response(snapshot);
let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_response_body(&challenge, &response)
.map_err(|_| Status::internal("scanner dirty usage snapshot response is too large to authenticate"))?;
response.response_proof = sign_tonic_rpc_response_proof(&canonical)
.map_err(|_| Status::unavailable("scanner dirty usage snapshot response authentication is unavailable"))?
.into();
Ok(Response::new(response))
}
async fn acquire_scanner_publication_lease(
&self,
request: Request<ScannerPublicationLeaseRequest>,
@@ -2579,12 +2636,12 @@ mod tests {
LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest,
MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest,
ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest,
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerPublicationLeaseReleaseRequest,
ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest, SignalServiceRequest,
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, StartDecommissionRequest,
StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationFailureClass,
TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest,
VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest,
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest,
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest,
SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest,
TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest,
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest,
heal_control_service_client::HealControlServiceClient,
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
node_service_client::NodeServiceClient,
@@ -5883,6 +5940,13 @@ mod tests {
acknowledge_dirty_usage_generation: 0,
}
);
assert_tampered!(
scanner_dirty_usage_snapshot,
ScannerDirtyUsageSnapshotRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
}
);
assert_tampered!(
acquire_scanner_publication_lease,
ScannerPublicationLeaseRequest {
@@ -6056,6 +6120,71 @@ mod tests {
assert_eq!(unavailable.code(), tonic::Code::Unavailable);
}
#[tokio::test]
async fn test_scanner_dirty_usage_snapshot_requires_body_bound_auth_and_signs_a_consistent_view() {
let _ = rustfs_credentials::set_global_rpc_secret("scanner-dirty-usage-snapshot-test-secret".to_string());
let service = create_test_node_service();
let unsigned = service
.scanner_dirty_usage_snapshot(Request::new(ScannerDirtyUsageSnapshotRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
}))
.await
.expect_err("unsigned scanner dirty usage snapshot requests must fail");
assert_eq!(unsigned.code(), tonic::Code::PermissionDenied);
let mut unsupported = Request::new(ScannerDirtyUsageSnapshotRequest {
challenge: vec![7; 16].into(),
protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION + 1,
});
let unsupported_body = rustfs_protos::canonical_scanner_dirty_usage_snapshot_request_body(unsupported.get_ref())
.expect("scanner dirty usage snapshot request should encode");
set_tonic_canonical_body_digest(&mut unsupported, &unsupported_body).expect("digest metadata should encode");
mark_v2_authenticated(&mut unsupported);
let unsupported = service
.scanner_dirty_usage_snapshot(unsupported)
.await
.expect_err("unsupported scanner dirty usage snapshot protocols must fail closed");
assert_eq!(unsupported.code(), tonic::Code::FailedPrecondition);
let mut malformed = Request::new(ScannerDirtyUsageSnapshotRequest {
challenge: vec![7; 15].into(),
protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
});
let malformed_body = rustfs_protos::canonical_scanner_dirty_usage_snapshot_request_body(malformed.get_ref())
.expect("scanner dirty usage snapshot request should encode");
set_tonic_canonical_body_digest(&mut malformed, &malformed_body).expect("digest metadata should encode");
mark_v2_authenticated(&mut malformed);
let malformed = service
.scanner_dirty_usage_snapshot(malformed)
.await
.expect_err("malformed scanner dirty usage snapshot challenges must fail closed");
assert_eq!(malformed.code(), tonic::Code::InvalidArgument);
let challenge = [7; 16];
let mut signed = Request::new(ScannerDirtyUsageSnapshotRequest {
challenge: challenge.to_vec().into(),
protocol_version: rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
});
let signed_body = rustfs_protos::canonical_scanner_dirty_usage_snapshot_request_body(signed.get_ref())
.expect("scanner dirty usage snapshot request should encode");
set_tonic_canonical_body_digest(&mut signed, &signed_body).expect("digest metadata should encode");
mark_v2_authenticated(&mut signed);
let response = service
.scanner_dirty_usage_snapshot(signed)
.await
.expect("an authenticated scanner dirty usage snapshot request should succeed")
.into_inner();
assert_eq!(response.instance_id, rustfs_scanner::scanner_activity_epoch());
assert_eq!(response.protocol_version, rustfs_scanner::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION);
let bucket_count = u64::try_from(response.buckets.len()).expect("snapshot bucket count should fit in u64");
assert_eq!(response.complete, response.pending_bucket_count == bucket_count);
let canonical = rustfs_protos::canonical_scanner_dirty_usage_snapshot_response_body(&challenge, &response)
.expect("scanner dirty usage snapshot response should encode");
crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
.expect("scanner dirty usage snapshot response proof should verify");
}
#[test]
fn test_scanner_activity_response_uses_process_epoch_and_generations() {
let response = scanner_activity_response_v7(
rustfs/src/version.rs
+6 -162
View File
@@ -14,8 +14,6 @@
use const_str::concat;
use shadow_rs::shadow;
use std::path::Path;
use std::process::Command;
shadow!(build);
@@ -47,10 +45,6 @@ pub const DISPLAY_VERSION: &str = {
type VersionParseResult = Result<(u32, u32, u32, Option<String>), Box<dyn std::error::Error>>;
fn build_version_override() -> Option<&'static str> {
BUILD_VERSION_OVERRIDE.filter(|version| !version.is_empty())
}
fn version_ref(version: &str) -> String {
if version.starts_with("refs/tags/") || version.starts_with('@') {
version.to_string()
@@ -61,91 +55,7 @@ fn version_ref(version: &str) -> String {
#[allow(clippy::const_is_empty)]
pub fn get_version() -> String {
if let Some(version) = build_version_override() {
return version_ref(version);
}
// Get the latest tag
if let Ok(latest_tag) = get_latest_tag() {
// Check if current commit is newer than the latest tag
if is_head_newer_than_tag(&latest_tag) {
// If current commit is newer, increment the version number
if let Ok(new_version) = increment_version(&latest_tag) {
return format!("refs/tags/{new_version}");
}
}
// If current commit is the latest tag, or version increment failed, return current tag
return format!("refs/tags/{latest_tag}");
}
// If no tag exists, use original logic
if !build::TAG.is_empty() {
format!("refs/tags/{}", build::TAG)
} else if !build::SHORT_COMMIT.is_empty() {
format!("@{}", build::SHORT_COMMIT)
} else {
format!("refs/tags/{}", build::PKG_VERSION)
}
}
/// Get the latest git tag
fn get_latest_tag() -> Result<String, Box<dyn std::error::Error>> {
let output = Command::new("git").args(["describe", "--tags", "--abbrev=0"]).output()?;
if output.status.success() {
let tag = String::from_utf8(output.stdout)?;
Ok(tag.trim().to_string())
} else {
Err("Failed to get latest tag".into())
}
}
/// Check if current HEAD is newer than specified tag
fn is_head_newer_than_tag(tag: &str) -> bool {
is_head_newer_than_tag_in(Path::new("."), tag)
}
fn is_head_newer_than_tag_in(repo: &Path, tag: &str) -> bool {
let head = Command::new("git").current_dir(repo).args(["rev-parse", "HEAD"]).output();
let tag_commit = Command::new("git")
.current_dir(repo)
.args(["rev-list", "-n", "1", tag])
.output();
let (Ok(head), Ok(tag_commit)) = (head, tag_commit) else {
return false;
};
if !head.status.success() || !tag_commit.status.success() || head.stdout == tag_commit.stdout {
return false;
}
let output = Command::new("git")
.current_dir(repo)
.args(["merge-base", "--is-ancestor", tag, "HEAD"])
.output();
match output {
Ok(result) => result.status.success(),
Err(_) => false,
}
}
/// Increment version number (increase patch version)
fn increment_version(version: &str) -> Result<String, Box<dyn std::error::Error>> {
// Parse version number, e.g. "1.0.0-alpha.19" -> (1, 0, 0, Some("alpha.19"))
let (major, minor, patch, pre_release) = parse_version(version)?;
// If there's a pre-release identifier, increment the pre-release version number
if let Some(pre) = pre_release
&& let Some(new_pre) = increment_pre_release(&pre)
{
return Ok(format!("{major}.{minor}.{patch}-{new_pre}"));
}
// Otherwise increment patch version number
Ok(format!("{major}.{minor}.{}", patch + 1))
version_ref(DISPLAY_VERSION)
}
/// Parse version number
@@ -166,28 +76,6 @@ pub fn parse_version(version: &str) -> VersionParseResult {
Ok((major, minor, patch, pre_release))
}
/// Increment pre-release version number
fn increment_pre_release(pre_release: &str) -> Option<String> {
// Handle pre-release versions like "alpha.19"
let parts: Vec<&str> = pre_release.split('.').collect();
if parts.len() == 2
&& let Ok(num) = parts[1].parse::<u32>()
{
return Some(format!("{}.{}", parts[0], num + 1));
}
// Handle pre-release versions like "alpha19"
if let Some(pos) = pre_release.rfind(|c: char| c.is_alphabetic()) {
let prefix = &pre_release[..=pos];
let suffix = &pre_release[pos + 1..];
if let Ok(num) = suffix.parse::<u32>() {
return Some(format!("{prefix}{}", num + 1));
}
}
None
}
/// Clean version string - removes common prefixes
pub fn clean_version(version: &str) -> String {
version
@@ -284,34 +172,6 @@ mod tests {
use super::*;
use tracing::debug;
fn run_git(repo: &Path, args: &[&str]) {
let status = Command::new("git").current_dir(repo).args(args).status().unwrap();
assert!(status.success(), "git command failed: git {}", args.join(" "));
}
#[test]
fn test_is_head_newer_than_tag_requires_strict_descendant() {
let repo = tempfile::tempdir().unwrap();
run_git(repo.path(), &["init", "--quiet"]);
run_git(repo.path(), &["config", "user.name", "RustFS Tests"]);
run_git(repo.path(), &["config", "user.email", "rustfs@example.com"]);
run_git(repo.path(), &["commit", "--allow-empty", "--quiet", "-m", "tagged commit"]);
run_git(repo.path(), &["tag", "--annotate", "1.2.3", "--message", "1.2.3"]);
assert!(!is_head_newer_than_tag_in(repo.path(), "1.2.3"));
run_git(repo.path(), &["commit", "--allow-empty", "--quiet", "-m", "newer commit"]);
assert!(is_head_newer_than_tag_in(repo.path(), "1.2.3"));
}
#[test]
fn build_version_override_is_used_for_current_version_when_set() {
if let Some(version) = build_version_override() {
assert_eq!(get_version(), version_ref(version));
}
}
#[test]
fn version_ref_keeps_existing_ref_prefixes() {
assert_eq!(version_ref("1.2.3"), "refs/tags/1.2.3");
@@ -319,6 +179,11 @@ mod tests {
assert_eq!(version_ref("@abc123"), "@abc123");
}
#[test]
fn get_version_uses_build_metadata() {
assert_eq!(get_version(), version_ref(DISPLAY_VERSION));
}
#[test]
fn test_parse_version() {
// Test standard version parsing
@@ -336,27 +201,6 @@ mod tests {
assert_eq!(pre_release, Some("alpha.19".to_string()));
}
#[test]
fn test_increment_pre_release() {
// Test alpha.19 -> alpha.20
assert_eq!(increment_pre_release("alpha.19"), Some("alpha.20".to_string()));
// Test beta.5 -> beta.6
assert_eq!(increment_pre_release("beta.5"), Some("beta.6".to_string()));
// Test unparsable case
assert_eq!(increment_pre_release("unknown"), None);
}
#[test]
fn test_increment_version() {
// Test pre-release version increment
assert_eq!(increment_version("1.0.0-alpha.19").unwrap(), "1.0.0-alpha.20");
// Test standard version increment
assert_eq!(increment_version("1.0.0").unwrap(), "1.0.1");
}
#[test]
fn test_version_format() {
// Test if version format starts with refs/tags/