mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-29 09:38:59 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 唐小鸭
|
2222b68a81 | ||
|
唐小鸭
|
2aaac85160 |
@@ -141,7 +141,7 @@ jobs:
|
||||
name: Test and Lint
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
|
||||
Generated
+29
-31
@@ -1703,9 +1703,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "camino"
|
||||
version = "1.2.5"
|
||||
version = "1.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb1307f12aa967b5a58416e87b3653360e0fd614a016b6e970db08fecbb1b80d"
|
||||
checksum = "5f2d30e4173c4026932d51d31d6b0613b1fd3014bf3f9f8943d4ba139c437ba0"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
@@ -3625,13 +3625,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.7"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
|
||||
checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3941,15 +3941,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "event-listener"
|
||||
version = "5.4.2"
|
||||
version = "5.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2"
|
||||
checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab"
|
||||
dependencies = [
|
||||
"concurrent-queue",
|
||||
"parking",
|
||||
"pin-project-lite",
|
||||
]
|
||||
@@ -5360,7 +5361,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6109,9 +6110,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique"
|
||||
version = "0.1.29"
|
||||
version = "0.1.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2e394c63e2d1a30aeb3b9392ecf3439d8475d2df810a8f4f6e66d6866754017"
|
||||
checksum = "aa466af30a9fe0b1db1dae097e0ce7ddac4b666b1d3a9f5c682e889272511dd8"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"jiff",
|
||||
@@ -6139,9 +6140,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique-macro"
|
||||
version = "0.1.20"
|
||||
version = "0.1.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "786df1fd0abebd0db685f7e9a353c78756d4b370fb98a52376c2015fa55f141f"
|
||||
checksum = "1f5febfaf14fea234b60e0ad43c728db274033893a38d0fa1f87d9b7056d3d61"
|
||||
dependencies = [
|
||||
"Inflector",
|
||||
"darling 0.23.0",
|
||||
@@ -6168,9 +6169,9 @@ checksum = "2faca4e4480069ff02b1763b3b79f5cec7e8628e24d9dc5b6073f53d2577a4d9"
|
||||
|
||||
[[package]]
|
||||
name = "metrique-writer"
|
||||
version = "0.1.25"
|
||||
version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "82cdde44d241dab7fc8b7a32e0eb5dae6cd28f8de80b59f9a1e9f2f0b05e485e"
|
||||
checksum = "124326a2ac4c4f61562fa4d071735a1c463f9ba0317d1564f75dc01313dc12d7"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"crossbeam-queue",
|
||||
@@ -6189,9 +6190,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "metrique-writer-core"
|
||||
version = "0.1.19"
|
||||
version = "0.1.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e57379b7ee2272efaeaaa6de062503563e57333b24aadc7f2255b3d602899e8b"
|
||||
checksum = "55b5bbb6d88bde29f6ed74a574cbe49e51ea0c36cccc7e63eee2040db5675b15"
|
||||
dependencies = [
|
||||
"derive-where",
|
||||
"itertools 0.14.0",
|
||||
@@ -8096,7 +8097,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9105,7 +9106,6 @@ dependencies = [
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
"async-recursion",
|
||||
@@ -9121,7 +9121,6 @@ dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"bytesize",
|
||||
"chacha20poly1305",
|
||||
"chrono",
|
||||
"criterion",
|
||||
"enumset",
|
||||
@@ -9174,7 +9173,6 @@ dependencies = [
|
||||
"rustfs-erasure-codec",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-kms",
|
||||
"rustfs-lifecycle",
|
||||
"rustfs-lock",
|
||||
"rustfs-madmin",
|
||||
@@ -10224,7 +10222,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10297,7 +10295,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10460,9 +10458,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "1.2.2"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
|
||||
checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc"
|
||||
dependencies = [
|
||||
"dyn-clone",
|
||||
"ref-cast",
|
||||
@@ -10695,7 +10693,7 @@ dependencies = [
|
||||
"indexmap 1.9.3",
|
||||
"indexmap 2.14.0",
|
||||
"schemars 0.9.0",
|
||||
"schemars 1.2.2",
|
||||
"schemars 1.2.1",
|
||||
"serde_core",
|
||||
"serde_json",
|
||||
"serde_with_macros",
|
||||
@@ -11497,10 +11495,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11875,9 +11873,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml_parser"
|
||||
version = "1.1.3+spec-1.1.0"
|
||||
version = "1.1.2+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
|
||||
checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
|
||||
dependencies = [
|
||||
"winnow",
|
||||
]
|
||||
@@ -12601,7 +12599,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -116,27 +116,6 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
|
||||
/// Default: bitrot verification is enabled on GetObject reads (do not skip).
|
||||
pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
|
||||
|
||||
/// Request writing the complete remote-tier version state into object metadata.
|
||||
///
|
||||
/// This remains ineffective until
|
||||
/// [`ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED`] is also enabled.
|
||||
pub const ENV_TIER_REMOTE_VERSION_STATE_WRITE: &str = "RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE";
|
||||
pub const DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE: bool = false;
|
||||
|
||||
/// Operator-attested fleet-wide confirmation for
|
||||
/// [`ENV_TIER_REMOTE_VERSION_STATE_WRITE`].
|
||||
///
|
||||
/// This flag is an operational contract, not automatic capability discovery.
|
||||
/// Operators may enable it only after every node that can write or read
|
||||
/// transitioned object metadata supports the remote version-state schema and
|
||||
/// semantics. Keeping the confirmation separate makes a single-node request or
|
||||
/// a writer whose local opt-in is removed fail closed.
|
||||
pub const ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: &str = "RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
|
||||
|
||||
// =============================================================================
|
||||
// Concurrent Request Fix - Timeout and Backpressure Configuration
|
||||
// =============================================================================
|
||||
@@ -638,15 +617,3 @@ pub const ENV_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: &str = "RUSTFS_OBJ
|
||||
|
||||
/// Default read-ahead disable concurrency threshold: 4.
|
||||
pub const DEFAULT_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY: usize = 4;
|
||||
|
||||
#[cfg(test)]
|
||||
mod remote_version_state_tests {
|
||||
#[test]
|
||||
fn remote_version_state_gate_uses_stable_environment_names() {
|
||||
assert_eq!(super::ENV_TIER_REMOTE_VERSION_STATE_WRITE, "RUSTFS_TIER_REMOTE_VERSION_STATE_WRITE");
|
||||
assert_eq!(
|
||||
super::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
"RUSTFS_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,10 +45,10 @@
|
||||
//! * Parity reconstruction: one data disk is taken offline
|
||||
//! (`take_disk_offline`) and the SAME object matrix is GET both ways while
|
||||
//! the EC 2+2 set rebuilds each large object from the surviving shards. The
|
||||
//! eager first/single-part setup may keep its conservative whole-request
|
||||
//! fallback when shard placement makes codec streaming unsafe, so this phase
|
||||
//! asserts byte- and header-equality vs the legacy path rather than requiring
|
||||
//! zero duplex fallbacks under degraded drive health.
|
||||
//! codec-streaming reader gate never inspects drive health, so the codec
|
||||
//! fast path is exercised end-to-end through reconstruction; the test
|
||||
//! asserts byte- and header-equality vs the legacy path AND that the codec
|
||||
//! phase never fell back to a duplex pipe while reconstructing.
|
||||
//! * Missing object: a GET for an absent key is compared across both phases
|
||||
//! to prove the error semantics (HTTP status + S3 error code) are identical
|
||||
//! — the codec env must not perturb the NoSuchKey negative path.
|
||||
@@ -475,14 +475,15 @@ mod tests {
|
||||
"ranged GET length diverged with codec streaming enabled"
|
||||
);
|
||||
|
||||
// ---- Phase B degraded: the same reconstruction, with codec gates open ----
|
||||
// Re-run the reconstruction A/B with codec-streaming enabled. If eager
|
||||
// first/single-part setup cannot prove the codec path is safe for the
|
||||
// surviving shards, the implementation intentionally preserves the
|
||||
// whole-request legacy fallback; later multipart parts can degrade in
|
||||
// place. This phase verifies parity-reconstructed bytes and headers,
|
||||
// while the healthy phase above remains the strict zero-duplex path
|
||||
// confirmation.
|
||||
// ---- Phase B degraded: the same reconstruction, now on the codec path ----
|
||||
// Re-run the reconstruction A/B with the codec-streaming gates still
|
||||
// open. The reader gate decision is independent of drive health (it
|
||||
// never inspects disk state), so the codec fast path is exercised
|
||||
// end-to-end while the EC set rebuilds each large object from the
|
||||
// surviving shards — this is a real codec-vs-legacy reconstruction test,
|
||||
// not legacy-vs-legacy. Snapshot the duplex count first (the range GET
|
||||
// above already used the duplex path) so we can measure only the markers
|
||||
// these degraded codec GETs add.
|
||||
let dup_codec_before_degraded = count_marker(&codec_log, DUPLEX_MARKER);
|
||||
harness.take_disk_offline(0)?;
|
||||
let mut codec_degraded: BTreeMap<String, GetView> = BTreeMap::new();
|
||||
@@ -510,11 +511,16 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
// Keep degraded duplex markers as diagnostic evidence only: eager setup
|
||||
// may fall back before streaming when shard safety cannot be proven.
|
||||
// Path confirmation under reconstruction: the codec fast path must have
|
||||
// served the reconstructed large objects without ever falling back to
|
||||
// the legacy duplex pipe. Without this, the equivalence above could be
|
||||
// legacy-vs-legacy and prove nothing about codec reconstruction.
|
||||
sleep(Duration::from_millis(300)).await;
|
||||
let dup_codec_degraded = count_marker(&codec_log, DUPLEX_MARKER).saturating_sub(dup_codec_before_degraded);
|
||||
info!(dup_codec_degraded, "codec phase degraded-read legacy duplex marker count");
|
||||
assert_eq!(
|
||||
dup_codec_degraded, 0,
|
||||
"codec phase created {dup_codec_degraded} duplex pipe(s) while reconstructing large objects with disk0 offline; the codec fast path was not exercised under degraded reads (see {codec_log})"
|
||||
);
|
||||
|
||||
info!(
|
||||
objects = baseline.len(),
|
||||
|
||||
@@ -57,7 +57,6 @@ rustfs-policy.workspace = true
|
||||
rustfs-protos.workspace = true
|
||||
rustfs-replication.workspace = true
|
||||
rustfs-lifecycle.workspace = true
|
||||
rustfs-kms.workspace = true
|
||||
rustfs-s3-types = { workspace = true }
|
||||
rustfs-data-usage.workspace = true
|
||||
rustfs-object-capacity.workspace = true
|
||||
@@ -124,8 +123,6 @@ libc.workspace = true
|
||||
rustix = { workspace = true, features = ["process", "fs"] }
|
||||
rustfs-madmin.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
aes-gcm = { workspace = true, features = ["rand_core"] }
|
||||
chacha20poly1305.workspace = true
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
urlencoding = { workspace = true }
|
||||
smallvec = { workspace = true, features = ["serde"] }
|
||||
|
||||
@@ -67,14 +67,6 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod transition_transaction {
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod evaluator {
|
||||
pub use crate::bucket::lifecycle::evaluator::Evaluator;
|
||||
}
|
||||
@@ -384,16 +376,17 @@ pub mod metrics {
|
||||
pub mod notification {
|
||||
pub use crate::services::notification_sys::{
|
||||
NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
|
||||
start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod object {
|
||||
pub use crate::object_api::{
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource,
|
||||
GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, StreamConsumer,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
|
||||
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, ObjectEncryptionResolver, ObjectInfo,
|
||||
ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode,
|
||||
ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
|
||||
register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
|
||||
unregister_object_mutation_hook,
|
||||
};
|
||||
pub use crate::store::PreparedGetObjectReader;
|
||||
}
|
||||
|
||||
@@ -554,7 +554,6 @@ async fn delete_free_version_remote_object(
|
||||
oi: &ObjectInfo,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let version_id_exact = validate_transition_remote_version(oi)?;
|
||||
let identity = tier_destination_id_from_metadata(&oi.user_defined)?
|
||||
.ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
@@ -563,7 +562,7 @@ async fn delete_free_version_remote_object(
|
||||
&oi.transitioned_object.tier,
|
||||
identity,
|
||||
tier_config_mgr,
|
||||
version_id_exact,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
@@ -4202,23 +4201,6 @@ pub async fn get_transitioned_object_reader(
|
||||
get_transitioned_object_reader_with_tier_manager(bucket, object, rs, h, oi, opts, &tier_config_mgr).await
|
||||
}
|
||||
|
||||
fn validate_transition_remote_version(oi: &ObjectInfo) -> Result<bool, std::io::Error> {
|
||||
let version = oi.transitioned_object.version_id.as_str();
|
||||
match oi.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object version state is unknown",
|
||||
)),
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled if version.is_empty() => Ok(false),
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull if version == "null" => Ok(true),
|
||||
rustfs_filemeta::TransitionVersionState::Exact if !version.is_empty() && version != "null" => Ok(true),
|
||||
_ => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object version state conflicts with its version ID",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -4228,7 +4210,6 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
opts: &ObjectOptions,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
) -> Result<GetObjectReader, std::io::Error> {
|
||||
validate_transition_remote_version(oi)?;
|
||||
let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?;
|
||||
let lease = match expected_identity {
|
||||
Some(identity) => {
|
||||
@@ -5525,7 +5506,6 @@ mod tests {
|
||||
tier: tier.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -5589,7 +5569,6 @@ mod tests {
|
||||
tier,
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -5612,70 +5591,6 @@ mod tests {
|
||||
assert_eq!(backend.get_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_rejects_unknown_version_state_before_backend_io() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 1,
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: String::new(),
|
||||
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
tier,
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = match get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("unknown remote version state must fail before backend IO"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.get_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_delete_rejects_unknown_version_state_before_backend_io() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let backend = register_mock_tier(&manager, "WARM").await;
|
||||
let object_info = ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: "legacy-version".to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect_err("unknown remote version state must fail before backend IO");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_remote_delete_requires_persisted_destination_identity() {
|
||||
@@ -5725,7 +5640,6 @@ mod tests {
|
||||
oi.transitioned_object.tier = "WARM".to_string();
|
||||
oi.transitioned_object.name = "remote/object".to_string();
|
||||
oi.transitioned_object.version_id = "remote-version".to_string();
|
||||
oi.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
|
||||
let local_delete_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
|
||||
let legacy_err = delete_free_version_remote_object_then(&oi, &manager, {
|
||||
@@ -5736,8 +5650,7 @@ mod tests {
|
||||
})
|
||||
.await
|
||||
.expect_err("legacy free-version without identity must be retained");
|
||||
assert_eq!(legacy_err.kind(), std::io::ErrorKind::Other);
|
||||
assert_eq!(old_backend.remove_count().await, 0);
|
||||
assert!(legacy_err.to_string().contains("no durable backend identity"));
|
||||
assert_eq!(local_delete_calls.load(Ordering::Relaxed), 0);
|
||||
|
||||
let mut invalid_metadata = HashMap::new();
|
||||
@@ -5868,7 +5781,6 @@ mod tests {
|
||||
oi.transitioned_object.tier = "WARM".to_string();
|
||||
oi.transitioned_object.name = "remote/object".to_string();
|
||||
oi.transitioned_object.version_id = "remote-version".to_string();
|
||||
oi.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
|
||||
|
||||
let err = match get_transitioned_object_reader_with_tier_manager(
|
||||
"bucket",
|
||||
@@ -5884,12 +5796,7 @@ mod tests {
|
||||
Ok(_) => panic!("identity-bound GET must reject a same-name tier rebind"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::Other);
|
||||
let admin_err = err
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<crate::client::admin_handler_utils::AdminError>())
|
||||
.expect("identity mismatch should retain the typed tier error");
|
||||
assert_eq!(admin_err.code, crate::services::tier::tier::ERR_TIER_INVALID_CONFIG.code);
|
||||
assert!(err.to_string().contains("identity no longer matches"));
|
||||
assert_eq!(new_backend.get_count().await, 0);
|
||||
|
||||
oi.user_defined = Arc::new(HashMap::new());
|
||||
@@ -5995,8 +5902,7 @@ mod tests {
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some([1; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
|
||||
let err = state
|
||||
@@ -6107,8 +6013,7 @@ mod tests {
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some([1; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
|
||||
state
|
||||
@@ -10176,87 +10081,6 @@ mod tests {
|
||||
(backend, identity_hex)
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn journal_replay_rejects_unknown_version_state_before_backend_io() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
|
||||
let identity = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available")
|
||||
.backend_identity();
|
||||
let je = Jentry {
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "legacy-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some(identity),
|
||||
version_id_exact: false,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
};
|
||||
|
||||
let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
|
||||
.await
|
||||
.expect_err("unknown journal state must fail before backend IO");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn journal_replay_deletes_confirmed_exact_provider_token() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available");
|
||||
let identity = lease.backend_identity();
|
||||
backend
|
||||
.set_put_remote_version(Some("provider-version-token".to_string()))
|
||||
.await;
|
||||
lease
|
||||
.put(
|
||||
"remote/object",
|
||||
crate::client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"candidate")),
|
||||
9,
|
||||
)
|
||||
.await
|
||||
.expect("confirmed remote candidate should be seeded");
|
||||
backend.set_remove_failure(true);
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
let je = Jentry {
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "provider-version-token".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some(identity),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
};
|
||||
|
||||
crate::set_disk::cleanup_rejected_transition_upload_durably(
|
||||
&lease,
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
true,
|
||||
Some(ecstore.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("failed immediate cleanup should remain durable in the journal");
|
||||
assert!(backend.contains(&je.obj_name).await);
|
||||
|
||||
backend.set_remove_failure(false);
|
||||
crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
|
||||
.await
|
||||
.expect("identity-bound exact journal must retry confirmed candidate cleanup");
|
||||
|
||||
assert!(!backend.contains(&je.obj_name).await);
|
||||
assert_eq!(backend.exact_remove_count(), 2);
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![("remote/object".to_string(), "provider-version-token".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
async fn seed_recoverable_free_version(
|
||||
disk_paths: &[PathBuf],
|
||||
bucket: &str,
|
||||
@@ -10276,7 +10100,6 @@ mod tests {
|
||||
identity,
|
||||
);
|
||||
}
|
||||
let transition_version_id = Uuid::new_v4();
|
||||
let mut metadata = FileMeta::new();
|
||||
metadata
|
||||
.add_version(FileInfo {
|
||||
@@ -10285,9 +10108,7 @@ mod tests {
|
||||
version_id: Some(object_version_id),
|
||||
transition_status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
transitioned_objname: format!("remote/{bucket}/{object}"),
|
||||
transition_version_id: Some(transition_version_id),
|
||||
transition_version: Some(transition_version_id.to_string()),
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
transition_version_id: Some(Uuid::new_v4()),
|
||||
transition_tier: "WARM".to_string(),
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
metadata: transitioned_metadata,
|
||||
|
||||
@@ -20,10 +20,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
Jentry, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
@@ -45,7 +42,6 @@ const TIER_DELETE_JOURNAL_RECOVERY_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
|
||||
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
|
||||
const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
|
||||
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -59,35 +55,24 @@ struct PersistedTierDeleteJournalEntry {
|
||||
backend_identity: Option<[u8; 32]>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
version_id_exact: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
version_state: Option<rustfs_filemeta::TransitionVersionState>,
|
||||
}
|
||||
|
||||
impl PersistedTierDeleteJournalEntry {
|
||||
fn from_jentry(je: &Jentry) -> Result<Self> {
|
||||
validate_version_state(je.version_state, &je.version_id, je.version_id_exact)?;
|
||||
let legacy_unknown = je.version_state == rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
let version = if legacy_unknown {
|
||||
if je.backend_identity.is_some() {
|
||||
fn from_jentry(je: &Jentry) -> Self {
|
||||
Self {
|
||||
version: if je.version_id_exact {
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION
|
||||
} else if je.backend_identity.is_some() {
|
||||
TIER_DELETE_JOURNAL_VERSION
|
||||
} else {
|
||||
1
|
||||
}
|
||||
} else {
|
||||
if je.backend_identity.is_none() {
|
||||
return Err(Error::other("new tier delete journal entry is missing its backend identity"));
|
||||
}
|
||||
TIER_DELETE_JOURNAL_STATE_VERSION
|
||||
};
|
||||
Ok(Self {
|
||||
version,
|
||||
},
|
||||
obj_name: je.obj_name.clone(),
|
||||
version_id: je.version_id.clone(),
|
||||
tier_name: je.tier_name.clone(),
|
||||
backend_identity: je.backend_identity,
|
||||
version_id_exact: je.version_id_exact.then_some(true),
|
||||
version_state: (!legacy_unknown).then_some(je.version_state),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn into_jentry(self) -> Result<Jentry> {
|
||||
@@ -99,23 +84,19 @@ impl PersistedTierDeleteJournalEntry {
|
||||
if self.obj_name.is_empty() || self.tier_name.is_empty() {
|
||||
return Err(Error::other("tier delete journal entry is incomplete"));
|
||||
}
|
||||
if self.version != TIER_DELETE_JOURNAL_EXACT_VERSION
|
||||
&& self.version != TIER_DELETE_JOURNAL_STATE_VERSION
|
||||
&& self.version_id_exact.unwrap_or(false)
|
||||
{
|
||||
if self.version != TIER_DELETE_JOURNAL_EXACT_VERSION && self.version_id_exact.unwrap_or(false) {
|
||||
return Err(Error::other(
|
||||
"legacy tier delete journal entry has an unsupported exact version constraint",
|
||||
));
|
||||
}
|
||||
let (backend_identity, version_id_exact, version_state) = match self.version {
|
||||
1 => (None, false, rustfs_filemeta::TransitionVersionState::Unknown),
|
||||
let (backend_identity, version_id_exact) = match self.version {
|
||||
1 => (None, false),
|
||||
TIER_DELETE_JOURNAL_VERSION => (
|
||||
Some(
|
||||
self.backend_identity
|
||||
.ok_or_else(|| Error::other("tier delete journal v2 entry is missing its backend identity"))?,
|
||||
),
|
||||
false,
|
||||
rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
),
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION => {
|
||||
if self.version_id.is_empty() || self.version_id_exact != Some(true) {
|
||||
@@ -127,22 +108,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
.ok_or_else(|| Error::other("tier delete journal v3 entry is missing its backend identity"))?,
|
||||
),
|
||||
true,
|
||||
rustfs_filemeta::TransitionVersionState::Exact,
|
||||
)
|
||||
}
|
||||
TIER_DELETE_JOURNAL_STATE_VERSION => {
|
||||
let state = self
|
||||
.version_state
|
||||
.ok_or_else(|| Error::other("tier delete journal v4 entry is missing its version state"))?;
|
||||
let exact = self.version_id_exact.unwrap_or(false);
|
||||
validate_version_state(state, &self.version_id, exact)?;
|
||||
(
|
||||
Some(
|
||||
self.backend_identity
|
||||
.ok_or_else(|| Error::other("tier delete journal v4 entry is missing its backend identity"))?,
|
||||
),
|
||||
exact,
|
||||
state,
|
||||
)
|
||||
}
|
||||
version => return Err(Error::other(format!("unsupported tier delete journal version {version}"))),
|
||||
@@ -153,30 +118,10 @@ impl PersistedTierDeleteJournalEntry {
|
||||
tier_name: self.tier_name,
|
||||
backend_identity,
|
||||
version_id_exact,
|
||||
version_state,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_version_state(
|
||||
state: rustfs_filemeta::TransitionVersionState,
|
||||
version_id: &str,
|
||||
version_id_exact: bool,
|
||||
) -> Result<()> {
|
||||
use rustfs_filemeta::TransitionVersionState::{Exact, KnownDisabled, SuspendedNull, Unknown};
|
||||
|
||||
let valid = match state {
|
||||
Unknown => !version_id_exact,
|
||||
KnownDisabled => version_id.is_empty() && !version_id_exact,
|
||||
SuspendedNull => version_id == "null" && version_id_exact,
|
||||
Exact => !version_id.is_empty() && version_id != "null" && version_id_exact,
|
||||
};
|
||||
if !valid {
|
||||
return Err(Error::other("tier delete journal version state conflicts with its version id"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TierDeleteJournalRecoveryStats {
|
||||
pub scanned: usize,
|
||||
@@ -214,7 +159,7 @@ pub(crate) fn decode_tier_delete_journal_entry(data: &[u8]) -> Result<Jentry> {
|
||||
}
|
||||
|
||||
pub(crate) fn encode_tier_delete_journal_entry(je: &Jentry) -> Result<Vec<u8>> {
|
||||
serde_json::to_vec(&PersistedTierDeleteJournalEntry::from_jentry(je)?)
|
||||
serde_json::to_vec(&PersistedTierDeleteJournalEntry::from_jentry(je))
|
||||
.map_err(|err| Error::other(format!("encode tier delete journal failed: {err}")))
|
||||
}
|
||||
|
||||
@@ -264,35 +209,18 @@ where
|
||||
}
|
||||
|
||||
pub async fn process_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
if je.version_state == rustfs_filemeta::TransitionVersionState::Unknown {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"tier delete journal remote version state is unknown",
|
||||
));
|
||||
}
|
||||
let backend_identity = je
|
||||
.backend_identity
|
||||
.ok_or_else(|| std::io::Error::other("legacy tier delete journal has no durable backend identity"))?;
|
||||
if je.version_id_exact {
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
je.version_id_exact,
|
||||
)
|
||||
.await?;
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
@@ -478,9 +406,8 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, await_tier_delete_journal_recovery,
|
||||
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
|
||||
tier_delete_journal_object_name,
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, await_tier_delete_journal_recovery, decode_tier_delete_journal_entry,
|
||||
encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity, tier_delete_journal_object_name,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::Jentry;
|
||||
use crate::error::Result;
|
||||
@@ -493,8 +420,7 @@ mod tests {
|
||||
version_id: "remote-version".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some([7; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -510,7 +436,6 @@ mod tests {
|
||||
assert_eq!(decoded.tier_name, je.tier_name);
|
||||
assert_eq!(decoded.backend_identity, je.backend_identity);
|
||||
assert_eq!(decoded.version_id_exact, je.version_id_exact);
|
||||
assert_eq!(decoded.version_state, je.version_state);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -525,7 +450,7 @@ mod tests {
|
||||
let persisted: serde_json::Value = serde_json::from_slice(&encoded).expect("exact journal JSON should decode");
|
||||
let decoded = decode_tier_delete_journal_entry(&encoded).expect("exact journal entry should decode");
|
||||
|
||||
assert_eq!(persisted["version"], TIER_DELETE_JOURNAL_STATE_VERSION);
|
||||
assert_eq!(persisted["version"], TIER_DELETE_JOURNAL_EXACT_VERSION);
|
||||
assert_eq!(persisted["version_id_exact"], true);
|
||||
assert!(decoded.version_id_exact);
|
||||
assert_ne!(tier_delete_journal_object_name(&exact), tier_delete_journal_object_name(&normalized));
|
||||
@@ -588,46 +513,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_rejects_conflicting_v4_version_states() {
|
||||
let identity = vec![7_u8; 32];
|
||||
let invalid = [
|
||||
("known-disabled", "unexpected", false),
|
||||
("suspended-null", "", true),
|
||||
("suspended-null", "null", false),
|
||||
("exact", "", true),
|
||||
("exact", "null", true),
|
||||
("exact", "version", false),
|
||||
("unknown", "version", true),
|
||||
];
|
||||
|
||||
for (state, version_id, exact) in invalid {
|
||||
let persisted = serde_json::json!({
|
||||
"version": TIER_DELETE_JOURNAL_STATE_VERSION,
|
||||
"obj_name": "remote/object",
|
||||
"version_id": version_id,
|
||||
"tier_name": "WARM",
|
||||
"backend_identity": identity,
|
||||
"version_id_exact": exact.then_some(true),
|
||||
"version_state": state,
|
||||
});
|
||||
let encoded = serde_json::to_vec(&persisted).expect("invalid journal fixture should encode");
|
||||
decode_tier_delete_journal_entry(&encoded).expect_err("conflicting v4 version state must fail closed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_journals_decode_with_unknown_version_state() {
|
||||
let v1 = br#"{"version":1,"obj_name":"remote/object","version_id":"opaque","tier_name":"WARM"}"#;
|
||||
let v2 = br#"{"version":2,"obj_name":"remote/object","version_id":"opaque","tier_name":"WARM","backend_identity":[7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]}"#;
|
||||
|
||||
for payload in [v1.as_slice(), v2.as_slice()] {
|
||||
let decoded = decode_tier_delete_journal_entry(payload).expect("legacy journal should decode");
|
||||
assert_eq!(decoded.version_state, rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
assert!(!decoded.version_id_exact);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_path_is_stable_and_sanitized() {
|
||||
let je = journal_entry();
|
||||
@@ -645,8 +530,6 @@ mod tests {
|
||||
fn tier_delete_journal_paths_separate_legacy_and_backend_identities() {
|
||||
let mut legacy = journal_entry();
|
||||
legacy.backend_identity = None;
|
||||
legacy.version_id_exact = false;
|
||||
legacy.version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
let mut backend_a = journal_entry();
|
||||
backend_a.backend_identity = Some([1; 32]);
|
||||
let mut backend_b = journal_entry();
|
||||
@@ -692,8 +575,6 @@ mod tests {
|
||||
fn tier_delete_journal_without_transition_identity_stays_legacy() {
|
||||
let mut je = journal_entry();
|
||||
je.backend_identity = None;
|
||||
je.version_id_exact = false;
|
||||
je.version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
|
||||
let encoded = encode_tier_delete_journal_entry(&je).expect("legacy journal should remain encodable");
|
||||
let persisted: serde_json::Value = serde_json::from_slice(&encoded).expect("journal JSON should decode");
|
||||
|
||||
@@ -185,7 +185,6 @@ struct ObjSweeper {
|
||||
transition_status: String,
|
||||
transition_tier: String,
|
||||
transition_version_id: String,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
remote_object: String,
|
||||
}
|
||||
|
||||
@@ -232,9 +231,7 @@ impl ObjSweeper {
|
||||
}
|
||||
|
||||
pub fn should_remove_remote_object(&self) -> Option<Jentry> {
|
||||
if self.transition_status != lifecycle::TRANSITION_COMPLETE
|
||||
|| self.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
{
|
||||
if self.transition_status != lifecycle::TRANSITION_COMPLETE {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -252,11 +249,7 @@ impl ObjSweeper {
|
||||
version_id: self.transition_version_id.clone(),
|
||||
tier_name: self.transition_tier.clone(),
|
||||
backend_identity: None,
|
||||
version_id_exact: matches!(
|
||||
self.transition_version_state,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
|
||||
),
|
||||
version_state: self.transition_version_state,
|
||||
version_id_exact: false,
|
||||
});
|
||||
}
|
||||
None
|
||||
@@ -293,7 +286,6 @@ pub struct Jentry {
|
||||
pub(crate) tier_name: String,
|
||||
pub(crate) backend_identity: Option<TierDestinationId>,
|
||||
pub(crate) version_id_exact: bool,
|
||||
pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
|
||||
}
|
||||
|
||||
impl ExpiryOp for Jentry {
|
||||
@@ -338,7 +330,7 @@ async fn delete_object_from_remote_tier_raw_with_manager(
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await
|
||||
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false).await
|
||||
}
|
||||
|
||||
async fn delete_object_from_remote_tier_raw_with_lease(
|
||||
@@ -346,11 +338,8 @@ async fn delete_object_from_remote_tier_raw_with_lease(
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
validate_remote_version_id: bool,
|
||||
) -> Result<(), std::io::Error> {
|
||||
if validate_remote_version_id {
|
||||
lease.validate_remote_version_id(rv_id)?;
|
||||
}
|
||||
lease.validate_remote_version_id(rv_id)?;
|
||||
|
||||
if remote_delete_breaker_is_open(Instant::now()).await {
|
||||
metrics::counter!(METRIC_DELETE_REMOTE_BREAKER_TOTAL).increment(1);
|
||||
@@ -446,53 +435,7 @@ pub(crate) async fn delete_object_from_remote_tier_with_lease_idempotent(
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
delete_object_from_remote_tier_with_lease_idempotent_inner(obj_name, rv_id, lease, version_id_exact, true).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idempotent(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
if rv_id.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"confirmed versioned transition candidate requires a non-empty remote version",
|
||||
));
|
||||
}
|
||||
#[cfg(test)]
|
||||
if obj_name == "remote/empty-guard-probe" {
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
delete_object_from_remote_tier_with_lease_idempotent_inner(obj_name, rv_id, lease, true, false).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
tier_name: &str,
|
||||
backend_identity: TierDestinationId,
|
||||
tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, tier_name, backend_identity)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent(obj_name, rv_id, &lease).await
|
||||
}
|
||||
|
||||
async fn delete_object_from_remote_tier_with_lease_idempotent_inner(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
validate_remote_version_id: bool,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact, validate_remote_version_id)
|
||||
.await
|
||||
{
|
||||
match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact).await {
|
||||
Ok(()) => Ok(RemoteTierDeleteOutcome::Deleted),
|
||||
Err(err) if is_remote_tier_not_found_error(&err) => Ok(RemoteTierDeleteOutcome::AlreadyRemoved),
|
||||
Err(err) => {
|
||||
@@ -517,7 +460,6 @@ pub fn transitioned_delete_journal_entry(
|
||||
versioned: bool,
|
||||
suspended: bool,
|
||||
transitioned: &TransitionedObject,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
) -> Option<Jentry> {
|
||||
let sweeper = ObjSweeper {
|
||||
version_id,
|
||||
@@ -526,7 +468,6 @@ pub fn transitioned_delete_journal_entry(
|
||||
transition_status: transitioned.status.clone(),
|
||||
transition_tier: transitioned.tier.clone(),
|
||||
transition_version_id: transitioned.version_id.clone(),
|
||||
transition_version_state,
|
||||
remote_object: transitioned.name.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -534,13 +475,8 @@ pub fn transitioned_delete_journal_entry(
|
||||
sweeper.should_remove_remote_object()
|
||||
}
|
||||
|
||||
pub fn transitioned_force_delete_journal_entry(
|
||||
transitioned: &TransitionedObject,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
) -> Option<Jentry> {
|
||||
if transitioned.status != lifecycle::TRANSITION_COMPLETE
|
||||
|| transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
{
|
||||
pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject) -> Option<Jentry> {
|
||||
if transitioned.status != lifecycle::TRANSITION_COMPLETE {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -549,11 +485,7 @@ pub fn transitioned_force_delete_journal_entry(
|
||||
version_id: transitioned.version_id.clone(),
|
||||
tier_name: transitioned.tier.clone(),
|
||||
backend_identity: None,
|
||||
version_id_exact: matches!(
|
||||
transition_version_state,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
|
||||
),
|
||||
version_state: transition_version_state,
|
||||
version_id_exact: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -562,14 +494,11 @@ mod test {
|
||||
use crate::client::signer_error::invalid_utf8_header_error;
|
||||
|
||||
use super::{
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
|
||||
RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, RemoteDeleteBreaker, RemoteTierDeleteOutcome,
|
||||
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure,
|
||||
};
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -613,43 +542,6 @@ mod test {
|
||||
assert!(should_record_remote_delete_failure(&Error::other("NoSuchVersion")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transitioned_delete_journal_preserves_remote_version_state() {
|
||||
let cases = [
|
||||
(TransitionVersionState::Unknown, "legacy-version", None),
|
||||
(TransitionVersionState::KnownDisabled, "", Some(false)),
|
||||
(TransitionVersionState::SuspendedNull, "null", Some(true)),
|
||||
(TransitionVersionState::Exact, "opaque-version", Some(true)),
|
||||
];
|
||||
|
||||
for (state, version_id, expected_exact) in cases {
|
||||
let transitioned = TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: version_id.to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
status: lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let regular = transitioned_delete_journal_entry(None, false, false, &transitioned, state);
|
||||
let forced = transitioned_force_delete_journal_entry(&transitioned, state);
|
||||
|
||||
match expected_exact {
|
||||
Some(expected_exact) => {
|
||||
let regular = regular.expect("known version state should produce a regular delete journal entry");
|
||||
assert_eq!(regular.version_state, state);
|
||||
assert_eq!(regular.version_id_exact, expected_exact);
|
||||
let forced = forced.expect("known version state should produce a forced delete journal entry");
|
||||
assert_eq!(forced.version_state, state);
|
||||
assert_eq!(forced.version_id_exact, expected_exact);
|
||||
}
|
||||
None => {
|
||||
assert!(regular.is_none());
|
||||
assert!(forced.is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn idempotent_remote_delete_treats_hooked_nosuchversion_as_already_removed() {
|
||||
@@ -772,55 +664,6 @@ mod test {
|
||||
assert_eq!(backend.remove_versions().await, vec![("remote/object".to_string(), String::new())]);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn confirmed_transition_cleanup_deletes_exact_provider_token() {
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||
let manager = crate::services::tier::tier::TierConfigMgr::new();
|
||||
let backend = crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await;
|
||||
let lease = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM")
|
||||
.await
|
||||
.expect("test tier lease should be available");
|
||||
let identity = lease.backend_identity();
|
||||
drop(lease);
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
|
||||
let outcome = delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
"remote/object",
|
||||
"provider-version-token",
|
||||
"WARM",
|
||||
identity,
|
||||
&manager,
|
||||
)
|
||||
.await
|
||||
.expect("confirmed upload compensation should delete the exact provider token");
|
||||
|
||||
assert_eq!(outcome, RemoteTierDeleteOutcome::Deleted);
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![("remote/object".to_string(), "provider-version-token".to_string())]
|
||||
);
|
||||
|
||||
let err = delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
"remote/empty-guard-probe",
|
||||
"",
|
||||
"WARM",
|
||||
identity,
|
||||
&manager,
|
||||
)
|
||||
.await
|
||||
.expect_err("confirmed versioned cleanup must reject an empty token");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
|
||||
assert_eq!(backend.remove_count().await, 1);
|
||||
assert_eq!(
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.load(std::sync::atomic::Ordering::Relaxed),
|
||||
0,
|
||||
"empty remote versions must be rejected before exact cleanup dispatch"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breaker_opens_at_threshold_and_recovers_after_window() {
|
||||
let mut breaker = RemoteDeleteBreaker::new(3, Duration::from_secs(30));
|
||||
|
||||
@@ -22,11 +22,7 @@ use uuid::Uuid;
|
||||
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idempotent_with_manager_and_identity;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result as EcstoreResult};
|
||||
use crate::object_api::ObjectOptions;
|
||||
@@ -617,183 +613,6 @@ pub enum TransitionTransactionRecoveryOutcome {
|
||||
Retained,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TransitionOperatorProbe {
|
||||
Missing,
|
||||
UnversionedPresent,
|
||||
VersionedPresent(String),
|
||||
Ambiguous,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
impl From<TransitionCandidateProbe> for TransitionOperatorProbe {
|
||||
fn from(value: TransitionCandidateProbe) -> Self {
|
||||
match value {
|
||||
TransitionCandidateProbe::Missing => Self::Missing,
|
||||
TransitionCandidateProbe::UnversionedPresent => Self::UnversionedPresent,
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => Self::VersionedPresent(version_id),
|
||||
TransitionCandidateProbe::Ambiguous => Self::Ambiguous,
|
||||
TransitionCandidateProbe::Unsupported => Self::Unsupported,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct TransitionOperatorStatus {
|
||||
pub transaction_id: Uuid,
|
||||
pub state: TransitionTransactionState,
|
||||
pub tier_name: String,
|
||||
pub remote_object: String,
|
||||
pub not_after_unix_nanos: i64,
|
||||
pub probe: TransitionOperatorProbe,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct TransitionOperatorDeleteResult {
|
||||
pub status: TransitionOperatorStatus,
|
||||
pub journal_observed_after_delete: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum TransitionOperatorError {
|
||||
#[error("transition transaction was not found")]
|
||||
NotFound,
|
||||
#[error("transition transaction is still inside its active ownership window")]
|
||||
NotExpired,
|
||||
#[error("transition transaction state is not eligible for operator reconciliation: {0:?}")]
|
||||
InvalidState(TransitionTransactionState),
|
||||
#[error("an exact non-empty remote version is required")]
|
||||
RemoteVersionRequired,
|
||||
#[error("remote candidate is not proven missing: {0:?}")]
|
||||
CandidateNotMissing(TransitionOperatorProbe),
|
||||
#[error("transition transaction store failed: {0}")]
|
||||
Store(#[source] Error),
|
||||
#[error("remote tier reconciliation failed: {0}")]
|
||||
Remote(#[source] std::io::Error),
|
||||
}
|
||||
|
||||
type TransitionOperatorResult<T> = std::result::Result<T, TransitionOperatorError>;
|
||||
|
||||
fn validate_operator_reconcile_transaction(
|
||||
transaction: &TransitionTransaction,
|
||||
now_unix_nanos: i128,
|
||||
) -> TransitionOperatorResult<()> {
|
||||
transaction
|
||||
.validate()
|
||||
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
|
||||
if transaction.state != TransitionTransactionState::UploadOutcomeUnknown {
|
||||
return Err(TransitionOperatorError::InvalidState(transaction.state));
|
||||
}
|
||||
if now_unix_nanos < i128::from(transaction.not_after_unix_nanos) {
|
||||
return Err(TransitionOperatorError::NotExpired);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load_operator_reconcile_transaction(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
) -> TransitionOperatorResult<TransitionTransaction> {
|
||||
match load_transition_transaction_record(api, transaction_id).await {
|
||||
Ok(transaction) => Ok(transaction),
|
||||
Err(Error::ConfigNotFound) => Err(TransitionOperatorError::NotFound),
|
||||
Err(err) => Err(TransitionOperatorError::Store(err)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn operator_probe_transition_candidate(
|
||||
api: Arc<ECStore>,
|
||||
transaction: &TransitionTransaction,
|
||||
) -> TransitionOperatorResult<TransitionOperatorProbe> {
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
|
||||
&api.tier_config_mgr(),
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
|
||||
lease
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.await
|
||||
.map(TransitionOperatorProbe::from)
|
||||
.map_err(TransitionOperatorError::Remote)
|
||||
}
|
||||
|
||||
pub async fn inspect_transition_transaction_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
) -> TransitionOperatorResult<TransitionOperatorStatus> {
|
||||
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
|
||||
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
|
||||
let probe = operator_probe_transition_candidate(api, &transaction).await?;
|
||||
Ok(TransitionOperatorStatus {
|
||||
transaction_id,
|
||||
state: transaction.state,
|
||||
tier_name: transaction.tier_name,
|
||||
remote_object: transaction.remote_object,
|
||||
not_after_unix_nanos: transaction.not_after_unix_nanos,
|
||||
probe,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn delete_transition_candidate_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
remote_version_id: &str,
|
||||
) -> TransitionOperatorResult<TransitionOperatorDeleteResult> {
|
||||
if remote_version_id.is_empty() {
|
||||
return Err(TransitionOperatorError::RemoteVersionRequired);
|
||||
}
|
||||
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
|
||||
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
|
||||
&api.tier_config_mgr(),
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
|
||||
lease
|
||||
.validate_remote_version_id(remote_version_id)
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent(&transaction.remote_object, remote_version_id, &lease)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Remote)?;
|
||||
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
|
||||
let journal_observed_after_delete = match load_transition_transaction_record(api, transaction_id).await {
|
||||
Ok(_) => true,
|
||||
Err(Error::ConfigNotFound) => false,
|
||||
Err(err) => return Err(TransitionOperatorError::Store(err)),
|
||||
};
|
||||
Ok(TransitionOperatorDeleteResult {
|
||||
status: TransitionOperatorStatus {
|
||||
transaction_id,
|
||||
state: transaction.state,
|
||||
tier_name: transaction.tier_name,
|
||||
remote_object: transaction.remote_object,
|
||||
not_after_unix_nanos: transaction.not_after_unix_nanos,
|
||||
probe,
|
||||
},
|
||||
journal_observed_after_delete,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn finalize_missing_transition_transaction_for_operator(
|
||||
api: Arc<ECStore>,
|
||||
transaction_id: Uuid,
|
||||
) -> TransitionOperatorResult<()> {
|
||||
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
|
||||
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
|
||||
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
|
||||
if probe != TransitionOperatorProbe::Missing {
|
||||
return Err(TransitionOperatorError::CandidateNotMissing(probe));
|
||||
}
|
||||
delete_transition_transaction_record(api, transaction_id)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Store)
|
||||
}
|
||||
|
||||
pub(crate) fn decode_transition_transaction_record(object: &str, data: &[u8]) -> Result<TransitionTransaction> {
|
||||
let transaction_id = transition_transaction_id_from_record_object_name(object)?;
|
||||
TransitionTransaction::decode(transaction_id, data)
|
||||
@@ -878,7 +697,7 @@ async fn recover_unknown_upload_outcome(
|
||||
.map_err(Error::other)?;
|
||||
|
||||
match lease
|
||||
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
|
||||
.probe_transition_candidate(&transaction.remote_object)
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
@@ -889,21 +708,6 @@ async fn recover_unknown_upload_outcome(
|
||||
TransitionCandidateProbe::UnversionedPresent => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id)
|
||||
if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) =>
|
||||
{
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
&transaction.remote_object,
|
||||
&version_id,
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
&api.tier_config_mgr(),
|
||||
)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
|
||||
}
|
||||
@@ -1275,27 +1079,6 @@ mod tests {
|
||||
.expect("upload state change should succeed")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operator_reconcile_requires_expired_unknown_upload_outcome() {
|
||||
let mut transaction = new_transaction();
|
||||
let active_deadline = transaction.not_after_unix_nanos;
|
||||
|
||||
assert!(matches!(
|
||||
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) + 1),
|
||||
Err(TransitionOperatorError::InvalidState(TransitionTransactionState::UploadStarted))
|
||||
));
|
||||
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("unknown upload outcome should be recorded");
|
||||
assert!(matches!(
|
||||
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) - 1),
|
||||
Err(TransitionOperatorError::NotExpired)
|
||||
));
|
||||
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline))
|
||||
.expect("expired unknown upload outcome should be eligible");
|
||||
}
|
||||
|
||||
fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof {
|
||||
TransitionCleanupProof {
|
||||
transaction_id: transaction.transaction_id,
|
||||
|
||||
@@ -172,7 +172,7 @@ impl ProviderVersionCapabilities {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
||||
fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
|
||||
if version_id.is_empty() {
|
||||
return Err(Error::new(
|
||||
ErrorKind::InvalidData,
|
||||
|
||||
@@ -46,16 +46,6 @@ lazy_static! {
|
||||
m.insert("x-amz-replication-status".to_string(), true);
|
||||
m
|
||||
};
|
||||
static ref SSE_HEADERS: HashMap<String, bool> = {
|
||||
let mut m = HashMap::new();
|
||||
m.insert("x-amz-server-side-encryption".to_string(), true);
|
||||
m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
|
||||
m.insert("x-amz-server-side-encryption-context".to_string(), true);
|
||||
m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
|
||||
m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
|
||||
m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
|
||||
m
|
||||
};
|
||||
}
|
||||
|
||||
pub fn is_standard_query_value(qs_key: &str) -> bool {
|
||||
@@ -70,16 +60,12 @@ pub fn is_standard_header(header_key: &str) -> bool {
|
||||
*SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
|
||||
}
|
||||
|
||||
pub fn is_sse_header(header_key: &str) -> bool {
|
||||
*SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
|
||||
}
|
||||
|
||||
pub fn is_amz_header(header_key: &str) -> bool {
|
||||
let key = header_key.to_lowercase();
|
||||
key.starts_with("x-amz-meta-")
|
||||
|| key.starts_with("x-amz-grant-")
|
||||
|| key == "x-amz-acl"
|
||||
|| is_sse_header(header_key)
|
||||
|| rustfs_utils::http::is_sse_header(header_key)
|
||||
|| key.starts_with("x-amz-checksum-")
|
||||
}
|
||||
|
||||
|
||||
@@ -224,21 +224,6 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
|
||||
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
|
||||
}
|
||||
|
||||
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
|
||||
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
|
||||
if topology_member != expected_member {
|
||||
return Err(Error::other(
|
||||
"peer returned a remote version state capability for a different topology member",
|
||||
));
|
||||
}
|
||||
let server_epoch =
|
||||
Uuid::from_slice(process_epoch).map_err(|_| Error::other("peer returned an invalid remote version state epoch"))?;
|
||||
if server_epoch.is_nil() {
|
||||
return Err(Error::other("peer returned a nil remote version state epoch"));
|
||||
}
|
||||
Ok(server_epoch)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerLiveEventsBatch {
|
||||
pub events: Vec<u8>,
|
||||
@@ -250,7 +235,6 @@ pub struct PeerLiveEventsBatch {
|
||||
pub struct PeerRestClient {
|
||||
pub host: XHost,
|
||||
pub grid_host: String,
|
||||
topology_member: String,
|
||||
offline: Arc<AtomicBool>,
|
||||
recovery_running: Arc<AtomicBool>,
|
||||
}
|
||||
@@ -343,11 +327,9 @@ impl PeerRestClient {
|
||||
}
|
||||
|
||||
pub fn new(host: XHost, grid_host: String) -> Self {
|
||||
let topology_member = host.to_string();
|
||||
Self {
|
||||
host,
|
||||
grid_host,
|
||||
topology_member,
|
||||
offline: Arc::new(AtomicBool::new(false)),
|
||||
recovery_running: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
@@ -367,11 +349,7 @@ impl PeerRestClient {
|
||||
|
||||
let client = match grid_host {
|
||||
Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
|
||||
Ok(host) => {
|
||||
let mut client = PeerRestClient::new(host, grid_host);
|
||||
client.topology_member = peer_host_port.clone();
|
||||
Some(client)
|
||||
}
|
||||
Ok(host) => Some(PeerRestClient::new(host, grid_host)),
|
||||
Err(err) => {
|
||||
warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
|
||||
None
|
||||
@@ -1183,15 +1161,6 @@ impl PeerRestClient {
|
||||
validate_heal_control_capability_proof(&canonical_ack, &proof)
|
||||
}
|
||||
|
||||
pub async fn probe_remote_version_state(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
|
||||
let probe = rustfs_protos::remote_version_state_capability_probe(Uuid::new_v4().as_bytes());
|
||||
let result = self
|
||||
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
|
||||
.await?;
|
||||
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
|
||||
Ok((self.topology_member.clone(), epoch))
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
@@ -2302,22 +2271,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_capability_decoder_fails_closed() {
|
||||
let epoch = Uuid::new_v4();
|
||||
let result = rustfs_protos::encode_remote_version_state_capability("node-a:9000", epoch.as_bytes())
|
||||
.expect("small capability response should encode");
|
||||
assert_eq!(
|
||||
decode_remote_version_state_capability("node-a:9000", &result).expect("valid epoch should decode"),
|
||||
epoch
|
||||
);
|
||||
assert!(decode_remote_version_state_capability("node-b:9000", &result).is_err());
|
||||
assert!(decode_remote_version_state_capability("node-a:9000", &result[..result.len() - 1]).is_err());
|
||||
let nil = rustfs_protos::encode_remote_version_state_capability("node-a:9000", Uuid::nil().as_bytes())
|
||||
.expect("small capability response should encode");
|
||||
assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
|
||||
}
|
||||
|
||||
struct TierMutationResponseFixture<'a> {
|
||||
version: u32,
|
||||
phase: TierMutationRpcPhase,
|
||||
|
||||
@@ -2543,7 +2543,6 @@ mod tests {
|
||||
data_dir: None,
|
||||
delete_marker: false,
|
||||
transitioned_object: Default::default(),
|
||||
transition_version_state: Default::default(),
|
||||
restore_ongoing: false,
|
||||
restore_expires: None,
|
||||
user_tags: Arc::new(String::new()),
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use std::fmt::{Display, Formatter};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReadEncryptionMode {
|
||||
Direct { base_nonce: [u8; 12] },
|
||||
Object,
|
||||
}
|
||||
|
||||
pub struct ReadEncryptionMaterial {
|
||||
pub key_bytes: [u8; 32],
|
||||
pub mode: ReadEncryptionMode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum EncryptionResolutionErrorKind {
|
||||
InvalidRequest,
|
||||
InvalidMetadata,
|
||||
ServiceUnavailable,
|
||||
DecryptionFailed,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct EncryptionResolutionError {
|
||||
kind: EncryptionResolutionErrorKind,
|
||||
message: String,
|
||||
}
|
||||
|
||||
impl EncryptionResolutionError {
|
||||
pub fn new(kind: EncryptionResolutionErrorKind, message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn kind(&self) -> EncryptionResolutionErrorKind {
|
||||
self.kind
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for EncryptionResolutionError {
|
||||
fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str(&self.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl Error for EncryptionResolutionError {}
|
||||
|
||||
pub struct ReadEncryptionRequest<'a> {
|
||||
pub bucket: &'a str,
|
||||
pub object: &'a str,
|
||||
pub metadata: &'a HashMap<String, String>,
|
||||
pub headers: &'a HeaderMap<HeaderValue>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait ObjectEncryptionResolver: Send + Sync {
|
||||
async fn resolve_read_material(
|
||||
&self,
|
||||
request: ReadEncryptionRequest<'_>,
|
||||
) -> Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError>;
|
||||
}
|
||||
@@ -84,6 +84,7 @@ pub(crate) fn legacy_encrypted_range_seek_enabled() -> bool {
|
||||
}
|
||||
|
||||
mod body_cache_hook;
|
||||
mod encryption;
|
||||
mod hook_slot;
|
||||
mod object_mutation_hook;
|
||||
mod readers;
|
||||
@@ -98,6 +99,10 @@ pub use body_cache_hook::{
|
||||
pub(crate) use body_cache_hook::{
|
||||
get_object_body_cache_hook, get_object_body_cache_hook_suppressed, without_get_object_body_cache_hook,
|
||||
};
|
||||
pub use encryption::{
|
||||
EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial,
|
||||
ReadEncryptionMode, ReadEncryptionRequest,
|
||||
};
|
||||
pub(crate) use object_mutation_hook::notify_object_mutation;
|
||||
pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook};
|
||||
pub use readers::*;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -180,7 +180,6 @@ pub struct ObjectInfo {
|
||||
pub data_dir: Option<Uuid>,
|
||||
pub delete_marker: bool,
|
||||
pub transitioned_object: TransitionedObject,
|
||||
pub transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
pub restore_ongoing: bool,
|
||||
pub restore_expires: Option<OffsetDateTime>,
|
||||
pub user_tags: Arc<String>,
|
||||
@@ -221,7 +220,6 @@ impl Clone for ObjectInfo {
|
||||
data_dir: self.data_dir,
|
||||
delete_marker: self.delete_marker,
|
||||
transitioned_object: self.transitioned_object.clone(),
|
||||
transition_version_state: self.transition_version_state,
|
||||
restore_ongoing: self.restore_ongoing,
|
||||
restore_expires: self.restore_expires,
|
||||
user_tags: self.user_tags.clone(),
|
||||
@@ -273,29 +271,9 @@ impl ObjectInfo {
|
||||
}
|
||||
|
||||
pub fn is_encrypted(&self) -> bool {
|
||||
// Corresponding to the logic in rustfs/src/sse.rs/encryption_material_to_metadata function
|
||||
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
|
||||
|
||||
self.user_defined.keys().any(|key| {
|
||||
let lower = key.to_ascii_lowercase();
|
||||
lower.starts_with("x-minio-encryption-")
|
||||
|| lower.starts_with("x-minio-internal-server-side-encryption-")
|
||||
|| matches!(
|
||||
lower.as_str(),
|
||||
"x-minio-internal-encrypted-multipart"
|
||||
| "x-rustfs-encryption-key"
|
||||
| "x-rustfs-encryption-algorithm"
|
||||
| "x-rustfs-encryption-iv"
|
||||
| "x-rustfs-encryption-key-id"
|
||||
| "x-rustfs-encryption-context"
|
||||
| "x-rustfs-encryption-tag"
|
||||
| "x-amz-server-side-encryption-aws-kms-key-id"
|
||||
| SSEC_ALGORITHM_HEADER
|
||||
| SSEC_KEY_HEADER
|
||||
| SSEC_KEY_MD5_HEADER
|
||||
| "x-amz-server-side-encryption"
|
||||
)
|
||||
})
|
||||
self.user_defined
|
||||
.keys()
|
||||
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|
||||
}
|
||||
|
||||
/// Maximum inline size for non-versioned objects (128 KiB).
|
||||
@@ -339,26 +317,7 @@ impl ObjectInfo {
|
||||
}
|
||||
|
||||
pub fn encryption_original_size(&self) -> std::io::Result<Option<i64>> {
|
||||
let actual_size = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE);
|
||||
if let Some(size_str) = self
|
||||
.user_defined
|
||||
.get("x-rustfs-encryption-original-size")
|
||||
.map(String::as_str)
|
||||
.or_else(|| {
|
||||
self.user_defined
|
||||
.get("x-amz-server-side-encryption-customer-original-size")
|
||||
.map(String::as_str)
|
||||
})
|
||||
.or(actual_size.as_deref())
|
||||
&& !size_str.is_empty()
|
||||
{
|
||||
let size = size_str
|
||||
.parse::<i64>()
|
||||
.map_err(|e| std::io::Error::other(format!("Failed to parse encryption original size: {e}")))?;
|
||||
return Ok(Some(size));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
rustfs_utils::http::get_object_encryption_original_size(&self.user_defined)
|
||||
}
|
||||
|
||||
pub fn decrypted_size(&self) -> std::io::Result<i64> {
|
||||
@@ -388,9 +347,6 @@ impl ObjectInfo {
|
||||
return Ok(actual_size);
|
||||
}
|
||||
|
||||
// Check if object is encrypted
|
||||
// Managed SSE stores original size in x-rustfs-encryption-original-size metadata
|
||||
// SSE-C stores original size in x-amz-server-side-encryption-customer-original-size
|
||||
if let Some(size) = self.encryption_original_size()? {
|
||||
return Ok(size);
|
||||
}
|
||||
@@ -466,11 +422,11 @@ impl ObjectInfo {
|
||||
|
||||
let transitioned_object = TransitionedObject {
|
||||
name: fi.transitioned_objname.clone(),
|
||||
version_id: fi
|
||||
.transition_version
|
||||
.clone()
|
||||
.or_else(|| fi.transition_version_id.map(|version_id| version_id.to_string()))
|
||||
.unwrap_or_default(),
|
||||
version_id: if let Some(transition_version_id) = fi.transition_version_id {
|
||||
transition_version_id.to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
status: fi.transition_status.clone(),
|
||||
free_version: fi.tier_free_version(),
|
||||
tier: fi.transition_tier.clone(),
|
||||
@@ -539,7 +495,6 @@ impl ObjectInfo {
|
||||
inlined,
|
||||
user_defined: Arc::new(metadata),
|
||||
transitioned_object,
|
||||
transition_version_state: fi.transition_version_state,
|
||||
checksum: fi.checksum.clone(),
|
||||
storage_class,
|
||||
restore_ongoing,
|
||||
@@ -881,6 +836,19 @@ mod tests {
|
||||
assert!(!object.is_inline_fast_path_eligible(), "transitioned objects must fall back");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minio_internal_encryption_metadata_is_not_treated_as_plaintext() {
|
||||
let object = ObjectInfo {
|
||||
user_defined: Arc::new(HashMap::from([(
|
||||
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key".to_string(),
|
||||
"sealed".to_string(),
|
||||
)])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(object.is_encrypted());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn versions_after_marker_handles_null_version_marker() {
|
||||
let first_version = Uuid::parse_str("11111111-2222-3333-4444-555555555555").unwrap();
|
||||
|
||||
@@ -46,6 +46,7 @@ use crate::bucket::metadata_sys::BucketMetadataSys;
|
||||
use crate::bucket::replication::{DynReplicationPool, ReplicationStats};
|
||||
use crate::disk::DiskStore;
|
||||
use crate::layout::endpoints::{EndpointServerPools, SetupType};
|
||||
use crate::object_api::ObjectEncryptionResolver;
|
||||
use crate::services::event_notification::EventNotifier;
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
|
||||
@@ -159,6 +160,8 @@ pub struct InstanceContext {
|
||||
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
|
||||
/// Replaces the process-global cancel-token static.
|
||||
background_cancel_token: OnceLock<CancellationToken>,
|
||||
/// Resolves object-encryption material at the application boundary.
|
||||
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
#[cfg(test)]
|
||||
@@ -197,6 +200,7 @@ impl InstanceContext {
|
||||
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
|
||||
bucket_metadata_sys: std::sync::Mutex::new(None),
|
||||
background_cancel_token: OnceLock::new(),
|
||||
object_encryption_resolver: OnceLock::new(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
#[cfg(test)]
|
||||
@@ -209,6 +213,19 @@ impl InstanceContext {
|
||||
self.lock_manager.clone()
|
||||
}
|
||||
|
||||
/// Install the application-owned object-encryption resolver once.
|
||||
pub fn set_object_encryption_resolver(
|
||||
&self,
|
||||
resolver: Arc<dyn ObjectEncryptionResolver>,
|
||||
) -> Result<(), Arc<dyn ObjectEncryptionResolver>> {
|
||||
self.object_encryption_resolver.set(resolver)
|
||||
}
|
||||
|
||||
/// Return the configured object-encryption resolver, if startup installed one.
|
||||
pub fn object_encryption_resolver(&self) -> Option<&dyn ObjectEncryptionResolver> {
|
||||
self.object_encryption_resolver.get().map(Arc::as_ref)
|
||||
}
|
||||
|
||||
/// Set this instance's S3 region.
|
||||
///
|
||||
/// Write-once: panics on a second write, preserving the startup fail-fast
|
||||
|
||||
@@ -46,7 +46,6 @@ use crate::{
|
||||
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
|
||||
use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config};
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service};
|
||||
use rustfs_lock::client::LockClient;
|
||||
use s3s::dto::BucketLifecycleConfiguration;
|
||||
use s3s::region::Region;
|
||||
@@ -105,10 +104,6 @@ pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_
|
||||
global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error);
|
||||
}
|
||||
|
||||
pub(crate) async fn object_encryption_service() -> Option<Arc<ObjectEncryptionService>> {
|
||||
get_global_encryption_service().await
|
||||
}
|
||||
|
||||
pub fn object_store_handle() -> Option<Arc<ECStore>> {
|
||||
resolve_object_store_handle()
|
||||
}
|
||||
|
||||
@@ -29,11 +29,11 @@ use rustfs_madmin::metrics::RealtimeMetrics;
|
||||
use rustfs_madmin::net::NetInfo;
|
||||
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
|
||||
use rustfs_utils::XHost;
|
||||
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
|
||||
use std::collections::{HashMap, hash_map::DefaultHasher};
|
||||
use std::future::Future;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::time::{sleep, timeout};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
@@ -47,9 +47,6 @@ const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation
|
||||
const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
|
||||
const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
|
||||
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Cached result from the last successful admin call to a peer.
|
||||
struct PeerAdminCache {
|
||||
@@ -94,180 +91,6 @@ lazy_static! {
|
||||
pub static ref GLOBAL_NOTIFICATION_SYS: OnceLock<Arc<NotificationSys>> = OnceLock::new();
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: String,
|
||||
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
||||
expires_at: Instant,
|
||||
}
|
||||
|
||||
impl RemoteVersionStateFleetProof {
|
||||
fn token(&self) -> RemoteVersionStateFleetProofToken {
|
||||
RemoteVersionStateFleetProofToken {
|
||||
topology_fingerprint: self.topology_fingerprint.clone(),
|
||||
peer_epochs: self.peer_epochs.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub(crate) struct RemoteVersionStateFleetProofToken {
|
||||
topology_fingerprint: String,
|
||||
peer_epochs: Arc<BTreeMap<String, Uuid>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RemoteVersionStateFleetProofState {
|
||||
proof: Option<RemoteVersionStateFleetProof>,
|
||||
topology_conflict: bool,
|
||||
}
|
||||
|
||||
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<RemoteVersionStateFleetProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
|
||||
fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<RemoteVersionStateFleetProofState> {
|
||||
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()))
|
||||
}
|
||||
|
||||
fn replace_remote_version_state_fleet_proof(proof: Option<RemoteVersionStateFleetProof>) {
|
||||
replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof);
|
||||
}
|
||||
|
||||
fn replace_remote_version_state_fleet_proof_in(
|
||||
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
|
||||
proof: Option<RemoteVersionStateFleetProof>,
|
||||
) {
|
||||
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
|
||||
}
|
||||
|
||||
fn publish_remote_version_state_probe_result(
|
||||
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
|
||||
topology_fingerprint: &str,
|
||||
result: Result<BTreeMap<String, Uuid>>,
|
||||
observed_at: Instant,
|
||||
) -> Option<Error> {
|
||||
match result {
|
||||
Ok(peer_epochs) => {
|
||||
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let peer_epochs = state
|
||||
.proof
|
||||
.as_ref()
|
||||
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
|
||||
.map(|proof| Arc::clone(&proof.peer_epochs))
|
||||
.unwrap_or_else(|| Arc::new(peer_epochs));
|
||||
state.proof = Some(RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: topology_fingerprint.to_string(),
|
||||
peer_epochs,
|
||||
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
|
||||
});
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
replace_remote_version_state_fleet_proof_in(slot, None);
|
||||
Some(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let state = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
acquire_remote_version_state_fleet_proof_from(&state, expected_topology, Instant::now())
|
||||
}
|
||||
|
||||
fn acquire_remote_version_state_fleet_proof_from(
|
||||
state: &RemoteVersionStateFleetProofState,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> Option<RemoteVersionStateFleetProofToken> {
|
||||
if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
|
||||
return None;
|
||||
}
|
||||
state.proof.as_ref().map(RemoteVersionStateFleetProof::token)
|
||||
}
|
||||
|
||||
pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
let state = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.topology_conflict {
|
||||
return false;
|
||||
}
|
||||
state.proof.as_ref().is_some_and(|current| {
|
||||
current.topology_fingerprint == *expected_topology
|
||||
&& current.topology_fingerprint == proof.topology_fingerprint
|
||||
&& Arc::ptr_eq(¤t.peer_epochs, &proof.peer_epochs)
|
||||
&& Instant::now() < current.expires_at
|
||||
})
|
||||
}
|
||||
|
||||
fn remote_version_state_fleet_proof_valid_at(
|
||||
proof: Option<&RemoteVersionStateFleetProof>,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
|
||||
}
|
||||
|
||||
fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
|
||||
if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
|
||||
return Err(Error::other("remote version state capability peer identity is invalid"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
|
||||
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
|
||||
let mut state = remote_version_state_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.topology_conflict = true;
|
||||
state.proof = None;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let result = match get_global_notification_sys() {
|
||||
Some(notification_sys) => {
|
||||
match timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
|
||||
}
|
||||
}
|
||||
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
|
||||
};
|
||||
let topology_conflict = remote_version_state_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.topology_conflict;
|
||||
if topology_conflict {
|
||||
replace_remote_version_state_fleet_proof(None);
|
||||
} else if let Some(err) = publish_remote_version_state_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
result,
|
||||
Instant::now(),
|
||||
) {
|
||||
debug!(error = %err, "remote version state fleet capability probe failed closed");
|
||||
}
|
||||
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
|
||||
let _ = GLOBAL_NOTIFICATION_SYS
|
||||
.set(Arc::new(NotificationSys::new(eps).await))
|
||||
@@ -292,17 +115,7 @@ pub struct NotificationSys {
|
||||
|
||||
impl NotificationSys {
|
||||
pub async fn new(eps: EndpointServerPools) -> Self {
|
||||
let expected_remote_hosts = eps
|
||||
.peer_grid_host_slots_sorted()
|
||||
.into_iter()
|
||||
.filter_map(|(peer, _, is_local)| (!is_local).then_some(peer))
|
||||
.collect::<Vec<_>>();
|
||||
let (peer_clients, all_peer_clients, peer_topology_hosts) = PeerRestClient::new_clients_with_topology(eps).await;
|
||||
let peer_topology_hosts = if peer_topology_hosts.is_empty() {
|
||||
expected_remote_hosts
|
||||
} else {
|
||||
peer_topology_hosts
|
||||
};
|
||||
let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
|
||||
Self {
|
||||
peer_clients,
|
||||
@@ -312,24 +125,6 @@ impl NotificationSys {
|
||||
tier_config_reload_workers: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_remote_version_state_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
|
||||
if self.peer_clients.len() != self.peer_topology_hosts.len() {
|
||||
return Err(Error::other("remote version state capability fleet membership is incomplete"));
|
||||
}
|
||||
let probes = self.peer_clients.iter().map(|client| async {
|
||||
let client = client
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("remote version state capability peer is unreachable"))?;
|
||||
client.probe_remote_version_state(topology_fingerprint.to_string()).await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
for result in join_all(probes).await {
|
||||
let (peer, epoch) = result?;
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
Ok(peer_epochs)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct NotificationPeerErr {
|
||||
@@ -2091,183 +1886,6 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
|
||||
let now = Instant::now();
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
|
||||
let proof = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(peer_epochs),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
|
||||
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
|
||||
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now));
|
||||
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
|
||||
assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_rejects_nil_process_epoch() {
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "peer-a".to_string(), Uuid::nil()).is_err());
|
||||
assert!(peer_epochs.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_accepts_single_node_membership() {
|
||||
let now = Instant::now();
|
||||
let proof = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::new()),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
|
||||
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
|
||||
let now = Instant::now();
|
||||
let proof = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
};
|
||||
let captured = proof.token();
|
||||
let restarted = RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: proof.topology_fingerprint.clone(),
|
||||
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
|
||||
expires_at: proof.expires_at,
|
||||
};
|
||||
|
||||
assert!(captured != restarted.token());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
|
||||
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
|
||||
let now = Instant::now();
|
||||
let epoch = Uuid::new_v4();
|
||||
let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
|
||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
|
||||
let original = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("successful probe should publish proof")
|
||||
.token();
|
||||
|
||||
assert!(
|
||||
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()
|
||||
);
|
||||
let renewed = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("renewal should retain proof")
|
||||
.token();
|
||||
assert!(Arc::ptr_eq(&original.peer_epochs, &renewed.peer_epochs));
|
||||
|
||||
let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(
|
||||
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2))
|
||||
.is_none()
|
||||
);
|
||||
let replaced = slot
|
||||
.read()
|
||||
.expect("proof slot should not poison")
|
||||
.proof
|
||||
.as_ref()
|
||||
.expect("restarted peer should publish a new proof")
|
||||
.token();
|
||||
assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
|
||||
let now = Instant::now();
|
||||
let mut state = RemoteVersionStateFleetProofState {
|
||||
proof: Some(RemoteVersionStateFleetProof {
|
||||
topology_fingerprint: "topology-a".to_string(),
|
||||
peer_epochs: Arc::new(BTreeMap::new()),
|
||||
expires_at: now + Duration::from_secs(1),
|
||||
}),
|
||||
topology_conflict: false,
|
||||
};
|
||||
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some());
|
||||
|
||||
state.topology_conflict = true;
|
||||
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_probe_rejects_duplicate_member_or_process_epoch() {
|
||||
let epoch = Uuid::new_v4();
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), epoch)
|
||||
.expect("first member should be admitted");
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-b:9000".to_string(), epoch).is_err());
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), Uuid::new_v4()).is_err());
|
||||
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-c:9000".to_string(), Uuid::nil()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
|
||||
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
|
||||
let now = Instant::now();
|
||||
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
|
||||
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
|
||||
|
||||
assert!(
|
||||
publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
|
||||
);
|
||||
assert!(slot.read().expect("proof slot should not poison").proof.is_none());
|
||||
|
||||
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
|
||||
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_version_state_fleet_probe_rejects_unreachable_member() {
|
||||
let notification_sys = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: vec![None, None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
|
||||
let err = notification_sys
|
||||
.probe_remote_version_state_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("an unreachable configured member must fail the fleet proof");
|
||||
assert!(err.to_string().contains("unreachable"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_version_state_fleet_probe_rejects_missing_member_slot() {
|
||||
let notification_sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: vec![None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: Vec::new(),
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
|
||||
let err = notification_sys
|
||||
.probe_remote_version_state_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("a missing configured member slot must fail the fleet proof");
|
||||
assert!(err.to_string().contains("incomplete"));
|
||||
}
|
||||
|
||||
fn build_props(endpoint: &str) -> ServerProperties {
|
||||
ServerProperties {
|
||||
endpoint: endpoint.to_string(),
|
||||
|
||||
@@ -931,10 +931,7 @@ pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str)
|
||||
status: fi.transition_status.clone(),
|
||||
tier: fi.transition_tier.clone(),
|
||||
remote_object: fi.transitioned_objname.clone(),
|
||||
remote_version_id: fi
|
||||
.transition_version
|
||||
.clone()
|
||||
.or_else(|| fi.transition_version_id.map(|id| id.to_string())),
|
||||
remote_version_id: fi.transition_version_id.map(|id| id.to_string()),
|
||||
free_version_count,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -236,7 +236,6 @@ struct TierPublishTransition {
|
||||
|
||||
struct PreparedTierDriver {
|
||||
tier_name: String,
|
||||
tier_config: TierConfig,
|
||||
config_fingerprint: TierDriverFingerprint,
|
||||
backend_identity: TierDestinationId,
|
||||
exact_get_delete: bool,
|
||||
@@ -1343,7 +1342,6 @@ fn tier_exact_get_delete(config: &TierConfig) -> bool {
|
||||
|
||||
struct TierDriverGeneration {
|
||||
tier_name: Arc<str>,
|
||||
tier_config: TierConfig,
|
||||
generation: DriverRevision,
|
||||
// Process-local only: this may reflect credential changes and must never be persisted or logged.
|
||||
config_fingerprint: TierDriverFingerprint,
|
||||
@@ -1351,9 +1349,6 @@ struct TierDriverGeneration {
|
||||
backend_identity: TierDestinationId,
|
||||
exact_get_delete: bool,
|
||||
driver: SharedWarmBackend,
|
||||
reconciler: tokio::sync::OnceCell<
|
||||
Option<Arc<dyn crate::services::tier::warm_backend::TransitionCandidateReconciler + Send + Sync + 'static>>,
|
||||
>,
|
||||
accepting: AtomicBool,
|
||||
active_leases: AtomicUsize,
|
||||
drained: tokio::sync::Notify,
|
||||
@@ -1478,35 +1473,6 @@ impl TierOperationLease {
|
||||
self.inner.backend_identity
|
||||
}
|
||||
|
||||
pub(crate) async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
transaction_id: uuid::Uuid,
|
||||
) -> io::Result<TransitionCandidateProbe> {
|
||||
let Some(reconciler) = self
|
||||
.inner
|
||||
.reconciler
|
||||
.get_or_try_init(|| async {
|
||||
crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
|
||||
.await
|
||||
.map(|reconciler| reconciler.map(Arc::from))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| io::Error::other(err.message))?
|
||||
else {
|
||||
return self.inner.driver.probe_transition_candidate(object).await;
|
||||
};
|
||||
reconciler
|
||||
.probe_transition_candidate_for(
|
||||
object,
|
||||
crate::services::tier::warm_backend::TransitionCandidateIdentity {
|
||||
transaction_id,
|
||||
destination_id: self.backend_identity(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
|
||||
self.inner.driver.validate_remote_version_id(remote_version_id)?;
|
||||
if !remote_version_id.is_empty() && !self.inner.exact_get_delete {
|
||||
@@ -2867,7 +2833,6 @@ impl TierConfigMgr {
|
||||
let exact_get_delete = tier_exact_get_delete(config);
|
||||
Some(PreparedTierDriver {
|
||||
tier_name: tier_name.to_string(),
|
||||
tier_config: config.clone(),
|
||||
config_fingerprint,
|
||||
backend_identity,
|
||||
exact_get_delete,
|
||||
@@ -2896,13 +2861,11 @@ impl TierConfigMgr {
|
||||
})?;
|
||||
let entry = Arc::new(TierDriverGeneration {
|
||||
tier_name: Arc::from(prepared.tier_name.as_str()),
|
||||
tier_config: prepared.tier_config.clone(),
|
||||
generation,
|
||||
config_fingerprint: prepared.config_fingerprint,
|
||||
backend_identity: prepared.backend_identity,
|
||||
exact_get_delete: prepared.exact_get_delete,
|
||||
driver: prepared.driver.clone(),
|
||||
reconciler: tokio::sync::OnceCell::new(),
|
||||
accepting: AtomicBool::new(true),
|
||||
active_leases: AtomicUsize::new(0),
|
||||
drained: tokio::sync::Notify::new(),
|
||||
@@ -3646,13 +3609,11 @@ impl TierConfigMgr {
|
||||
let driver: SharedWarmBackend = Arc::from(driver);
|
||||
let entry = Arc::new(TierDriverGeneration {
|
||||
tier_name: Arc::from(tier_name),
|
||||
tier_config: config.clone(),
|
||||
generation,
|
||||
config_fingerprint,
|
||||
backend_identity,
|
||||
exact_get_delete,
|
||||
driver: driver.clone(),
|
||||
reconciler: tokio::sync::OnceCell::new(),
|
||||
accepting: AtomicBool::new(true),
|
||||
active_leases: AtomicUsize::new(0),
|
||||
drained: tokio::sync::Notify::new(),
|
||||
@@ -9313,8 +9274,7 @@ mod tests {
|
||||
version_id: "v1".to_string(),
|
||||
tier_name: "COLD-A".to_string(),
|
||||
backend_identity: Some(current_identity),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
journal_store
|
||||
.insert_config_object(
|
||||
|
||||
@@ -73,21 +73,6 @@ pub enum TransitionCandidateProbe {
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct TransitionCandidateIdentity {
|
||||
pub transaction_id: uuid::Uuid,
|
||||
pub destination_id: [u8; 32],
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait TransitionCandidateReconciler {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait WarmBackend {
|
||||
async fn validate(&self) -> Result<(), std::io::Error> {
|
||||
@@ -204,20 +189,6 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
|
||||
metadata.remove(key);
|
||||
}
|
||||
|
||||
for suffix in [
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
] {
|
||||
for key in [
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix),
|
||||
format!("{}{}", rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, suffix),
|
||||
] {
|
||||
if let Some(value) = metadata.remove(&key) {
|
||||
metadata.insert(format!("x-amz-meta-{key}"), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
opts.user_metadata = metadata;
|
||||
opts
|
||||
}
|
||||
@@ -475,51 +446,6 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
Ok(d)
|
||||
}
|
||||
|
||||
pub(crate) async fn new_transition_candidate_reconciler(
|
||||
tier: &TierConfig,
|
||||
) -> Result<Option<Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>>, AdminError> {
|
||||
let reconciler: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static> = match tier.tier_type {
|
||||
TierType::S3 => Box::new(
|
||||
WarmBackendS3::new(tier.s3.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::MinIO => Box::new(
|
||||
WarmBackendMinIO::new(tier.minio.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::RustFS => Box::new(
|
||||
WarmBackendRustFS::new(tier.rustfs.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
TierType::R2 => Box::new(
|
||||
WarmBackendR2::new(tier.r2.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
admin_err.message = err.to_string();
|
||||
admin_err
|
||||
})?,
|
||||
),
|
||||
_ => return Ok(None),
|
||||
};
|
||||
Ok(Some(reconciler))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -849,37 +775,6 @@ mod tests {
|
||||
assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_persists_both_candidate_identity_keys_as_s3_metadata() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa".to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
"5a".repeat(32),
|
||||
);
|
||||
|
||||
let opts = build_transition_put_options("COLD".to_string(), metadata);
|
||||
|
||||
for suffix in [
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
] {
|
||||
assert!(opts.user_metadata.contains_key(&format!(
|
||||
"x-amz-meta-{}",
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix)
|
||||
)));
|
||||
assert!(opts.user_metadata.contains_key(&format!(
|
||||
"x-amz-meta-{}{suffix}",
|
||||
rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_transition_put_options_requests_no_checksum_and_content_md5() {
|
||||
// Regression for rustfs/rustfs#4811: transition uploads must leave the
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
@@ -46,19 +45,6 @@ const MAX_PARTS_COUNT: i64 = 10000;
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
|
||||
|
||||
fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
|
||||
if remote_version.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let generation = remote_version
|
||||
.parse::<i64>()
|
||||
.map_err(|_| Error::new(ErrorKind::InvalidData, "GCS remote version is not a valid generation"))?;
|
||||
if generation <= 0 {
|
||||
return Err(Error::new(ErrorKind::InvalidData, "GCS remote version generation must be positive"));
|
||||
}
|
||||
Ok(Some(generation))
|
||||
}
|
||||
|
||||
pub struct WarmBackendGCS {
|
||||
pub client: Arc<Storage>,
|
||||
pub control: Arc<StorageControl>,
|
||||
@@ -119,10 +105,6 @@ impl WarmBackendGCS {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for WarmBackendGCS {
|
||||
fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
|
||||
parse_generation(remote_version_id).map(|_| ())
|
||||
}
|
||||
|
||||
async fn put_with_meta(
|
||||
&self,
|
||||
object: &str,
|
||||
@@ -153,9 +135,6 @@ impl WarmBackend for WarmBackendGCS {
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
|
||||
if let Some(generation) = parse_generation(rv)? {
|
||||
req = req.set_generation(generation);
|
||||
}
|
||||
|
||||
// Honor the requested byte range so Range GETs on tiered objects return the exact
|
||||
// interval instead of the whole object (matches the s3/s3sdk/rustfs warm backends).
|
||||
@@ -185,15 +164,13 @@ impl WarmBackend for WarmBackendGCS {
|
||||
// gRPC v2 DeleteObject requires the bucket in resource-name form. Without this the
|
||||
// deleted tiered object was never removed from GCS (empty impl returned Ok), leaking
|
||||
// remote data forever.
|
||||
let mut req = self
|
||||
.control
|
||||
self.control
|
||||
.delete_object()
|
||||
.set_bucket(format!("projects/_/buckets/{}", self.bucket))
|
||||
.set_object(self.get_dest(object));
|
||||
if let Some(generation) = parse_generation(rv)? {
|
||||
req = req.set_generation(generation);
|
||||
}
|
||||
req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
.set_object(self.get_dest(object))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -214,30 +191,6 @@ impl WarmBackend for WarmBackendGCS {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::parse_generation;
|
||||
use std::io::ErrorKind;
|
||||
|
||||
#[test]
|
||||
fn generation_parser_preserves_exact_numeric_versions() {
|
||||
assert_eq!(parse_generation("").expect("empty generation means no version condition"), None);
|
||||
assert_eq!(parse_generation("1").expect("minimum generation should parse"), Some(1));
|
||||
assert_eq!(
|
||||
parse_generation(&i64::MAX.to_string()).expect("maximum generation should parse"),
|
||||
Some(i64::MAX)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generation_parser_rejects_unknown_or_non_positive_versions() {
|
||||
for value in ["unknown", "1.0", "-1", "0", "9223372036854775808"] {
|
||||
let err = parse_generation(value).expect_err("unknown generation must fail closed");
|
||||
assert_eq!(err.kind(), ErrorKind::InvalidData, "{value}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*fn gcs_to_object_error(err: Error, params: Vec<String>) -> Option<Error> {
|
||||
if err == nil {
|
||||
return nil
|
||||
|
||||
@@ -135,20 +135,6 @@ impl WarmBackend for WarmBackendMinIO {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&self.0, object, identity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
|
||||
@@ -135,20 +135,6 @@ impl WarmBackend for WarmBackendR2 {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendR2 {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&self.0, object, identity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
|
||||
@@ -132,20 +132,6 @@ impl WarmBackend for WarmBackendRustFS {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
|
||||
&self.0, object, identity,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
|
||||
@@ -29,7 +29,6 @@ use crate::client::{
|
||||
api_remove::{RemoveObjectOptions, RemoveObjectResult},
|
||||
api_s3_datatypes::ListVersionsResult,
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
provider_versions::validate_remote_version_id,
|
||||
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
@@ -37,10 +36,7 @@ use crate::error::ErrorResponse;
|
||||
use crate::error::error_resp_to_object_err;
|
||||
use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{
|
||||
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
|
||||
build_transition_put_options,
|
||||
},
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
@@ -193,188 +189,44 @@ impl WarmBackendS3 {
|
||||
remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate_versions(
|
||||
&self,
|
||||
object: &str,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let remote_object = self.get_dest(object);
|
||||
async fn list_transition_candidate_versions(&self, object: &str) -> Result<ListVersionsResult, std::io::Error> {
|
||||
let mut opts = ListObjectsOptions::default();
|
||||
opts.set("prefix", &remote_object);
|
||||
opts.set("max-keys", "1000");
|
||||
|
||||
let mut key_marker = String::new();
|
||||
let mut version_id_marker = String::new();
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
loop {
|
||||
let versions = self
|
||||
.client
|
||||
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
|
||||
.await?;
|
||||
candidates.extend(&remote_object, &versions);
|
||||
if candidates.is_ambiguous() {
|
||||
return Ok(TransitionCandidateProbe::Ambiguous);
|
||||
}
|
||||
if !versions.is_truncated {
|
||||
return classify_transition_candidates(candidates, bucket_versioning);
|
||||
}
|
||||
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
|
||||
}
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate_identity(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let remote_object = self.get_dest(object);
|
||||
let mut opts = ListObjectsOptions::default();
|
||||
opts.set("prefix", &remote_object);
|
||||
opts.set("max-keys", "1000");
|
||||
let mut key_marker = String::new();
|
||||
let mut version_id_marker = String::new();
|
||||
let mut matched_version = None;
|
||||
let mut saw_unproven_candidate = false;
|
||||
|
||||
loop {
|
||||
let versions = self
|
||||
.client
|
||||
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
|
||||
.await?;
|
||||
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
|
||||
let mut stat_opts = GetObjectOptions::default();
|
||||
stat_opts.version_id.clone_from(&version.version_id);
|
||||
let info = self.client.stat_object(&self.bucket, &remote_object, &stat_opts).await?;
|
||||
let mut metadata = info.user_metadata;
|
||||
for (name, value) in &info.metadata {
|
||||
if (name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
|
||||
|| name
|
||||
.as_str()
|
||||
.starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
|
||||
&& let Ok(value) = value.to_str()
|
||||
{
|
||||
metadata.insert(name.as_str().to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
if transition_candidate_metadata_matches(&metadata, identity)? {
|
||||
if matched_version.is_some() {
|
||||
return Ok(TransitionCandidateProbe::Ambiguous);
|
||||
}
|
||||
matched_version = Some(version.version_id.clone());
|
||||
} else {
|
||||
saw_unproven_candidate = true;
|
||||
}
|
||||
}
|
||||
if !versions.is_truncated {
|
||||
if matched_version.is_none() && saw_unproven_candidate {
|
||||
return Ok(TransitionCandidateProbe::Unsupported);
|
||||
}
|
||||
let candidates = TransitionCandidateVersions {
|
||||
version_id: matched_version,
|
||||
ambiguous: false,
|
||||
};
|
||||
return classify_transition_candidates(candidates, bucket_versioning);
|
||||
}
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
|
||||
}
|
||||
opts.set("prefix", &self.get_dest(object));
|
||||
opts.set("max-keys", "2");
|
||||
self.client.list_object_versions_query(&self.bucket, &opts, "", "", "").await
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_candidate_metadata_matches(
|
||||
metadata: &HashMap<String, String>,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<bool, std::io::Error> {
|
||||
use rustfs_utils::http::metadata_compat::{
|
||||
SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITION_TRANSACTION_ID, contains_key_str, get_consistent_str,
|
||||
};
|
||||
|
||||
let transaction_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID);
|
||||
let destination_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
|
||||
if transaction_id.is_none() || destination_id.is_none() {
|
||||
if contains_key_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID)
|
||||
|| contains_key_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
|
||||
{
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"transition candidate identity metadata is empty or conflicting",
|
||||
));
|
||||
}
|
||||
return Ok(false);
|
||||
}
|
||||
let expected_transaction_id = identity.transaction_id.to_string();
|
||||
let expected_destination_id = rustfs_utils::crypto::hex(identity.destination_id);
|
||||
Ok(transaction_id == Some(expected_transaction_id.as_str()) && destination_id == Some(expected_destination_id.as_str()))
|
||||
}
|
||||
|
||||
fn classify_transition_candidates(
|
||||
candidates: TransitionCandidateVersions,
|
||||
fn classify_transition_candidate_versions(
|
||||
remote_object: &str,
|
||||
bucket_versioning: RemoteBucketVersioning,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let probe = candidates.classify(bucket_versioning);
|
||||
if let TransitionCandidateProbe::VersionedPresent(version_id) = &probe {
|
||||
validate_remote_version_id(version_id)?;
|
||||
}
|
||||
Ok(probe)
|
||||
}
|
||||
|
||||
fn advance_version_markers(
|
||||
key_marker: &mut String,
|
||||
version_id_marker: &mut String,
|
||||
versions: &ListVersionsResult,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let next_markers = (&versions.next_key_marker, &versions.next_version_id_marker);
|
||||
if next_markers == (&*key_marker, &*version_id_marker) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"ListObjectVersions pagination markers did not advance",
|
||||
));
|
||||
}
|
||||
key_marker.clone_from(&versions.next_key_marker);
|
||||
version_id_marker.clone_from(&versions.next_version_id_marker);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TransitionCandidateVersions {
|
||||
version_id: Option<String>,
|
||||
ambiguous: bool,
|
||||
}
|
||||
|
||||
impl TransitionCandidateVersions {
|
||||
fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
|
||||
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
|
||||
if self.version_id.is_some() {
|
||||
self.ambiguous = true;
|
||||
return;
|
||||
}
|
||||
self.version_id = Some(version.version_id.clone());
|
||||
}
|
||||
) -> TransitionCandidateProbe {
|
||||
if versions.is_truncated {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
fn is_ambiguous(&self) -> bool {
|
||||
self.ambiguous
|
||||
if versions.delete_markers.iter().any(|marker| marker.key == remote_object) {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
|
||||
if self.ambiguous {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
let Some(version_id) = self.version_id else {
|
||||
return TransitionCandidateProbe::Missing;
|
||||
};
|
||||
let mut exact_versions = versions.versions.iter().filter(|version| version.key == remote_object);
|
||||
let Some(version) = exact_versions.next() else {
|
||||
return TransitionCandidateProbe::Missing;
|
||||
};
|
||||
if exact_versions.next().is_some() {
|
||||
return TransitionCandidateProbe::Ambiguous;
|
||||
}
|
||||
|
||||
match bucket_versioning {
|
||||
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
|
||||
RemoteBucketVersioning::Suspended if version_id == "null" => TransitionCandidateProbe::VersionedPresent(version_id),
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version_id.is_empty() => {
|
||||
TransitionCandidateProbe::VersionedPresent(version_id)
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
|
||||
match bucket_versioning {
|
||||
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
|
||||
RemoteBucketVersioning::Suspended if version.version_id == "null" => {
|
||||
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version.version_id.is_empty() => {
|
||||
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
|
||||
}
|
||||
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -423,161 +275,72 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_pages(bucket_versioning: RemoteBucketVersioning, pages: &[ListVersionsResult]) -> TransitionCandidateProbe {
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
for page in pages {
|
||||
candidates.extend("archive/object", page);
|
||||
}
|
||||
candidates.classify(bucket_versioning)
|
||||
}
|
||||
|
||||
fn candidate_identity() -> TransitionCandidateIdentity {
|
||||
TransitionCandidateIdentity {
|
||||
transaction_id: uuid::Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").unwrap(),
|
||||
destination_id: [0x5a; 32],
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate_metadata(identity: TransitionCandidateIdentity) -> HashMap<String, String> {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
identity.transaction_id.to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(identity.destination_id),
|
||||
);
|
||||
metadata
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_identity_requires_exact_compatible_metadata() {
|
||||
let identity = candidate_identity();
|
||||
let metadata = candidate_metadata(identity);
|
||||
assert!(transition_candidate_metadata_matches(&metadata, identity).unwrap());
|
||||
|
||||
let mut adjacent = metadata.clone();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut adjacent,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
uuid::Uuid::parse_str("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
);
|
||||
assert!(!transition_candidate_metadata_matches(&adjacent, identity).unwrap());
|
||||
|
||||
assert!(!transition_candidate_metadata_matches(&HashMap::new(), identity).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_identity_rejects_conflicting_compatibility_keys() {
|
||||
let identity = candidate_identity();
|
||||
let mut metadata = candidate_metadata(identity);
|
||||
metadata.insert(
|
||||
rustfs_utils::http::metadata_compat::internal_key_rustfs(
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
),
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
);
|
||||
assert!(transition_candidate_metadata_matches(&metadata, identity).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_classifier_is_fail_closed() {
|
||||
assert_eq!(
|
||||
classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[], &[], false)],),
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Disabled,
|
||||
&list_versions(&[], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[("archive/object", "")], &[], false)],),
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Disabled,
|
||||
&list_versions(&[("archive/object", "")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::UnversionedPresent
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[list_versions(&[("archive/object", "version-a")], &[], false)],
|
||||
&list_versions(&[("archive/object", "version-a")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Suspended,
|
||||
&[list_versions(&[("archive/object", "null")], &[], false)],
|
||||
&list_versions(&[("archive/object", "null")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("null".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(RemoteBucketVersioning::Enabled, &[list_versions(&[("archive/object", "")], &[], false)],),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[list_versions(
|
||||
&[("archive/object", "version-a"), ("archive/object", "version-b")],
|
||||
&[],
|
||||
false,
|
||||
)],
|
||||
&list_versions(&[("archive/object", "")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_reconciles_all_pages_and_ignores_delete_markers() {
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[
|
||||
list_versions(&[], &[("archive/object", "marker-a")], true),
|
||||
list_versions(&[("archive/object", "version-a"), ("archive/object-adjacent", "unrelated"),], &[], false,),
|
||||
],
|
||||
),
|
||||
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
classify_pages(
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&[
|
||||
list_versions(&[("archive/object", "version-a")], &[], true),
|
||||
list_versions(&[("archive/object", "version-b")], &[], false),
|
||||
],
|
||||
&list_versions(&[("archive/object", "version-a"), ("archive/object", "version-b")], &[], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a")], &[("archive/object", "marker-a")], false),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
assert_eq!(
|
||||
classify_transition_candidate_versions(
|
||||
"archive/object",
|
||||
RemoteBucketVersioning::Enabled,
|
||||
&list_versions(&[("archive/object", "version-a")], &[], true),
|
||||
),
|
||||
TransitionCandidateProbe::Ambiguous
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_pagination_advances_both_markers() {
|
||||
let mut key_marker = "old-key".to_string();
|
||||
let mut version_id_marker = "old-version".to_string();
|
||||
let page = ListVersionsResult {
|
||||
next_key_marker: "next-key".to_string(),
|
||||
next_version_id_marker: "next-version".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
|
||||
.expect("new ListObjectVersions markers should advance pagination");
|
||||
assert_eq!(key_marker, "next-key");
|
||||
assert_eq!(version_id_marker, "next-version");
|
||||
|
||||
let err = advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
|
||||
.expect_err("repeated ListObjectVersions markers must fail closed");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_candidate_probe_rejects_untrusted_version_ids() {
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
candidates.extend("archive/object", &list_versions(&[("archive/object", "version\ninjection")], &[], false));
|
||||
|
||||
let err = classify_transition_candidates(candidates, RemoteBucketVersioning::Enabled)
|
||||
.expect_err("control characters in listed version IDs must fail closed");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -634,7 +397,12 @@ impl WarmBackend for WarmBackendS3 {
|
||||
|
||||
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let bucket_versioning = self.remote_bucket_versioning().await?;
|
||||
self.probe_transition_candidate_versions(object, bucket_versioning).await
|
||||
let versions = self.list_transition_candidate_versions(object).await?;
|
||||
Ok(classify_transition_candidate_versions(
|
||||
&self.get_dest(object),
|
||||
bucket_versioning,
|
||||
&versions,
|
||||
))
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
@@ -646,16 +414,3 @@ impl WarmBackend for WarmBackendS3 {
|
||||
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TransitionCandidateReconciler for WarmBackendS3 {
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
identity: TransitionCandidateIdentity,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
let bucket_versioning = self.remote_bucket_versioning().await?;
|
||||
self.probe_transition_candidate_identity(object, identity, bucket_versioning)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -547,8 +547,6 @@ pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo
|
||||
&& left.transitioned_objname == right.transitioned_objname
|
||||
&& left.transition_tier == right.transition_tier
|
||||
&& left.transition_version_id == right.transition_version_id
|
||||
&& left.transition_version == right.transition_version
|
||||
&& left.transition_version_state == right.transition_version_state
|
||||
&& left.expire_restored == right.expire_restored
|
||||
&& left.size == right.size
|
||||
&& left.mod_time == right.mod_time
|
||||
|
||||
@@ -505,9 +505,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
fn file_info_has_encryption_metadata(meta: &FileInfo) -> bool {
|
||||
meta.metadata
|
||||
.keys()
|
||||
.any(|name| http::is_encryption_metadata_key(name) || http::is_sse_header(name))
|
||||
meta.metadata.keys().any(|name| http::is_object_encryption_marker(name))
|
||||
}
|
||||
|
||||
fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool {
|
||||
@@ -578,13 +576,6 @@ impl SetDisks {
|
||||
Self::update_hash_str(hasher, &meta.transition_tier);
|
||||
Self::update_hash_str(hasher, &meta.transitioned_objname);
|
||||
Self::update_hash_optional_uuid(hasher, meta.transition_version_id);
|
||||
Self::update_hash_optional_str(hasher, meta.transition_version.as_deref());
|
||||
hasher.update([match meta.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => 0,
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled => 1,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull => 2,
|
||||
rustfs_filemeta::TransitionVersionState::Exact => 3,
|
||||
}]);
|
||||
Self::update_hash_optional_u32(hasher, meta.mode);
|
||||
Self::update_hash_optional_u64(hasher, meta.written_by_version);
|
||||
|
||||
|
||||
@@ -143,15 +143,17 @@ use rustfs_object_capacity::capacity_scope::{
|
||||
CapacityScope, CapacityScopeDisk, current_dirty_generation, record_capacity_scope, record_global_dirty_scope,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
#[cfg(test)]
|
||||
use rustfs_utils::http::SSEC_ALGORITHM_HEADER;
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
|
||||
use rustfs_utils::http::headers::{
|
||||
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE,
|
||||
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, contains_key_str,
|
||||
get_header_map, get_str, insert_str, is_encryption_metadata_key, remove_header_map,
|
||||
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC,
|
||||
SUFFIX_RESTORE_OPERATION_ID, contains_key_str, get_header_map, get_str, insert_str, is_object_encryption_marker,
|
||||
remove_header_map,
|
||||
};
|
||||
use rustfs_utils::{
|
||||
HashAlgorithm,
|
||||
@@ -407,10 +409,7 @@ pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap<String, S
|
||||
}
|
||||
|
||||
fn should_persist_encryption_original_size(metadata: &HashMap<String, String>) -> bool {
|
||||
metadata.keys().any(|key| is_encryption_metadata_key(key))
|
||||
|| metadata.contains_key(SSEC_ALGORITHM_HEADER)
|
||||
|| metadata.contains_key(SSEC_KEY_HEADER)
|
||||
|| metadata.contains_key(SSEC_KEY_MD5_HEADER)
|
||||
metadata.keys().any(|key| is_object_encryption_marker(key))
|
||||
}
|
||||
|
||||
/// Per-set memoized capacity dirty scope.
|
||||
@@ -690,8 +689,6 @@ mod ops;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
||||
mod read;
|
||||
mod replication;
|
||||
pub(crate) mod shard_source;
|
||||
|
||||
@@ -25,10 +25,7 @@ use crate::set_disk::read::GetObjectDownstreamWriter;
|
||||
|
||||
use crate::bucket::lifecycle::{
|
||||
tier_delete_journal::{persist_tier_delete_journal_entry, remove_tier_delete_journal_entry},
|
||||
tier_sweeper::{
|
||||
Jentry, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_lease_idempotent,
|
||||
delete_object_from_remote_tier_with_lease_idempotent,
|
||||
},
|
||||
tier_sweeper::{Jentry, RemoteTierDeleteOutcome, delete_object_from_remote_tier_with_lease_idempotent},
|
||||
transition_transaction::{
|
||||
TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
|
||||
TransitionTransactionInit, TransitionTransactionState, delete_transition_transaction_record,
|
||||
@@ -42,6 +39,7 @@ use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppress
|
||||
use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
|
||||
use crate::store::ECStore;
|
||||
use futures::FutureExt as _;
|
||||
use http::HeaderValue;
|
||||
use std::future::Future;
|
||||
|
||||
fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Erasure> {
|
||||
@@ -49,6 +47,17 @@ fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Er
|
||||
.map_err(Error::from)
|
||||
}
|
||||
|
||||
async fn get_object_reader_with_context(
|
||||
ctx: &InstanceContext,
|
||||
reader: Box<dyn AsyncRead + Unpin + Send + Sync>,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
object_info: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
headers: &HeaderMap<HeaderValue>,
|
||||
) -> Result<(GetObjectReader, usize, i64)> {
|
||||
GetObjectReader::new_with_resolver(reader, range, object_info, opts, headers, ctx.object_encryption_resolver()).await
|
||||
}
|
||||
|
||||
/// Length of the full plaintext body when — and only when — this read's output
|
||||
/// is exactly the object's complete plaintext, so the app-layer body cache may
|
||||
/// serve it in place of the erasure read.
|
||||
@@ -707,7 +716,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
size_bucket,
|
||||
);
|
||||
record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket);
|
||||
let (mut reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?;
|
||||
let (mut reader, _offset, _length) =
|
||||
get_object_reader_with_context(&self.ctx, stream, range, &object_info, opts, &h).await?;
|
||||
// Carry the hook probe result so the app layer skips its
|
||||
// now-redundant lookup on the streaming miss path (ODC-16).
|
||||
reader.body_source = body_source;
|
||||
@@ -745,7 +755,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
let (rd, wd) = tokio::io::duplex(duplex_buffer_size);
|
||||
debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer");
|
||||
|
||||
let (mut reader, offset, length) = GetObjectReader::new(Box::new(rd), range, &object_info, opts, &h).await?;
|
||||
let (mut reader, offset, length) =
|
||||
get_object_reader_with_context(&self.ctx, Box::new(rd), range, &object_info, opts, &h).await?;
|
||||
// Carry the hook probe result so the app layer skips its now-redundant
|
||||
// lookup on the streaming miss path (ODC-16).
|
||||
reader.body_source = body_source;
|
||||
@@ -1666,11 +1677,7 @@ pub(crate) async fn cleanup_uncommitted_transition_upload(
|
||||
cleanup_version: &str,
|
||||
version_id_exact: bool,
|
||||
) -> std::io::Result<RemoteTierDeleteOutcome> {
|
||||
if version_id_exact {
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent(object, cleanup_version, lease).await
|
||||
} else {
|
||||
delete_object_from_remote_tier_with_lease_idempotent(object, cleanup_version, lease, false).await
|
||||
}
|
||||
delete_object_from_remote_tier_with_lease_idempotent(object, cleanup_version, lease, version_id_exact).await
|
||||
}
|
||||
|
||||
fn log_transition_upload_cleanup_failure(lease: &TierOperationLease, object: &str, cleanup_version: &str, err: &std::io::Error) {
|
||||
@@ -1807,7 +1814,7 @@ impl Drop for TransitionUploadCleanup {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cleanup_rejected_transition_upload_durably(
|
||||
async fn cleanup_rejected_transition_upload_durably(
|
||||
lease: &TierOperationLease,
|
||||
object: &str,
|
||||
cleanup_version: &str,
|
||||
@@ -1820,13 +1827,6 @@ pub(crate) async fn cleanup_rejected_transition_upload_durably(
|
||||
tier_name: lease.tier_name().to_string(),
|
||||
backend_identity: Some(lease.backend_identity()),
|
||||
version_id_exact,
|
||||
version_state: if !version_id_exact {
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled
|
||||
} else if cleanup_version == "null" {
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull
|
||||
} else {
|
||||
rustfs_filemeta::TransitionVersionState::Exact
|
||||
},
|
||||
};
|
||||
|
||||
let journal_error = if let Some(api) = api.as_ref() {
|
||||
@@ -1986,83 +1986,12 @@ async fn advance_and_save_transition_transaction(
|
||||
next: TransitionTransactionState,
|
||||
remote_version: Option<TransitionRemoteVersion>,
|
||||
) -> Result<()> {
|
||||
#[cfg(test)]
|
||||
record_transition_uploaded_save_attempt(transaction, next);
|
||||
transaction
|
||||
.advance(transaction.fence(), next, remote_version)
|
||||
.map_err(Error::other)?;
|
||||
save_transition_transaction_if_available(api, transaction).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct TransitionUploadedSaveProbeState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
attempts: std::sync::atomic::AtomicUsize,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct TransitionUploadedSaveProbe {
|
||||
state: Arc<TransitionUploadedSaveProbeState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static TRANSITION_UPLOADED_SAVE_PROBE: std::sync::OnceLock<std::sync::Mutex<Option<Arc<TransitionUploadedSaveProbeState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl TransitionUploadedSaveProbe {
|
||||
fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(TransitionUploadedSaveProbeState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
attempts: std::sync::atomic::AtomicUsize::new(0),
|
||||
});
|
||||
let mut slot = TRANSITION_UPLOADED_SAVE_PROBE
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition uploaded-save probe mutex should not poison");
|
||||
assert!(slot.is_none(), "transition uploaded-save probe must be installed by one test at a time");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
drop(slot);
|
||||
Self { state }
|
||||
}
|
||||
|
||||
fn attempts(&self) -> usize {
|
||||
self.state.attempts.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for TransitionUploadedSaveProbe {
|
||||
fn drop(&mut self) {
|
||||
let mut slot = TRANSITION_UPLOADED_SAVE_PROBE
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition uploaded-save probe mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn record_transition_uploaded_save_attempt(transaction: &TransitionTransaction, next: TransitionTransactionState) {
|
||||
if next != TransitionTransactionState::Uploaded {
|
||||
return;
|
||||
}
|
||||
let state = TRANSITION_UPLOADED_SAVE_PROBE
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition uploaded-save probe mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|state| state.bucket == transaction.source.bucket && state.object == transaction.source.object)
|
||||
.cloned();
|
||||
if let Some(state) = state {
|
||||
state.attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_transition_transaction_if_available(api: Option<&Arc<ECStore>>, transaction_id: Uuid) -> Result<()> {
|
||||
if let Some(api) = api {
|
||||
return delete_transition_transaction_record(api.clone(), transaction_id).await;
|
||||
@@ -2313,78 +2242,11 @@ async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCo
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn persisted_transition_version(
|
||||
remote_version: &str,
|
||||
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
|
||||
persisted_transition_version_with_gate(remote_version, remote_version_state_writer_enabled())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn remote_version_state_writer_enabled() -> bool {
|
||||
remote_version_state_writer_fleet_proof().is_some()
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof() -> Option<crate::services::notification_sys::RemoteVersionStateFleetProofToken> {
|
||||
remote_version_state_writer_requested()
|
||||
.then(crate::services::notification_sys::acquire_remote_version_state_fleet_proof)
|
||||
.flatten()
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_requested() -> bool {
|
||||
remote_version_state_writer_enabled_for(
|
||||
rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE,
|
||||
),
|
||||
rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
|
||||
),
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof_matches(
|
||||
proof: &crate::services::notification_sys::RemoteVersionStateFleetProofToken,
|
||||
) -> bool {
|
||||
remote_version_state_writer_fleet_proof_matches_for(
|
||||
remote_version_state_writer_requested(),
|
||||
crate::services::notification_sys::remote_version_state_fleet_proof_matches(proof),
|
||||
)
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_fleet_proof_matches_for(requested: bool, fleet_proof_matches: bool) -> bool {
|
||||
requested && fleet_proof_matches
|
||||
}
|
||||
|
||||
fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool, fleet_proof_valid: bool) -> bool {
|
||||
requested && fleet_confirmed && fleet_proof_valid
|
||||
}
|
||||
|
||||
fn persisted_transition_version_with_gate(
|
||||
remote_version: &str,
|
||||
remote_version_state_writer_enabled: bool,
|
||||
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
|
||||
fn parse_transition_version_id(remote_version: &str) -> std::result::Result<Option<Uuid>, uuid::Error> {
|
||||
if remote_version.is_empty() {
|
||||
return Ok((None, rustfs_filemeta::TransitionVersionState::KnownDisabled));
|
||||
}
|
||||
|
||||
match Uuid::parse_str(remote_version) {
|
||||
Ok(version_id) if version_id.is_nil() => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier returned a nil object version ID",
|
||||
)),
|
||||
Ok(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
|
||||
Err(_) if !remote_version_state_writer_enabled => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"opaque remote tier versions require the operator-attested live fleet capability gate",
|
||||
)),
|
||||
Err(_) if remote_version == "null" => {
|
||||
Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::SuspendedNull))
|
||||
}
|
||||
Err(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
|
||||
return Ok(None);
|
||||
}
|
||||
Uuid::parse_str(remote_version).map(|version_id| (!version_id.is_nil()).then_some(version_id))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2622,20 +2484,16 @@ mod transition_upload_completion_tests {
|
||||
|
||||
#[cfg(test)]
|
||||
mod transition_version_id_tests {
|
||||
use super::{
|
||||
TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate,
|
||||
remote_version_state_writer_enabled_for, remote_version_state_writer_fleet_proof_matches_for,
|
||||
};
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
use super::{TransitionUploadCandidate, parse_transition_version_id};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn normalizes_persisted_unversioned_ids_and_preserves_put_constraints() {
|
||||
assert_eq!(parse_transition_version_id("").expect("empty remote version should be valid"), None);
|
||||
assert_eq!(
|
||||
persisted_transition_version("").expect("empty remote version identifies an unversioned tier"),
|
||||
(None, TransitionVersionState::KnownDisabled)
|
||||
parse_transition_version_id(&Uuid::nil().to_string()).expect("nil remote version should be valid"),
|
||||
None
|
||||
);
|
||||
assert!(persisted_transition_version(&Uuid::nil().to_string()).is_err());
|
||||
let nil_put_response = Uuid::nil().to_string();
|
||||
let nil_candidate = TransitionUploadCandidate::from_put_response(nil_put_response.clone());
|
||||
assert_eq!(nil_candidate.cleanup_version(), nil_put_response);
|
||||
@@ -2647,14 +2505,12 @@ mod transition_version_id_tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_uuid_and_gates_opaque_remote_ids() {
|
||||
fn preserves_valid_remote_id_and_rejects_invalid_text() {
|
||||
let version_id = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
persisted_transition_version(&version_id.to_string()).expect("UUID remote version"),
|
||||
(Some(version_id.to_string()), TransitionVersionState::Exact)
|
||||
parse_transition_version_id(&version_id.to_string()).expect("UUID remote version should be valid"),
|
||||
Some(version_id)
|
||||
);
|
||||
assert!(persisted_transition_version("null").is_err());
|
||||
assert!(persisted_transition_version("opaque-version-token").is_err());
|
||||
assert_eq!(
|
||||
TransitionUploadCandidate::from_put_response(version_id.to_string()).cleanup_version(),
|
||||
version_id.to_string()
|
||||
@@ -2663,64 +2519,7 @@ mod transition_version_id_tests {
|
||||
TransitionUploadCandidate::from_put_response("opaque-version-token".to_string()).cleanup_version(),
|
||||
"opaque-version-token"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_writer_requires_request_and_fleet_confirmation() {
|
||||
for (case, requested, fleet_confirmed, fleet_proof_valid, expected) in [
|
||||
("old defaults", false, false, false, false),
|
||||
("missing fleet confirmation", true, false, true, false),
|
||||
("missing local opt-in", false, true, true, false),
|
||||
("missing fleet proof", true, true, false, false),
|
||||
("explicitly unconfirmed fleet", true, false, true, false),
|
||||
("rolled-back writer", false, true, true, false),
|
||||
("fully upgraded fleet", true, true, true, true),
|
||||
] {
|
||||
assert_eq!(
|
||||
remote_version_state_writer_enabled_for(requested, fleet_confirmed, fleet_proof_valid),
|
||||
expected,
|
||||
"{case}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_commit_rechecks_operator_gate_and_live_proof() {
|
||||
for (case, requested, fleet_proof_matches, expected) in [
|
||||
("operator gate closed", false, true, false),
|
||||
("fleet proof changed", true, false, false),
|
||||
("current authorization", true, true, true),
|
||||
] {
|
||||
assert_eq!(
|
||||
remote_version_state_writer_fleet_proof_matches_for(requested, fleet_proof_matches),
|
||||
expected,
|
||||
"{case}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fleet_gate_enables_null_and_opaque_remote_version_states() {
|
||||
for (remote_version, expected) in [
|
||||
("null", (Some("null".to_string()), TransitionVersionState::SuspendedNull)),
|
||||
(
|
||||
"opaque-version-token",
|
||||
(Some("opaque-version-token".to_string()), TransitionVersionState::Exact),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
persisted_transition_version_with_gate(remote_version, false).is_err(),
|
||||
"missing fleet confirmation must reject {remote_version:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
persisted_transition_version_with_gate(remote_version, true).expect("fleet-confirmed state must be persisted"),
|
||||
expected
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
persisted_transition_version_with_gate("", true).expect("empty remote version identifies an unversioned tier"),
|
||||
(None, TransitionVersionState::KnownDisabled)
|
||||
);
|
||||
assert!(parse_transition_version_id("not-a-uuid").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3880,16 +3679,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let dest_obj = transaction.remote_object.clone();
|
||||
let mut transition_meta = (*oi.user_defined).clone();
|
||||
transition_meta.insert("name".to_string(), object.to_string());
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut transition_meta,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
|
||||
transaction.transaction_id.to_string(),
|
||||
);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut transition_meta,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(transaction.backend_fingerprint),
|
||||
);
|
||||
|
||||
if let Some(content_type) = oi.content_type.as_ref().filter(|value| !value.is_empty()) {
|
||||
transition_meta.insert(CONTENT_TYPE.to_ascii_lowercase(), content_type.clone());
|
||||
@@ -4000,24 +3789,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
|
||||
return Err(err.into());
|
||||
}
|
||||
let fleet_proof = remote_version_state_writer_fleet_proof();
|
||||
let remote_version_requires_fleet_proof =
|
||||
!candidate.remote_version().is_empty() && Uuid::parse_str(candidate.remote_version()).is_err();
|
||||
let (transition_version_id, transition_version_state) =
|
||||
match persisted_transition_version_with_gate(candidate.remote_version(), fleet_proof.is_some()) {
|
||||
Ok(version) => version,
|
||||
Err(err) => {
|
||||
let cleanup_api = transition_cleanup_store(&self.ctx).await;
|
||||
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
|
||||
return Err(StorageError::Io(std::io::Error::other(format!(
|
||||
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
|
||||
))));
|
||||
}
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
if let Err(err) = advance_and_save_transition_transaction(
|
||||
transaction_api.as_ref(),
|
||||
&mut transaction,
|
||||
@@ -4035,6 +3806,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
|
||||
return Err(err);
|
||||
}
|
||||
let transition_version_id = match parse_transition_version_id(candidate.remote_version()) {
|
||||
Ok(version_id) => version_id,
|
||||
Err(err) => {
|
||||
if upload_cleanup.cleanup().await.is_ok() {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
}
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
|
||||
let mut commit_opts = opts.clone();
|
||||
commit_opts.no_lock = true;
|
||||
@@ -4092,11 +3873,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
current_fi.transition_status = TRANSITION_COMPLETE.to_string();
|
||||
current_fi.transitioned_objname = dest_obj;
|
||||
current_fi.transition_tier = opts.transition.tier.clone();
|
||||
current_fi.transition_version_id = transition_version_id
|
||||
.as_deref()
|
||||
.and_then(|version_id| Uuid::parse_str(version_id).ok());
|
||||
current_fi.transition_version = transition_version_id;
|
||||
current_fi.transition_version_state = transition_version_state;
|
||||
current_fi.transition_version_id = transition_version_id;
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut current_fi.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
@@ -4133,21 +3910,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
#[cfg(test)]
|
||||
pause_transition_commit(bucket, object, TransitionCommitPause::AfterLeaseValidation).await;
|
||||
// This check is the fleet-proof lease linearization point. Revocation
|
||||
// blocks later commits; an already-authorized local quorum commit is
|
||||
// allowed to finish without holding a synchronous lock across I/O.
|
||||
if remote_version_requires_fleet_proof
|
||||
&& !fleet_proof
|
||||
.as_ref()
|
||||
.is_some_and(remote_version_state_writer_fleet_proof_matches)
|
||||
{
|
||||
drop(transition_lock_guard);
|
||||
if upload_cleanup.cleanup().await.is_ok() {
|
||||
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
|
||||
.await;
|
||||
}
|
||||
return Err(Error::other("remote version state fleet capability changed during transition"));
|
||||
}
|
||||
if let Err(err) = advance_and_save_transition_transaction(
|
||||
transaction_api.as_ref(),
|
||||
&mut transaction,
|
||||
@@ -4527,6 +4289,61 @@ mod erasure_construction_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod object_encryption_resolver_wiring_tests {
|
||||
use super::*;
|
||||
use crate::object_api::{EncryptionResolutionError, ObjectEncryptionResolver, ReadEncryptionMaterial, ReadEncryptionRequest};
|
||||
use std::io::Cursor;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
struct CountingResolver {
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObjectEncryptionResolver for CountingResolver {
|
||||
async fn resolve_read_material(
|
||||
&self,
|
||||
_request: ReadEncryptionRequest<'_>,
|
||||
) -> std::result::Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
|
||||
self.calls.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_reader_forwards_instance_resolver() {
|
||||
let resolver = Arc::new(CountingResolver {
|
||||
calls: AtomicUsize::new(0),
|
||||
});
|
||||
let ctx = InstanceContext::new();
|
||||
assert!(
|
||||
ctx.set_object_encryption_resolver(resolver.clone()).is_ok(),
|
||||
"fresh context should accept resolver"
|
||||
);
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 1,
|
||||
user_defined: Arc::new(HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = get_object_reader_with_context(
|
||||
&ctx,
|
||||
Box::new(Cursor::new(Vec::<u8>::new())),
|
||||
None,
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&HeaderMap::new(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_err(), "resolver returning no material must fail closed");
|
||||
assert_eq!(resolver.calls.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
|
||||
//! Shared hermetic `SetDisks` construction for the ops tests below: the
|
||||
@@ -4920,33 +4737,6 @@ mod transition_commit_failure_tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_unsupported_remote_versions_are_cleaned_up() {
|
||||
for remote_version in ["null", "opaque-version-token"] {
|
||||
let manager = TierConfigMgr::new();
|
||||
let backend = register_mock_tier(&manager, "WARM").await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&manager, "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available");
|
||||
let candidate = TransitionUploadCandidate::from_put_response(remote_version.to_string());
|
||||
|
||||
persisted_transition_version(candidate.remote_version()).expect_err("unsupported writer version must fail closed");
|
||||
cleanup_rejected_transition_upload_durably(
|
||||
&lease,
|
||||
"remote/object",
|
||||
candidate.cleanup_version(),
|
||||
candidate.cleanup_version_is_exact(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("rejected remote upload must be cleaned up");
|
||||
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![("remote/object".to_string(), candidate.cleanup_version().to_string())]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[serial_test::serial(restore_multipart_failure_point)]
|
||||
async fn multipart_restore_aborts_every_post_create_failure() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -6894,45 +6684,6 @@ mod transition_upload_integrity_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn unversioned_remote_version_is_persisted_without_version_id() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "transition-unversioned-tier-bucket";
|
||||
let object = "object.bin";
|
||||
let payload = b"unversioned remote tier must commit without a version id".repeat(1024);
|
||||
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
backend.set_put_remote_version(Some(String::new())).await;
|
||||
let save_probe = TransitionUploadedSaveProbe::install(bucket, object);
|
||||
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect("an unversioned remote version must commit");
|
||||
let (fi, _, _) = set_disks
|
||||
.get_object_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("committed unversioned transition metadata should be readable");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version, None);
|
||||
assert_eq!(fi.transition_version_state, rustfs_filemeta::TransitionVersionState::KnownDisabled);
|
||||
assert_eq!(save_probe.attempts(), 1);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn opaque_remote_version_is_cleaned_before_parse_failure() {
|
||||
@@ -6948,7 +6699,7 @@ mod transition_upload_integrity_tests {
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect_err("an opaque remote version must fail closed until the capability gate is active");
|
||||
.expect_err("an unparseable remote version must fail closed");
|
||||
let removed_versions = backend.remove_versions().await;
|
||||
assert_eq!(removed_versions.len(), 1);
|
||||
assert_eq!(removed_versions[0].1, "opaque-version-token");
|
||||
@@ -6956,38 +6707,6 @@ mod transition_upload_integrity_tests {
|
||||
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn nil_remote_version_is_cleaned_exactly_before_transaction_persistence() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "transition-nil-version-bucket";
|
||||
let object = "object.bin";
|
||||
let payload = b"nil remote version must retain local data".repeat(1024);
|
||||
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let remote_version = Uuid::nil().to_string();
|
||||
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
|
||||
backend.set_put_remote_version(Some(remote_version.clone())).await;
|
||||
let save_probe = TransitionUploadedSaveProbe::install(bucket, object);
|
||||
|
||||
set_disks
|
||||
.transition_object(bucket, object, &transition_options(&original, tier_name))
|
||||
.await
|
||||
.expect_err("a nil remote version must fail closed before transaction persistence");
|
||||
let put_versions = backend.put_versions().await;
|
||||
let removed_versions = backend.remove_versions().await;
|
||||
assert_eq!(removed_versions, put_versions);
|
||||
assert_eq!(removed_versions.len(), 1);
|
||||
assert_eq!(
|
||||
removed_versions.first().map(|(_, version)| version.as_str()),
|
||||
Some(remote_version.as_str())
|
||||
);
|
||||
assert_eq!(save_probe.attempts(), 0, "nil remote version must be rejected before saving Uploaded");
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
assert_eq!(backend.object_count().await, 0);
|
||||
assert_local_source_intact(&set_disks, bucket, object, &payload).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn authoritative_read_failure_after_upload_cleans_exact_candidate_and_preserves_source() {
|
||||
@@ -7335,36 +7054,23 @@ mod transition_source_identity_matrix_tests {
|
||||
let object = format!("identity-{index}.bin");
|
||||
let payload = vec![u8::try_from(index + 1).expect("matrix index should fit u8"); 1024 * 1024];
|
||||
let mut reader = PutObjReader::from_vec(payload);
|
||||
let source_version_id = Uuid::new_v4();
|
||||
let source_opts = ObjectOptions {
|
||||
version_id: Some(source_version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
let original = set_disks
|
||||
.put_object(bucket, &object, &mut reader, &source_opts)
|
||||
.put_object(bucket, &object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let (source, _, _) = set_disks
|
||||
.get_object_fileinfo(bucket, &object, &source_opts, true, false)
|
||||
.get_object_fileinfo(bucket, &object, &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
.expect("source metadata should resolve");
|
||||
assert_eq!(source.version_id, Some(source_version_id));
|
||||
assert_eq!(
|
||||
transition_source_identity(bucket, &object, &source, &source_opts, &get_raw_etag(&source.metadata))
|
||||
.expect("persisted versioned source identity should build")
|
||||
.version_mode,
|
||||
TransitionSourceVersionMode::Versioned
|
||||
);
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
versioned: true,
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.clone(),
|
||||
etag: original.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
@@ -7377,10 +7083,7 @@ mod transition_source_identity_matrix_tests {
|
||||
|
||||
let mut changed = source.clone();
|
||||
match field {
|
||||
IdentityField::VersionId => {
|
||||
changed.version_id = Some(Uuid::new_v4());
|
||||
changed.fresh = true;
|
||||
}
|
||||
IdentityField::VersionId => changed.version_id = Some(Uuid::new_v4()),
|
||||
IdentityField::DataDir => changed.data_dir = Some(Uuid::new_v4()),
|
||||
IdentityField::ModTime => {
|
||||
changed.mod_time = changed.mod_time.map(|value| value + time::Duration::nanoseconds(1));
|
||||
@@ -7390,27 +7093,17 @@ mod transition_source_identity_matrix_tests {
|
||||
changed.metadata.insert("etag".to_string(), format!("changed-{index}"));
|
||||
}
|
||||
}
|
||||
// Replace the object version list so VersionId drift removes the
|
||||
// accepted source version instead of appending a second version.
|
||||
changed.fresh = true;
|
||||
for disk in &disk_stores {
|
||||
disk.write_metadata("", bucket, &object, changed.clone())
|
||||
.await
|
||||
.expect("single-field metadata drift should be written");
|
||||
}
|
||||
let persisted_opts = ObjectOptions {
|
||||
version_id: changed.version_id.map(|version_id| version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
let (persisted, _, _) = set_disks
|
||||
.get_object_fileinfo(bucket, &object, &persisted_opts, true, false)
|
||||
.await
|
||||
.expect("drifted source metadata should resolve");
|
||||
put_barrier.release();
|
||||
|
||||
let result = transition.await.expect("transition task should not panic");
|
||||
assert!(result.is_err(), "transition must reject {field:?} drift");
|
||||
transition
|
||||
.await
|
||||
.expect("transition task should not panic")
|
||||
.expect_err("transition must reject a source whose identity changed after upload");
|
||||
let expected_attempts = index + 1;
|
||||
assert_eq!(backend.put_count().await, expected_attempts);
|
||||
assert_eq!(backend.remove_count().await, expected_attempts);
|
||||
@@ -7421,26 +7114,26 @@ mod transition_source_identity_matrix_tests {
|
||||
);
|
||||
|
||||
match field {
|
||||
IdentityField::VersionId => assert_ne!(source.version_id, persisted.version_id),
|
||||
IdentityField::DataDir => assert_ne!(source.data_dir, persisted.data_dir),
|
||||
IdentityField::ModTime => assert_ne!(source.mod_time, persisted.mod_time),
|
||||
IdentityField::Size => assert_ne!(source.size, persisted.size),
|
||||
IdentityField::Etag => assert_ne!(get_raw_etag(&source.metadata), get_raw_etag(&persisted.metadata)),
|
||||
IdentityField::VersionId => assert_ne!(source.version_id, changed.version_id),
|
||||
IdentityField::DataDir => assert_ne!(source.data_dir, changed.data_dir),
|
||||
IdentityField::ModTime => assert_ne!(source.mod_time, changed.mod_time),
|
||||
IdentityField::Size => assert_ne!(source.size, changed.size),
|
||||
IdentityField::Etag => assert_ne!(get_raw_etag(&source.metadata), get_raw_etag(&changed.metadata)),
|
||||
}
|
||||
if !matches!(field, IdentityField::VersionId) {
|
||||
assert_eq!(source.version_id, persisted.version_id);
|
||||
assert_eq!(source.version_id, changed.version_id);
|
||||
}
|
||||
if !matches!(field, IdentityField::DataDir) {
|
||||
assert_eq!(source.data_dir, persisted.data_dir);
|
||||
assert_eq!(source.data_dir, changed.data_dir);
|
||||
}
|
||||
if !matches!(field, IdentityField::ModTime) {
|
||||
assert_eq!(source.mod_time, persisted.mod_time);
|
||||
assert_eq!(source.mod_time, changed.mod_time);
|
||||
}
|
||||
if !matches!(field, IdentityField::Size) {
|
||||
assert_eq!(source.size, persisted.size);
|
||||
assert_eq!(source.size, changed.size);
|
||||
}
|
||||
if !matches!(field, IdentityField::Etag) {
|
||||
assert_eq!(get_raw_etag(&source.metadata), get_raw_etag(&persisted.metadata));
|
||||
assert_eq!(get_raw_etag(&source.metadata), get_raw_etag(&changed.metadata));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2016,15 +2016,11 @@ mod metadata_cache_tests {
|
||||
fi.size = 1;
|
||||
fi.erasure.index = 1;
|
||||
fi.metadata.insert("etag".to_string(), "etag-1".to_string());
|
||||
fi.add_object_part(1, "part-etag".to_string(), 1, fi.mod_time, 1, None, None);
|
||||
fi
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_with_fileinfo_rejects_positive_size_without_parts() {
|
||||
let mut fi = valid_test_fileinfo("object");
|
||||
fi.parts.clear();
|
||||
|
||||
let mut output = Vec::new();
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
"bucket",
|
||||
@@ -2032,7 +2028,7 @@ mod metadata_cache_tests {
|
||||
0,
|
||||
1,
|
||||
&mut output,
|
||||
fi,
|
||||
valid_test_fileinfo("object"),
|
||||
Vec::new(),
|
||||
&[],
|
||||
0,
|
||||
@@ -2123,6 +2119,12 @@ mod metadata_cache_tests {
|
||||
|
||||
let mut invalid_erasure = valid_test_fileinfo(object);
|
||||
invalid_erasure.erasure.block_size = 0;
|
||||
invalid_erasure.parts.push(ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
});
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
@@ -2157,7 +2159,6 @@ mod metadata_cache_tests {
|
||||
let object = "empty";
|
||||
let mut fi = valid_test_fileinfo(object);
|
||||
fi.size = 0;
|
||||
fi.parts.clear();
|
||||
|
||||
let mut output = Vec::new();
|
||||
SetDisks::get_object_with_fileinfo(
|
||||
@@ -2190,6 +2191,12 @@ mod metadata_cache_tests {
|
||||
let mut fi = valid_test_fileinfo(object);
|
||||
fi.erasure.block_size = 1;
|
||||
fi.erasure.distribution = vec![1, 2, 3, 4];
|
||||
fi.parts.push(ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let mut output = Vec::new();
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
|
||||
@@ -562,12 +562,10 @@ mod tests {
|
||||
},
|
||||
tier_sweeper::Jentry,
|
||||
transition_transaction::{
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
|
||||
TransitionOperatorProbe, TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode,
|
||||
TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
|
||||
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
|
||||
inspect_transition_transaction_for_operator, load_transition_transaction_record,
|
||||
recover_transition_transaction_records, save_transition_transaction_record,
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionRemoteVersion,
|
||||
TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit,
|
||||
TransitionTransactionState, load_transition_transaction_record, recover_transition_transaction_records,
|
||||
save_transition_transaction_record,
|
||||
},
|
||||
},
|
||||
client::transition_api::ReaderImpl,
|
||||
@@ -577,7 +575,6 @@ mod tests {
|
||||
services::tier::{
|
||||
test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
|
||||
tier::{TIER_CONFIG_FILE, TierConfigMgr},
|
||||
tier_config::{TierConfig, TierType, TierWasabi},
|
||||
tier_mutation_intent::{
|
||||
TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
|
||||
TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
|
||||
@@ -1166,37 +1163,6 @@ mod tests {
|
||||
.len()
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn register_operator_reconcile_test_tier(
|
||||
handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
tier_name: &str,
|
||||
) -> MockWarmBackend {
|
||||
let backend = MockWarmBackend::new();
|
||||
let mut manager = handle.write().await;
|
||||
manager.tiers.insert(
|
||||
tier_name.to_string(),
|
||||
TierConfig {
|
||||
version: "v1".to_string(),
|
||||
tier_type: TierType::Wasabi,
|
||||
name: tier_name.to_string(),
|
||||
wasabi: Some(TierWasabi {
|
||||
name: tier_name.to_string(),
|
||||
endpoint: "https://s3.wasabisys.com".to_string(),
|
||||
access_key: "test-access-key".to_string(),
|
||||
secret_key: "test-secret-key".to_string(),
|
||||
bucket: "mock-tier".to_string(),
|
||||
prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
|
||||
region: "us-east-1".to_string(),
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
manager
|
||||
.install_test_driver(tier_name, Box::new(backend.clone()))
|
||||
.expect("mock fallback tier driver should install");
|
||||
backend
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn wait_for_tier_delete_journal_recovery(
|
||||
store: Arc<crate::store::ECStore>,
|
||||
@@ -1345,16 +1311,14 @@ mod tests {
|
||||
version_id: "version-a".to_string(),
|
||||
tier_name: tier_a.to_string(),
|
||||
backend_identity: Some(identity_a),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
let entry_b = Jentry {
|
||||
obj_name: "remote-b".to_string(),
|
||||
version_id: "version-b".to_string(),
|
||||
tier_name: tier_b.to_string(),
|
||||
backend_identity: Some(identity_b),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
version_id_exact: false,
|
||||
};
|
||||
let remove_a = backend_a.arm_failing_remove_barrier().await;
|
||||
persist_tier_delete_journal_entry(store_a.clone(), &entry_a)
|
||||
@@ -2756,12 +2720,10 @@ mod tests {
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn transition_transaction_recovery_deletes_provider_recovered_unknown_upload() {
|
||||
let versioned_remote = uuid::Uuid::new_v4().to_string();
|
||||
let nil_remote = uuid::Uuid::nil().to_string();
|
||||
for (case, tier_name, remote_version) in [
|
||||
("missing", "TXPROBEMISSING", None),
|
||||
("unversioned", "TXPROBEUNVERSIONED", Some(String::new())),
|
||||
("versioned", "TXPROBEVERSIONED", Some(versioned_remote)),
|
||||
("nil-version", "TXPROBENILVERSION", Some(nil_remote)),
|
||||
] {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
|
||||
@@ -2843,154 +2805,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXOPERATOR";
|
||||
let backend = register_operator_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
.backend_identity();
|
||||
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
owner_epoch: uuid::Uuid::new_v4(),
|
||||
write_id: uuid::Uuid::new_v4(),
|
||||
source: TransitionSourceIdentity {
|
||||
bucket: "source-bucket".to_string(),
|
||||
object: "source-object".to_string(),
|
||||
version_id: None,
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: 1_770_000_000_000_000_000,
|
||||
size: 42,
|
||||
etag: "source-etag".to_string(),
|
||||
version_mode: TransitionSourceVersionMode::Unversioned,
|
||||
},
|
||||
tier_name: tier_name.to_string(),
|
||||
backend_fingerprint: backend_identity,
|
||||
not_after_unix_nanos: 1,
|
||||
})
|
||||
.expect("transaction should build");
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("transaction should enter unknown upload outcome state");
|
||||
|
||||
let remote_version = uuid::Uuid::new_v4().to_string();
|
||||
backend.set_put_remote_version(Some(remote_version.clone())).await;
|
||||
let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
|
||||
backend
|
||||
.put(
|
||||
&transaction.remote_object,
|
||||
ReaderImpl::Body(candidate.clone()),
|
||||
i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
|
||||
)
|
||||
.await
|
||||
.expect("mock backend should accept candidate");
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
|
||||
let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("operator inspection should probe the candidate");
|
||||
assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
|
||||
|
||||
let wrong_version = uuid::Uuid::new_v4().to_string();
|
||||
let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
|
||||
.await
|
||||
.expect("an exact delete of an absent version should be idempotent");
|
||||
assert_eq!(
|
||||
result.status.probe,
|
||||
TransitionOperatorProbe::VersionedPresent(remote_version.clone()),
|
||||
"an incorrect exact version must not delete the provider-confirmed candidate"
|
||||
);
|
||||
assert!(result.journal_observed_after_delete);
|
||||
load_transition_transaction_record(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("an incorrect exact version must retain the transaction journal");
|
||||
|
||||
let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
|
||||
.await
|
||||
.expect("operator-confirmed exact candidate should be deleted");
|
||||
assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
|
||||
assert!(result.journal_observed_after_delete);
|
||||
assert_eq!(backend.exact_remove_count(), 2);
|
||||
assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
|
||||
load_transition_transaction_record(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("candidate deletion must retain the transaction journal");
|
||||
|
||||
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
|
||||
.await
|
||||
.expect("a separately confirmed missing candidate should permit finalization");
|
||||
assert!(matches!(
|
||||
load_transition_transaction_record(store, transaction.transaction_id).await,
|
||||
Err(Error::ConfigNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn operator_finalize_retains_record_without_missing_proof() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
|
||||
let tier_name = "TXOPERATORFAIL";
|
||||
let backend = register_operator_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
|
||||
.await
|
||||
.expect("tier lease should resolve")
|
||||
.backend_identity();
|
||||
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
|
||||
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
|
||||
transaction_id: uuid::Uuid::new_v4(),
|
||||
owner_epoch: uuid::Uuid::new_v4(),
|
||||
write_id: uuid::Uuid::new_v4(),
|
||||
source: TransitionSourceIdentity {
|
||||
bucket: "source-bucket".to_string(),
|
||||
object: "source-object".to_string(),
|
||||
version_id: None,
|
||||
data_dir: uuid::Uuid::new_v4(),
|
||||
mod_time_unix_nanos: 1_770_000_000_000_000_000,
|
||||
size: 42,
|
||||
etag: "source-etag".to_string(),
|
||||
version_mode: TransitionSourceVersionMode::Unversioned,
|
||||
},
|
||||
tier_name: tier_name.to_string(),
|
||||
backend_fingerprint: backend_identity,
|
||||
not_after_unix_nanos: 1,
|
||||
})
|
||||
.expect("transaction should build");
|
||||
transaction
|
||||
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
|
||||
.expect("transaction should enter unknown upload outcome state");
|
||||
save_transition_transaction_record(store.clone(), &transaction)
|
||||
.await
|
||||
.expect("transaction record should persist");
|
||||
backend
|
||||
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
|
||||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
|
||||
Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
|
||||
));
|
||||
load_transition_transaction_record(store, transaction.transaction_id)
|
||||
.await
|
||||
.expect("an unsupported probe must retain the transaction journal");
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
## MinIO-generated encrypted fixtures
|
||||
|
||||
`minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
|
||||
`rustfs/src/storage/minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
|
||||
`.\rustfs\scripts\minio_fixture_lab\lab.py`.
|
||||
|
||||
It currently covers multipart fixtures for:
|
||||
@@ -20,5 +20,5 @@ Example:
|
||||
```powershell
|
||||
$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key'
|
||||
$env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '<base64-32-byte-local-minio-kms-key>'
|
||||
cargo +1.97.1 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
|
||||
cargo +1.97.1 test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored
|
||||
```
|
||||
|
||||
@@ -219,16 +219,6 @@ impl ErasureInfo {
|
||||
}
|
||||
|
||||
// #[derive(Debug, Clone)]
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Copy, Default)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum TransitionVersionState {
|
||||
#[default]
|
||||
Unknown,
|
||||
KnownDisabled,
|
||||
SuspendedNull,
|
||||
Exact,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
|
||||
pub struct FileInfo {
|
||||
pub volume: String,
|
||||
@@ -240,10 +230,6 @@ pub struct FileInfo {
|
||||
pub transitioned_objname: String,
|
||||
pub transition_tier: String,
|
||||
pub transition_version_id: Option<Uuid>,
|
||||
#[serde(default)]
|
||||
pub transition_version: Option<String>,
|
||||
#[serde(default)]
|
||||
pub transition_version_state: TransitionVersionState,
|
||||
pub expire_restored: bool,
|
||||
pub data_dir: Option<Uuid>,
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
@@ -473,10 +459,6 @@ impl FileInfo {
|
||||
if self.mod_time.is_none_or(|mod_time| mod_time <= OffsetDateTime::UNIX_EPOCH)
|
||||
|| (!allow_nil_version_id && self.version_id.is_some_and(|version_id| version_id.is_nil()))
|
||||
|| self.transition_version_id.is_some_and(|version_id| version_id.is_nil())
|
||||
|| self
|
||||
.transition_version
|
||||
.as_ref()
|
||||
.is_some_and(|version_id| version_id.is_empty())
|
||||
|| self.size != 0
|
||||
|| self.data_dir.is_some()
|
||||
|| self.mode.is_some()
|
||||
@@ -510,7 +492,6 @@ impl FileInfo {
|
||||
|| !self.transitioned_objname.is_empty()
|
||||
|| !self.transition_tier.is_empty()
|
||||
|| self.transition_version_id.is_some()
|
||||
|| self.transition_version.is_some()
|
||||
|| self.expire_restored
|
||||
|| self.size != 0
|
||||
|| self.data_dir.is_some()
|
||||
@@ -555,25 +536,6 @@ impl FileInfo {
|
||||
/// return `None`.
|
||||
pub fn validate(&self, mode: ValidationMode) -> Result<Option<ValidatedErasureLayout>> {
|
||||
self.validate_collection_bounds()?;
|
||||
if let (Some(version), Some(version_id)) = (&self.transition_version, self.transition_version_id)
|
||||
&& Uuid::parse_str(version).ok() != Some(version_id)
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let transition_state_valid = match self.transition_version_state {
|
||||
TransitionVersionState::Unknown => true,
|
||||
TransitionVersionState::KnownDisabled => self.transition_version.is_none() && self.transition_version_id.is_none(),
|
||||
TransitionVersionState::SuspendedNull => {
|
||||
self.transition_version.as_deref() == Some("null") && self.transition_version_id.is_none()
|
||||
}
|
||||
TransitionVersionState::Exact => self
|
||||
.transition_version
|
||||
.as_deref()
|
||||
.is_some_and(|version| version != "null" && !version.is_empty()),
|
||||
};
|
||||
if !transition_state_valid {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
|
||||
let erasure_layout = match mode {
|
||||
ValidationMode::RequireErasure => Some(self.validate_erasure_geometry()?),
|
||||
@@ -870,8 +832,6 @@ impl FileInfo {
|
||||
&& self.transition_tier == other.transition_tier
|
||||
&& self.transitioned_objname == other.transitioned_objname
|
||||
&& self.transition_version_id == other.transition_version_id
|
||||
&& self.transition_version == other.transition_version
|
||||
&& self.transition_version_state == other.transition_version_state
|
||||
}
|
||||
|
||||
/// Check if metadata maps are equal
|
||||
@@ -1391,15 +1351,6 @@ mod tests {
|
||||
assert_file_corrupt(&fi, ValidationMode::DeleteOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_read_validation_rejects_conflicting_transition_versions() {
|
||||
let mut fi = one_shard_validation_fileinfo(1);
|
||||
fi.transition_version_id = Some(Uuid::new_v4());
|
||||
fi.transition_version = Some(Uuid::new_v4().to_string());
|
||||
|
||||
assert_file_corrupt(&fi, ValidationMode::RequireErasure);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_read_validation_requires_canonical_delete_marker_shape() {
|
||||
let marker = FileInfo {
|
||||
@@ -1771,12 +1722,6 @@ mod tests {
|
||||
transitioned_objname,
|
||||
transition_tier,
|
||||
transition_version_id,
|
||||
transition_version: transition_version_id.map(|version_id| version_id.to_string()),
|
||||
transition_version_state: if transition_version_id.is_some() {
|
||||
TransitionVersionState::Exact
|
||||
} else {
|
||||
TransitionVersionState::Unknown
|
||||
},
|
||||
expire_restored,
|
||||
data_dir,
|
||||
mod_time,
|
||||
|
||||
@@ -26,14 +26,13 @@ use super::msgp_decode::{
|
||||
PrependByteReader, prealloc_hint, read_exact_vec, read_nil_or_array_len, read_nil_or_map_len, skip_msgp_value,
|
||||
};
|
||||
use super::*;
|
||||
use crate::{ChecksumInfo, TransitionVersionState};
|
||||
use crate::ChecksumInfo;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use rustfs_utils::http::{
|
||||
RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PURGESTATUS, SUFFIX_TIER_FV_ID,
|
||||
SUFFIX_TIER_FV_MARKER, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_bytes,
|
||||
get_bytes, get_consistent_bytes, get_str, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes,
|
||||
strip_internal_prefix,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, contains_key_bytes, get_bytes, get_consistent_bytes, get_str,
|
||||
has_internal_suffix, insert_bytes, is_internal_key, remove_bytes, strip_internal_prefix,
|
||||
};
|
||||
|
||||
const MSGPACK_EXT8: u8 = 0xc7;
|
||||
@@ -44,7 +43,6 @@ const MSGPACK_FIXEXT8: u8 = 0xd7;
|
||||
const MSGPACK_TIME_EXT_LEGACY: i8 = 5;
|
||||
const MSGPACK_TIME_EXT_OFFICIAL: i8 = -1;
|
||||
const MSGPACK_TIME_LEN: u8 = 12;
|
||||
const MAX_TRANSITION_VERSION_LEN: usize = 1024;
|
||||
|
||||
/// Sentinel signature returned when a version has no computable body (invalid /
|
||||
/// missing inner object). Mirrors MinIO's `signatureErr` so such versions never
|
||||
@@ -253,93 +251,23 @@ fn parse_legacy_uuid_bytes(bytes: &[u8], field: &str) -> Result<Option<Uuid>> {
|
||||
|
||||
/// Decode a stored transitioned-version-id from a version's `meta_sys`.
|
||||
///
|
||||
/// Legacy RustFS writes used 16 raw UUID bytes. New writes and MinIO-migrated
|
||||
/// records use the provider's exact UTF-8 version text. Empty, nil UUID, and
|
||||
/// malformed bytes are not usable remote versions.
|
||||
fn transitioned_version_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Result<Option<String>> {
|
||||
if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(value) = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let value = value.to_vec();
|
||||
/// RustFS writes it as 16 raw UUID bytes; MinIO-migrated tiered objects store
|
||||
/// the remote tier's version id as a UUID *string*. Accept both, and treat any
|
||||
/// absent / nil / otherwise-unparseable value as "no tier version" (matching the
|
||||
/// tolerant pre-hardening behavior) rather than failing the whole object read —
|
||||
/// a malformed tier id must not make an otherwise-readable object unreadable.
|
||||
fn transitioned_version_id_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<Uuid> {
|
||||
let value = get_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)?;
|
||||
if value.is_empty() {
|
||||
return Ok(None);
|
||||
return None;
|
||||
}
|
||||
if let Ok(id) = Uuid::from_slice(&value) {
|
||||
return Ok((!id.is_nil()).then(|| id.to_string()));
|
||||
return (!id.is_nil()).then_some(id);
|
||||
}
|
||||
let Ok(value) = String::from_utf8(value) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if value.is_empty()
|
||||
|| value.len() > MAX_TRANSITION_VERSION_LEN
|
||||
|| value.chars().any(char::is_control)
|
||||
|| Uuid::parse_str(&value).is_ok_and(|id| id.is_nil())
|
||||
{
|
||||
Ok(None)
|
||||
} else {
|
||||
Ok(Some(value))
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_version_state_from_meta_sys(
|
||||
meta_sys: &HashMap<String, Vec<u8>>,
|
||||
version: Option<&str>,
|
||||
) -> Result<TransitionVersionState> {
|
||||
if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) {
|
||||
return Ok(TransitionVersionState::Unknown);
|
||||
}
|
||||
let value = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or(Error::FileCorrupt)?;
|
||||
let state = match value {
|
||||
b"known-disabled" => TransitionVersionState::KnownDisabled,
|
||||
b"suspended-null" => TransitionVersionState::SuspendedNull,
|
||||
b"exact" => TransitionVersionState::Exact,
|
||||
b"unknown" => TransitionVersionState::Unknown,
|
||||
_ => return Err(Error::FileCorrupt),
|
||||
};
|
||||
let valid = match state {
|
||||
TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(),
|
||||
TransitionVersionState::SuspendedNull => version == Some("null"),
|
||||
TransitionVersionState::Exact => version.is_some_and(|value| value != "null"),
|
||||
};
|
||||
valid.then_some(state).ok_or(Error::FileCorrupt)
|
||||
}
|
||||
|
||||
fn transition_version_state_bytes(state: TransitionVersionState) -> &'static [u8] {
|
||||
match state {
|
||||
TransitionVersionState::Unknown => b"unknown",
|
||||
TransitionVersionState::KnownDisabled => b"known-disabled",
|
||||
TransitionVersionState::SuspendedNull => b"suspended-null",
|
||||
TransitionVersionState::Exact => b"exact",
|
||||
}
|
||||
}
|
||||
|
||||
fn set_transition_version_state(meta_sys: &mut HashMap<String, Vec<u8>>, state: TransitionVersionState) {
|
||||
if state == TransitionVersionState::Unknown {
|
||||
remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE);
|
||||
} else {
|
||||
insert_bytes(
|
||||
meta_sys,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
transition_version_state_bytes(state).to_vec(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn legacy_transitioned_version_id_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<Uuid> {
|
||||
transitioned_version_from_meta_sys(meta_sys)
|
||||
std::str::from_utf8(&value)
|
||||
.ok()
|
||||
.flatten()
|
||||
.and_then(|value| Uuid::parse_str(&value).ok())
|
||||
}
|
||||
|
||||
fn transitioned_version_bytes(fi: &FileInfo) -> Option<Vec<u8>> {
|
||||
fi.transition_version
|
||||
.as_ref()
|
||||
.map(|version| version.as_bytes().to_vec())
|
||||
.or_else(|| fi.transition_version_id.map(|version_id| version_id.as_bytes().to_vec()))
|
||||
.and_then(|s| Uuid::parse_str(s.trim()).ok())
|
||||
.filter(|id| !id.is_nil())
|
||||
}
|
||||
|
||||
fn parse_legacy_erasure_algo(value: &str) -> ErasureAlgo {
|
||||
@@ -2470,9 +2398,7 @@ impl MetaObject {
|
||||
let transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME)
|
||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
||||
.unwrap_or_default();
|
||||
let transition_version = transitioned_version_from_meta_sys(&self.meta_sys)?;
|
||||
let transition_version_state = transition_version_state_from_meta_sys(&self.meta_sys, transition_version.as_deref())?;
|
||||
let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
|
||||
let transition_version_id = transitioned_version_id_from_meta_sys(&self.meta_sys);
|
||||
let transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER)
|
||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
||||
.unwrap_or_default();
|
||||
@@ -2493,8 +2419,6 @@ impl MetaObject {
|
||||
transition_status,
|
||||
transitioned_objname,
|
||||
transition_version_id,
|
||||
transition_version,
|
||||
transition_version_state,
|
||||
transition_tier,
|
||||
..Default::default()
|
||||
})
|
||||
@@ -2507,12 +2431,13 @@ impl MetaObject {
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
fi.transitioned_objname.as_bytes().to_vec(),
|
||||
);
|
||||
if let Some(transition_version) = transitioned_version_bytes(fi) {
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
} else {
|
||||
remove_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
|
||||
if let Some(transition_version_id) = fi.transition_version_id.as_ref() {
|
||||
insert_bytes(
|
||||
&mut self.meta_sys,
|
||||
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
transition_version_id.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
set_transition_version_state(&mut self.meta_sys, fi.transition_version_state);
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec());
|
||||
if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) {
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes());
|
||||
@@ -2576,7 +2501,6 @@ impl MetaObject {
|
||||
SUFFIX_TRANSITION_TIER,
|
||||
SUFFIX_TRANSITIONED_OBJECTNAME,
|
||||
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
] {
|
||||
if let Some(v) = get_bytes(&self.meta_sys, suffix) {
|
||||
insert_bytes(&mut delete_marker.meta_sys, suffix, v);
|
||||
@@ -2638,11 +2562,8 @@ impl From<FileInfo> for MetaObject {
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(transition_version) = transitioned_version_bytes(&value) {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
}
|
||||
if !value.transition_status.is_empty() {
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state);
|
||||
if let Some(vid) = &value.transition_version_id {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, vid.as_bytes().to_vec());
|
||||
}
|
||||
|
||||
if !value.transition_tier.is_empty() {
|
||||
@@ -2785,11 +2706,7 @@ impl MetaDeleteMarker {
|
||||
.map(|v| String::from_utf8_lossy(&v).to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
fi.transition_version = transitioned_version_from_meta_sys(&self.meta_sys).ok().flatten();
|
||||
fi.transition_version_id = legacy_transitioned_version_id_from_meta_sys(&self.meta_sys);
|
||||
fi.transition_version_state =
|
||||
transition_version_state_from_meta_sys(&self.meta_sys, fi.transition_version.as_deref())
|
||||
.unwrap_or(TransitionVersionState::Unknown);
|
||||
fi.transition_version_id = transitioned_version_id_from_meta_sys(&self.meta_sys);
|
||||
}
|
||||
|
||||
fi
|
||||
@@ -2942,11 +2859,8 @@ impl From<FileInfo> for MetaDeleteMarker {
|
||||
value.transitioned_objname.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
if let Some(transition_version) = transitioned_version_bytes(&value) {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
}
|
||||
if !value.transition_status.is_empty() || value.tier_free_version() {
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state);
|
||||
if let Some(version_id) = value.transition_version_id {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, version_id.as_bytes().to_vec());
|
||||
}
|
||||
if !value.transition_tier.is_empty() {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
|
||||
@@ -3498,7 +3412,7 @@ mod tests {
|
||||
.insert("x-rustfs-internal-healing".to_string(), "true".to_string());
|
||||
marker.metadata.insert("content-type".to_string(), "text/plain".to_string());
|
||||
let remote_version_id = Uuid::new_v4();
|
||||
marker.transition_version = Some(remote_version_id.to_string());
|
||||
marker.transition_version_id = Some(remote_version_id);
|
||||
|
||||
let converted = MetaDeleteMarker::from(marker);
|
||||
|
||||
@@ -3506,19 +3420,7 @@ mod tests {
|
||||
assert_eq!(converted.meta_sys.get("x-minio-internal-purgestatus"), Some(&b"pending".to_vec()));
|
||||
assert_eq!(
|
||||
get_bytes(&converted.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID),
|
||||
Some(remote_version_id.to_string().into_bytes())
|
||||
);
|
||||
assert_eq!(
|
||||
converted
|
||||
.meta_sys
|
||||
.get(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}")),
|
||||
Some(&remote_version_id.to_string().into_bytes())
|
||||
);
|
||||
assert_eq!(
|
||||
converted
|
||||
.meta_sys
|
||||
.get(&format!("{}{SUFFIX_TRANSITIONED_VERSION_ID}", rustfs_utils::http::MINIO_INTERNAL_PREFIX)),
|
||||
Some(&remote_version_id.to_string().into_bytes())
|
||||
Some(remote_version_id.as_bytes().to_vec())
|
||||
);
|
||||
assert!(!converted.meta_sys.contains_key("x-rustfs-internal-healing"));
|
||||
assert!(!converted.meta_sys.contains_key("content-type"));
|
||||
@@ -4195,129 +4097,19 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("into_fileinfo");
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_id_opaque_text_is_preserved() {
|
||||
fn meta_object_transition_version_id_unparseable_stays_readable_as_none() {
|
||||
// A non-UUID / non-16-byte tier version id must NOT make the object
|
||||
// unreadable; it is tolerated as "no tier version" (compat with
|
||||
// pre-hardening behavior and foreign/edge metadata).
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"opaque-generation-42".to_vec());
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"not-a-uuid".to_vec());
|
||||
let fi = make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("opaque transition version id must decode");
|
||||
.expect("unparseable transition version id must not fail the object read");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42"));
|
||||
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_state_exact_round_trips_dual_keys() {
|
||||
let id = sample_version_id();
|
||||
let expected_version = id.to_string();
|
||||
let fi = FileInfo {
|
||||
transition_status: "complete".to_string(),
|
||||
transition_version: Some(expected_version.clone()),
|
||||
transition_version_state: TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let object = MetaObject::from(fi);
|
||||
assert_eq!(
|
||||
object
|
||||
.meta_sys
|
||||
.get(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"))
|
||||
.map(Vec::as_slice),
|
||||
Some(b"exact".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
object
|
||||
.meta_sys
|
||||
.get(&format!(
|
||||
"{}{SUFFIX_TRANSITIONED_VERSION_STATE}",
|
||||
rustfs_utils::http::MINIO_INTERNAL_PREFIX
|
||||
))
|
||||
.map(Vec::as_slice),
|
||||
Some(b"exact".as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
legacy_transitioned_version_id_from_meta_sys(&object.meta_sys),
|
||||
Some(id),
|
||||
"UUID exact writes must remain readable by the legacy UUID consumer"
|
||||
);
|
||||
let decoded = object.into_fileinfo("b", "k", false).expect("exact state should round trip");
|
||||
assert_eq!(decoded.transition_version_state, TransitionVersionState::Exact);
|
||||
assert_eq!(decoded.transition_version.as_deref(), Some(expected_version.as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_transition_known_disabled_removes_stale_version_dual_keys() {
|
||||
let mut meta_sys = HashMap::new();
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, b"stale-legacy-version".to_vec());
|
||||
let mut object = make_meta_object_with_sys(meta_sys);
|
||||
object.set_transition(&FileInfo {
|
||||
transition_status: TRANSITION_COMPLETE.to_string(),
|
||||
transitioned_objname: "remote/object".to_string(),
|
||||
transition_version_state: TransitionVersionState::KnownDisabled,
|
||||
transition_tier: "WARM".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(get_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID), None);
|
||||
assert!(
|
||||
!object
|
||||
.meta_sys
|
||||
.contains_key(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}"))
|
||||
);
|
||||
assert!(
|
||||
!object
|
||||
.meta_sys
|
||||
.contains_key(&format!("{}{SUFFIX_TRANSITIONED_VERSION_ID}", rustfs_utils::http::MINIO_INTERNAL_PREFIX))
|
||||
);
|
||||
let decoded = object
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("known-disabled transition must remain readable after replacing stale metadata");
|
||||
assert_eq!(decoded.transition_version, None);
|
||||
assert_eq!(decoded.transition_version_state, TransitionVersionState::KnownDisabled);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_state_conflict_fails_closed() {
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, sample_version_id().as_bytes().to_vec());
|
||||
sys.insert(format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"), b"exact".to_vec());
|
||||
sys.insert(
|
||||
format!("{}{SUFFIX_TRANSITIONED_VERSION_STATE}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
|
||||
b"known-disabled".to_vec(),
|
||||
);
|
||||
|
||||
make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect_err("conflicting state keys must fail closed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_id_invalid_utf8_yields_none() {
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, vec![0xff]);
|
||||
let fi = make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("invalid transition version bytes must not fail the object read");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_id_unsafe_text_yields_none() {
|
||||
for value in [b"opaque\0version".to_vec(), vec![b'x'; MAX_TRANSITION_VERSION_LEN + 1]] {
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, value);
|
||||
let fi = make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("unsafe transition version text must not fail the object read");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version, None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4331,7 +4123,6 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("string-form transition version id must decode");
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4361,14 +4152,16 @@ mod tests {
|
||||
}
|
||||
.into_fileinfo("b", "k", false);
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_marker_free_version_transition_version_id_opaque_text_is_preserved() {
|
||||
fn delete_marker_free_version_transition_version_id_unparseable_stays_readable() {
|
||||
// A malformed tier version id must not make a free-version record corrupt:
|
||||
// it decodes to None and stays readable. Otherwise free-version expiry
|
||||
// fails and the remote-tier object leaks.
|
||||
let mut sys = HashMap::new();
|
||||
insert_bytes(&mut sys, SUFFIX_FREE_VERSION, vec![]);
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"opaque-generation-42".to_vec());
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"not-a-uuid".to_vec());
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITION_TIER, b"WARM".to_vec());
|
||||
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_OBJECTNAME, b"remote-object".to_vec());
|
||||
let fi = MetaDeleteMarker {
|
||||
@@ -4379,9 +4172,8 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false);
|
||||
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42"));
|
||||
fi.validate_for_metadata_read()
|
||||
.expect("free-version record with an opaque tier id must remain readable");
|
||||
.expect("free-version record with an unparseable tier id must remain readable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4401,7 +4193,6 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false);
|
||||
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -89,7 +89,7 @@ pub use error::{KmsError, KmsUnavailableError, Result};
|
||||
pub use manager::KmsManager;
|
||||
pub use service::{DataKey, ObjectEncryptionService};
|
||||
pub use service_manager::{
|
||||
KmsServiceManager, KmsServiceStatus, get_global_encryption_service, get_global_kms_service_manager,
|
||||
KmsServiceManager, KmsServiceStatus, KmsStartOutcome, get_global_encryption_service, get_global_kms_service_manager,
|
||||
init_global_kms_service_manager,
|
||||
};
|
||||
pub use types::*;
|
||||
|
||||
@@ -20,11 +20,12 @@ use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::service::ObjectEncryptionService;
|
||||
use arc_swap::ArcSwap;
|
||||
use std::future::Future;
|
||||
use std::sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
const LOG_COMPONENT_KMS: &str = "kms";
|
||||
@@ -44,6 +45,13 @@ pub enum KmsServiceStatus {
|
||||
Error(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum KmsStartOutcome {
|
||||
Started,
|
||||
Restarted,
|
||||
AlreadyRunning,
|
||||
}
|
||||
|
||||
/// Service version information for zero-downtime reconfiguration
|
||||
#[derive(Clone)]
|
||||
struct ServiceVersion {
|
||||
@@ -55,16 +63,17 @@ struct ServiceVersion {
|
||||
manager: Arc<KmsManager>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RuntimeState {
|
||||
config: Option<KmsConfig>,
|
||||
status: KmsServiceStatus,
|
||||
current_service: Option<ServiceVersion>,
|
||||
}
|
||||
|
||||
/// Dynamic KMS service manager with versioned services for zero-downtime reconfiguration
|
||||
pub struct KmsServiceManager {
|
||||
/// Current service version (if running)
|
||||
/// Uses ArcSwap for atomic, lock-free service switching
|
||||
/// This allows instant atomic updates without blocking readers
|
||||
current_service: ArcSwap<Option<ServiceVersion>>,
|
||||
/// Current configuration
|
||||
config: Arc<RwLock<Option<KmsConfig>>>,
|
||||
/// Current status
|
||||
status: Arc<RwLock<KmsServiceStatus>>,
|
||||
/// Atomically published configuration, status, and current service.
|
||||
state: ArcSwap<RuntimeState>,
|
||||
/// Version counter (monotonically increasing)
|
||||
version_counter: Arc<AtomicU64>,
|
||||
/// Mutex to protect lifecycle operations (start, stop, reconfigure)
|
||||
@@ -76,9 +85,11 @@ impl KmsServiceManager {
|
||||
/// Create a new KMS service manager (not configured)
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
current_service: ArcSwap::from_pointee(None),
|
||||
config: Arc::new(RwLock::new(None)),
|
||||
status: Arc::new(RwLock::new(KmsServiceStatus::NotConfigured)),
|
||||
state: ArcSwap::from_pointee(RuntimeState {
|
||||
config: None,
|
||||
status: KmsServiceStatus::NotConfigured,
|
||||
current_service: None,
|
||||
}),
|
||||
version_counter: Arc::new(AtomicU64::new(0)),
|
||||
lifecycle_mutex: Arc::new(Mutex::new(())),
|
||||
}
|
||||
@@ -86,39 +97,65 @@ impl KmsServiceManager {
|
||||
|
||||
/// Get current service status
|
||||
pub async fn get_status(&self) -> KmsServiceStatus {
|
||||
self.status.read().await.clone()
|
||||
self.state.load().status.clone()
|
||||
}
|
||||
|
||||
/// Get current configuration (if any)
|
||||
pub async fn get_config(&self) -> Option<KmsConfig> {
|
||||
self.config.read().await.clone()
|
||||
self.state.load().config.clone()
|
||||
}
|
||||
|
||||
/// Get configuration for status and management responses without static key material.
|
||||
pub async fn get_redacted_config(&self) -> Option<KmsConfig> {
|
||||
let mut config = self.config.read().await.clone()?;
|
||||
let mut config = self.state.load().config.clone()?;
|
||||
Self::redact_config(&mut config);
|
||||
Some(config)
|
||||
}
|
||||
|
||||
/// Get status and redacted configuration from the same published snapshot.
|
||||
pub async fn get_redacted_state(&self) -> (KmsServiceStatus, Option<KmsConfig>) {
|
||||
let state = self.state.load();
|
||||
let mut config = state.config.clone();
|
||||
if let Some(config) = &mut config {
|
||||
Self::redact_config(config);
|
||||
}
|
||||
(state.status.clone(), config)
|
||||
}
|
||||
|
||||
fn redact_config(config: &mut KmsConfig) {
|
||||
if let BackendConfig::Static(static_config) = &mut config.backend_config {
|
||||
use zeroize::Zeroize;
|
||||
static_config.secret_key.zeroize();
|
||||
}
|
||||
Some(config)
|
||||
}
|
||||
|
||||
/// Configure KMS with new configuration
|
||||
pub async fn configure(&self, new_config: KmsConfig) -> Result<()> {
|
||||
self.configure_with_persistence(new_config, || async { Ok(()) }).await
|
||||
}
|
||||
|
||||
/// Configure KMS and publish the in-memory state only after persistence succeeds.
|
||||
///
|
||||
/// The persistence callback runs under the lifecycle lock and must not call
|
||||
/// another lifecycle method on this manager.
|
||||
pub async fn configure_with_persistence<Persist, PersistFuture>(&self, new_config: KmsConfig, persist: Persist) -> Result<()>
|
||||
where
|
||||
Persist: FnOnce() -> PersistFuture,
|
||||
PersistFuture: Future<Output = Result<()>>,
|
||||
{
|
||||
new_config.validate()?;
|
||||
|
||||
// Update configuration
|
||||
{
|
||||
let mut config = self.config.write().await;
|
||||
*config = Some(new_config.clone());
|
||||
}
|
||||
|
||||
// Update status
|
||||
{
|
||||
let mut status = self.status.write().await;
|
||||
*status = KmsServiceStatus::Configured;
|
||||
let _guard = self.lifecycle_mutex.lock().await;
|
||||
if self.state.load().current_service.is_some() {
|
||||
return Err(KmsError::configuration_error(
|
||||
"Cannot configure KMS while it is running; use reconfigure instead",
|
||||
));
|
||||
}
|
||||
persist().await?;
|
||||
self.state.store(Arc::new(RuntimeState {
|
||||
config: Some(new_config),
|
||||
status: KmsServiceStatus::Configured,
|
||||
current_service: None,
|
||||
}));
|
||||
|
||||
debug!(
|
||||
event = EVENT_KMS_SERVICE_STATE,
|
||||
@@ -136,19 +173,35 @@ impl KmsServiceManager {
|
||||
self.start_internal().await
|
||||
}
|
||||
|
||||
/// Start or restart KMS with the running-state decision serialized with the lifecycle action.
|
||||
pub async fn start_or_restart(&self, force: bool) -> Result<KmsStartOutcome> {
|
||||
let _guard = self.lifecycle_mutex.lock().await;
|
||||
let running = self.state.load().current_service.is_some();
|
||||
if running && !force {
|
||||
return Ok(KmsStartOutcome::AlreadyRunning);
|
||||
}
|
||||
self.start_internal().await?;
|
||||
Ok(if running {
|
||||
KmsStartOutcome::Restarted
|
||||
} else {
|
||||
KmsStartOutcome::Started
|
||||
})
|
||||
}
|
||||
|
||||
/// Internal start implementation (called within lifecycle mutex)
|
||||
async fn start_internal(&self) -> Result<()> {
|
||||
let config = {
|
||||
let config_guard = self.config.read().await;
|
||||
match config_guard.as_ref() {
|
||||
Some(config) => config.clone(),
|
||||
None => {
|
||||
let err_msg = "Cannot start KMS: no configuration provided";
|
||||
error!("{}", err_msg);
|
||||
let mut status = self.status.write().await;
|
||||
*status = KmsServiceStatus::Error(err_msg.to_string());
|
||||
return Err(KmsError::configuration_error(err_msg));
|
||||
}
|
||||
let state = self.state.load_full();
|
||||
let config = match state.config.as_ref() {
|
||||
Some(config) => config.clone(),
|
||||
None => {
|
||||
let err_msg = "Cannot start KMS: no configuration provided";
|
||||
error!("{}", err_msg);
|
||||
self.state.store(Arc::new(RuntimeState {
|
||||
config: None,
|
||||
status: KmsServiceStatus::Error(err_msg.to_string()),
|
||||
current_service: None,
|
||||
}));
|
||||
return Err(KmsError::configuration_error(err_msg));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -161,17 +214,9 @@ impl KmsServiceManager {
|
||||
"KMS service starting"
|
||||
);
|
||||
|
||||
match self.create_service_version(&config).await {
|
||||
match self.create_healthy_service_version(&config).await {
|
||||
Ok(service_version) => {
|
||||
// Atomically update to new service version (lock-free, instant)
|
||||
// ArcSwap::store() is a true atomic operation using CAS
|
||||
self.current_service.store(Arc::new(Some(service_version)));
|
||||
|
||||
// Update status
|
||||
{
|
||||
let mut status = self.status.write().await;
|
||||
*status = KmsServiceStatus::Running;
|
||||
}
|
||||
self.publish_running(config, service_version);
|
||||
|
||||
debug!(
|
||||
event = EVENT_KMS_SERVICE_STATE,
|
||||
@@ -185,13 +230,24 @@ impl KmsServiceManager {
|
||||
Err(e) => {
|
||||
let err_msg = format!("Failed to create KMS backend: {e}");
|
||||
error!("{}", err_msg);
|
||||
let mut status = self.status.write().await;
|
||||
*status = KmsServiceStatus::Error(err_msg.clone());
|
||||
if state.current_service.is_none() {
|
||||
self.state.store(Arc::new(RuntimeState {
|
||||
config: state.config.clone(),
|
||||
status: KmsServiceStatus::Error(err_msg.clone()),
|
||||
current_service: None,
|
||||
}));
|
||||
}
|
||||
Err(KmsError::backend_error(&err_msg))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace the running service without exposing a stopped interval.
|
||||
pub async fn restart(&self) -> Result<()> {
|
||||
let _guard = self.lifecycle_mutex.lock().await;
|
||||
self.start_internal().await
|
||||
}
|
||||
|
||||
/// Stop KMS service
|
||||
///
|
||||
/// Note: This stops accepting new operations, but existing operations using
|
||||
@@ -213,15 +269,16 @@ impl KmsServiceManager {
|
||||
|
||||
// Atomically clear current service version (lock-free, instant)
|
||||
// Note: Existing Arc references will keep the service alive until operations complete
|
||||
self.current_service.store(Arc::new(None));
|
||||
|
||||
// Update status (keep configuration)
|
||||
{
|
||||
let mut status = self.status.write().await;
|
||||
if !matches!(*status, KmsServiceStatus::NotConfigured) {
|
||||
*status = KmsServiceStatus::Configured;
|
||||
}
|
||||
}
|
||||
let state = self.state.load_full();
|
||||
self.state.store(Arc::new(RuntimeState {
|
||||
config: state.config.clone(),
|
||||
status: if state.config.is_some() {
|
||||
KmsServiceStatus::Configured
|
||||
} else {
|
||||
KmsServiceStatus::NotConfigured
|
||||
},
|
||||
current_service: None,
|
||||
}));
|
||||
|
||||
debug!(
|
||||
event = EVENT_KMS_SERVICE_STATE,
|
||||
@@ -244,6 +301,22 @@ impl KmsServiceManager {
|
||||
/// This ensures zero downtime during reconfiguration, even for long-running
|
||||
/// operations like encrypting large files.
|
||||
pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> {
|
||||
self.reconfigure_with_persistence(new_config, || async { Ok(()) }).await
|
||||
}
|
||||
|
||||
/// Reconfigure KMS after the candidate is healthy and persistence succeeds.
|
||||
///
|
||||
/// The persistence callback runs under the lifecycle lock and must not call
|
||||
/// another lifecycle method on this manager.
|
||||
pub async fn reconfigure_with_persistence<Persist, PersistFuture>(
|
||||
&self,
|
||||
new_config: KmsConfig,
|
||||
persist: Persist,
|
||||
) -> Result<()>
|
||||
where
|
||||
Persist: FnOnce() -> PersistFuture,
|
||||
PersistFuture: Future<Output = Result<()>>,
|
||||
{
|
||||
let _guard = self.lifecycle_mutex.lock().await;
|
||||
|
||||
debug!(
|
||||
@@ -255,29 +328,16 @@ impl KmsServiceManager {
|
||||
);
|
||||
new_config.validate()?;
|
||||
|
||||
// Configure with new config
|
||||
{
|
||||
let mut config = self.config.write().await;
|
||||
*config = Some(new_config.clone());
|
||||
}
|
||||
|
||||
// Create new service version without stopping old one
|
||||
// This allows existing operations to continue while new operations use new service
|
||||
match self.create_service_version(&new_config).await {
|
||||
match self.create_healthy_service_version(&new_config).await {
|
||||
Ok(new_service_version) => {
|
||||
// Get old version for logging (lock-free read)
|
||||
let old_version = self.current_service.load().as_ref().as_ref().map(|sv| sv.version);
|
||||
let old_version = self.state.load().current_service.as_ref().map(|sv| sv.version);
|
||||
|
||||
// Atomically switch to new service version (lock-free, instant CAS operation)
|
||||
// This is a true atomic operation - no waiting for locks, instant switch
|
||||
// Old service will be dropped when no more Arc references exist
|
||||
self.current_service.store(Arc::new(Some(new_service_version.clone())));
|
||||
persist().await?;
|
||||
|
||||
// Update status
|
||||
{
|
||||
let mut status = self.status.write().await;
|
||||
*status = KmsServiceStatus::Running;
|
||||
}
|
||||
self.publish_running(new_config, new_service_version.clone());
|
||||
|
||||
if let Some(old_ver) = old_version {
|
||||
info!(
|
||||
@@ -304,8 +364,6 @@ impl KmsServiceManager {
|
||||
Err(e) => {
|
||||
let err_msg = format!("Failed to reconfigure KMS: {e}");
|
||||
error!("{}", err_msg);
|
||||
let mut status = self.status.write().await;
|
||||
*status = KmsServiceStatus::Error(err_msg.clone());
|
||||
Err(KmsError::backend_error(&err_msg))
|
||||
}
|
||||
}
|
||||
@@ -316,7 +374,7 @@ impl KmsServiceManager {
|
||||
/// Returns the manager from the current service version.
|
||||
/// Uses lock-free atomic load for optimal performance.
|
||||
pub async fn get_manager(&self) -> Option<Arc<KmsManager>> {
|
||||
self.current_service.load().as_ref().as_ref().map(|sv| sv.manager.clone())
|
||||
self.state.load().current_service.as_ref().map(|sv| sv.manager.clone())
|
||||
}
|
||||
|
||||
/// Get encryption service (if running)
|
||||
@@ -326,7 +384,7 @@ impl KmsServiceManager {
|
||||
/// This ensures new operations always use the latest service version,
|
||||
/// while existing operations continue using their Arc references.
|
||||
pub async fn get_encryption_service(&self) -> Option<Arc<ObjectEncryptionService>> {
|
||||
self.current_service.load().as_ref().as_ref().map(|sv| sv.service.clone())
|
||||
self.state.load().current_service.as_ref().map(|sv| sv.service.clone())
|
||||
}
|
||||
|
||||
/// Get current service version number
|
||||
@@ -334,14 +392,16 @@ impl KmsServiceManager {
|
||||
/// Useful for monitoring and debugging.
|
||||
/// Uses lock-free atomic load.
|
||||
pub async fn get_service_version(&self) -> Option<u64> {
|
||||
self.current_service.load().as_ref().as_ref().map(|sv| sv.version)
|
||||
self.state.load().current_service.as_ref().map(|sv| sv.version)
|
||||
}
|
||||
|
||||
/// Health check for the KMS service
|
||||
pub async fn health_check(&self) -> Result<bool> {
|
||||
let manager = self.get_manager().await;
|
||||
match manager {
|
||||
Some(manager) => {
|
||||
let checked_state = self.state.load_full();
|
||||
match checked_state.current_service.as_ref() {
|
||||
Some(service_version) => {
|
||||
let manager = service_version.manager.clone();
|
||||
let checked_version = service_version.version;
|
||||
// Perform health check on the backend
|
||||
match manager.health_check().await {
|
||||
Ok(healthy) => {
|
||||
@@ -352,9 +412,8 @@ impl KmsServiceManager {
|
||||
}
|
||||
Err(e) => {
|
||||
error!("KMS health check error: {}", e);
|
||||
// Update status to error
|
||||
let mut status = self.status.write().await;
|
||||
*status = KmsServiceStatus::Error(format!("Health check failed: {e}"));
|
||||
let _guard = self.lifecycle_mutex.lock().await;
|
||||
self.mark_health_error_if_current(checked_version, &e);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
@@ -413,6 +472,33 @@ impl KmsServiceManager {
|
||||
manager: kms_manager,
|
||||
})
|
||||
}
|
||||
|
||||
async fn create_healthy_service_version(&self, config: &KmsConfig) -> Result<ServiceVersion> {
|
||||
let service_version = self.create_service_version(config).await?;
|
||||
if !service_version.manager.health_check().await? {
|
||||
return Err(KmsError::backend_error("KMS backend health check failed"));
|
||||
}
|
||||
Ok(service_version)
|
||||
}
|
||||
|
||||
fn publish_running(&self, config: KmsConfig, service_version: ServiceVersion) {
|
||||
self.state.store(Arc::new(RuntimeState {
|
||||
config: Some(config),
|
||||
status: KmsServiceStatus::Running,
|
||||
current_service: Some(service_version),
|
||||
}));
|
||||
}
|
||||
|
||||
fn mark_health_error_if_current(&self, checked_version: u64, error: &KmsError) {
|
||||
let current = self.state.load_full();
|
||||
if current.current_service.as_ref().map(|version| version.version) == Some(checked_version) {
|
||||
self.state.store(Arc::new(RuntimeState {
|
||||
config: current.config.clone(),
|
||||
status: KmsServiceStatus::Error(format!("Health check failed: {error}")),
|
||||
current_service: current.current_service.clone(),
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for KmsServiceManager {
|
||||
@@ -445,6 +531,11 @@ pub async fn get_global_encryption_service() -> Option<Arc<ObjectEncryptionServi
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
|
||||
|
||||
fn static_config(key_id: &str, fill: u8) -> KmsConfig {
|
||||
KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32]))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configure_rejects_insecure_development_defaults_before_state_update() {
|
||||
@@ -462,8 +553,6 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn redacted_config_omits_static_key_material() {
|
||||
use base64::Engine as _;
|
||||
|
||||
let manager = KmsServiceManager::new();
|
||||
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
|
||||
manager
|
||||
@@ -477,4 +566,134 @@ mod tests {
|
||||
};
|
||||
assert!(static_config.secret_key.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configure_persistence_failure_leaves_state_unchanged() {
|
||||
let manager = KmsServiceManager::new();
|
||||
|
||||
let result = manager
|
||||
.configure_with_persistence(static_config("key-a", 0x11), || async { Err(KmsError::backend_error("persist failed")) })
|
||||
.await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert_eq!(manager.get_status().await, KmsServiceStatus::NotConfigured);
|
||||
assert!(manager.get_config().await.is_none());
|
||||
assert!(manager.get_encryption_service().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configure_rejects_running_service_without_changing_snapshot() {
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
|
||||
manager.start().await.expect("start");
|
||||
let version = manager.get_service_version().await;
|
||||
|
||||
let result = manager.configure(static_config("key-b", 0x22)).await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
|
||||
assert_eq!(manager.get_service_version().await, version);
|
||||
assert_eq!(
|
||||
manager.get_config().await.and_then(|config| config.default_key_id),
|
||||
Some("key-a".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconfigure_persistence_failure_keeps_old_running_snapshot() {
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
|
||||
manager.start().await.expect("start");
|
||||
let old_version = manager.get_service_version().await;
|
||||
let old_service = manager.get_encryption_service().await.expect("old service");
|
||||
|
||||
let result = manager
|
||||
.reconfigure_with_persistence(static_config("key-b", 0x22), || async {
|
||||
Err(KmsError::backend_error("persist failed"))
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
|
||||
assert_eq!(manager.get_service_version().await, old_version);
|
||||
assert_eq!(
|
||||
manager.get_config().await.and_then(|config| config.default_key_id),
|
||||
Some("key-a".to_string())
|
||||
);
|
||||
assert!(Arc::ptr_eq(
|
||||
&old_service,
|
||||
&manager.get_encryption_service().await.expect("old service remains")
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconfigure_candidate_failure_keeps_old_running_snapshot() {
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
|
||||
manager.start().await.expect("start");
|
||||
let old_version = manager.get_service_version().await;
|
||||
let invalid_parent = tempfile::NamedTempFile::new().expect("temporary file");
|
||||
let invalid_config = KmsConfig::local(invalid_parent.path().join("keys")).with_insecure_development_defaults();
|
||||
|
||||
let result = manager.reconfigure(invalid_config).await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
|
||||
assert_eq!(manager.get_service_version().await, old_version);
|
||||
assert_eq!(
|
||||
manager.get_config().await.and_then(|config| config.default_key_id),
|
||||
Some("key-a".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restart_never_unpublishes_the_running_service() {
|
||||
let manager = Arc::new(KmsServiceManager::new());
|
||||
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
|
||||
manager.start().await.expect("start");
|
||||
let old_version = manager.get_service_version().await.expect("old version");
|
||||
let restarting = {
|
||||
let manager = manager.clone();
|
||||
tokio::spawn(async move { manager.restart().await })
|
||||
};
|
||||
|
||||
while !restarting.is_finished() {
|
||||
assert!(manager.get_encryption_service().await.is_some());
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
restarting.await.expect("restart task").expect("restart");
|
||||
|
||||
assert!(manager.get_encryption_service().await.is_some());
|
||||
assert!(manager.get_service_version().await.expect("new version") > old_version);
|
||||
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn start_or_restart_decides_under_the_lifecycle_lock() {
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
|
||||
|
||||
assert_eq!(manager.start_or_restart(false).await.expect("initial start"), KmsStartOutcome::Started);
|
||||
let first_version = manager.get_service_version().await.expect("first version");
|
||||
assert_eq!(
|
||||
manager.start_or_restart(false).await.expect("already running"),
|
||||
KmsStartOutcome::AlreadyRunning
|
||||
);
|
||||
assert_eq!(manager.get_service_version().await, Some(first_version));
|
||||
assert_eq!(manager.start_or_restart(true).await.expect("forced restart"), KmsStartOutcome::Restarted);
|
||||
assert!(manager.get_service_version().await.expect("restarted version") > first_version);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_health_failure_cannot_poison_new_service_status() {
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
|
||||
manager.start().await.expect("start");
|
||||
let old_version = manager.get_service_version().await.expect("old version");
|
||||
manager.restart().await.expect("restart");
|
||||
|
||||
manager.mark_health_error_if_current(old_version, &KmsError::backend_error("stale failure"));
|
||||
|
||||
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,9 +25,6 @@ keywords = ["file-system", "notification", "real-time", "rustfs", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "filesystem"]
|
||||
documentation = "https://docs.rs/rustfs-notify/latest/rustfs_notify/"
|
||||
|
||||
[features]
|
||||
demo-examples = []
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["notify", "server-config-model"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
@@ -74,18 +71,6 @@ workspace = true
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
[[example]]
|
||||
name = "full_demo"
|
||||
required-features = ["demo-examples"]
|
||||
|
||||
[[example]]
|
||||
name = "full_demo_one"
|
||||
required-features = ["demo-examples"]
|
||||
|
||||
[[example]]
|
||||
name = "webhook"
|
||||
required-features = ["demo-examples"]
|
||||
|
||||
[[bench]]
|
||||
name = "snapshot_mode_scan"
|
||||
harness = false
|
||||
|
||||
@@ -171,7 +171,6 @@ pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SI
|
||||
pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2;
|
||||
pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1;
|
||||
pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0";
|
||||
pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0";
|
||||
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
|
||||
pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096;
|
||||
@@ -198,49 +197,6 @@ pub fn is_heal_control_capability_probe(command: &[u8]) -> bool {
|
||||
command.len() == HEAL_CONTROL_CAPABILITY_PROBE_PREFIX.len() + 16 && command.starts_with(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX)
|
||||
}
|
||||
|
||||
pub fn remote_version_state_capability_probe(nonce: &[u8; 16]) -> Vec<u8> {
|
||||
let mut probe = Vec::with_capacity(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX.len() + nonce.len());
|
||||
probe.extend_from_slice(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX);
|
||||
probe.extend_from_slice(nonce);
|
||||
probe
|
||||
}
|
||||
|
||||
pub fn is_remote_version_state_capability_probe(command: &[u8]) -> bool {
|
||||
command.len() == REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX.len() + 16
|
||||
&& command.starts_with(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX)
|
||||
}
|
||||
|
||||
pub fn encode_remote_version_state_capability(
|
||||
topology_member: &str,
|
||||
process_epoch: &[u8; 16],
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let topology_member = topology_member.as_bytes();
|
||||
let mut result = Vec::with_capacity(8 + topology_member.len() + process_epoch.len());
|
||||
result.extend_from_slice(&u64::try_from(topology_member.len())?.to_be_bytes());
|
||||
result.extend_from_slice(topology_member);
|
||||
result.extend_from_slice(process_epoch);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn decode_remote_version_state_capability(result: &[u8]) -> Result<(&str, &[u8; 16]), &'static str> {
|
||||
let member_len = result
|
||||
.get(..8)
|
||||
.and_then(|value| value.try_into().ok())
|
||||
.map(u64::from_be_bytes)
|
||||
.ok_or("remote version state capability is truncated")?;
|
||||
let member_len = usize::try_from(member_len).map_err(|_| "remote version state member length cannot be represented")?;
|
||||
let member_end = 8_usize
|
||||
.checked_add(member_len)
|
||||
.ok_or("remote version state member length overflow")?;
|
||||
let topology_member = std::str::from_utf8(result.get(8..member_end).ok_or("remote version state member is truncated")?)
|
||||
.map_err(|_| "remote version state member is not UTF-8")?;
|
||||
let process_epoch = result
|
||||
.get(member_end..)
|
||||
.and_then(|value| value.try_into().ok())
|
||||
.ok_or("remote version state process epoch has an invalid length")?;
|
||||
Ok((topology_member, process_epoch))
|
||||
}
|
||||
|
||||
/// Builds the stable byte representation authenticated for a heal-control request.
|
||||
///
|
||||
/// This deliberately does not reuse protobuf encoding: mixed-version peers may
|
||||
@@ -1247,12 +1203,10 @@ mod scanner_activity_tests {
|
||||
#[cfg(test)]
|
||||
mod heal_control_tests {
|
||||
use super::{
|
||||
HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX,
|
||||
canonical_heal_control_capability_ack, canonical_heal_control_request_body, canonical_heal_control_response_body,
|
||||
decode_remote_version_state_capability, encode_remote_version_state_capability, heal_control_capability_probe,
|
||||
HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, canonical_heal_control_capability_ack,
|
||||
canonical_heal_control_request_body, canonical_heal_control_response_body, heal_control_capability_probe,
|
||||
heal_control_coordinator_epoch, heal_control_execution_timeout, heal_control_execution_timeout_for,
|
||||
internode_rpc_timeout, is_heal_control_capability_probe, is_remote_version_state_capability_probe,
|
||||
normalize_internode_rpc_timeout, remote_version_state_capability_probe,
|
||||
internode_rpc_timeout, is_heal_control_capability_probe, normalize_internode_rpc_timeout,
|
||||
};
|
||||
use crate::heal_control;
|
||||
use std::time::Duration;
|
||||
@@ -1309,29 +1263,6 @@ mod heal_control_tests {
|
||||
assert!(!is_heal_control_capability_probe(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_capability_probe_requires_exact_nonce() {
|
||||
let probe = remote_version_state_capability_probe(&[7; 16]);
|
||||
assert!(is_remote_version_state_capability_probe(&probe));
|
||||
assert!(!is_remote_version_state_capability_probe(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_capability_binds_member_and_process_epoch() {
|
||||
let encoded =
|
||||
encode_remote_version_state_capability("node-a:9000", &[7; 16]).expect("small capability response should encode");
|
||||
assert_eq!(
|
||||
decode_remote_version_state_capability(&encoded).expect("capability response should decode"),
|
||||
("node-a:9000", &[7; 16])
|
||||
);
|
||||
assert!(decode_remote_version_state_capability(&encoded[..encoded.len() - 1]).is_err());
|
||||
|
||||
let mut invalid_utf8 =
|
||||
encode_remote_version_state_capability("node-a", &[7; 16]).expect("small capability response should encode");
|
||||
invalid_utf8[8] = 0xff;
|
||||
assert!(decode_remote_version_state_capability(&invalid_utf8).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_response_binds_request_and_result() {
|
||||
let baseline = canonical_heal_control_response_body(2, "abcdef", b"query", b"result").unwrap();
|
||||
|
||||
@@ -137,7 +137,7 @@ tests read):
|
||||
./capture_via_docker.sh
|
||||
|
||||
RUSTFS_MINIO_STATIC_KMS_KEY_B64=IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= \
|
||||
cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
|
||||
cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored
|
||||
```
|
||||
|
||||
This is exactly what the nightly `minio-interop` GitHub Actions workflow runs
|
||||
|
||||
@@ -25,6 +25,11 @@ const RUSTFS_PREFIX: &str = "x-rustfs-";
|
||||
const MINIO_PREFIX: &str = "x-minio-";
|
||||
const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
|
||||
const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
|
||||
const MINIO_INTERNAL_ENCRYPTION_PREFIX: &str = "x-minio-internal-server-side-encryption-";
|
||||
const MINIO_INTERNAL_ENCRYPTED_MULTIPART: &str = "x-minio-internal-encrypted-multipart";
|
||||
const RUSTFS_ENCRYPTION_ORIGINAL_SIZE: &str = "x-rustfs-encryption-original-size";
|
||||
const MINIO_ENCRYPTION_ORIGINAL_SIZE: &str = "x-minio-encryption-original-size";
|
||||
const SSEC_ORIGINAL_SIZE: &str = "x-amz-server-side-encryption-customer-original-size";
|
||||
|
||||
// Suffix constants (part after x-rustfs- or x-minio-). Use with get_header/insert_header.
|
||||
pub const SUFFIX_FORCE_DELETE: &str = "force-delete";
|
||||
@@ -40,11 +45,49 @@ pub const SUFFIX_SOURCE_REPLICATION_REQUEST: &str = "source-replication-request"
|
||||
pub const SUFFIX_SOURCE_REPLICATION_CHECK: &str = "source-replication-check";
|
||||
pub const SUFFIX_REPLICATION_SSEC_CRC: &str = "replication-ssec-crc";
|
||||
|
||||
/// Returns true if the key is an internal encryption metadata key (x-rustfs-encryption-* or
|
||||
/// x-minio-encryption-*). Case-insensitive for metadata filtering.
|
||||
/// Returns true if the key is object-encryption metadata understood by RustFS or MinIO.
|
||||
/// Case-insensitive for metadata filtering.
|
||||
pub fn is_encryption_metadata_key(key: &str) -> bool {
|
||||
let lower = key.to_lowercase();
|
||||
lower.starts_with(RUSTFS_ENCRYPTION_PREFIX) || lower.starts_with(MINIO_ENCRYPTION_PREFIX)
|
||||
lower.starts_with(RUSTFS_ENCRYPTION_PREFIX)
|
||||
|| lower.starts_with(MINIO_ENCRYPTION_PREFIX)
|
||||
|| lower.starts_with(MINIO_INTERNAL_ENCRYPTION_PREFIX)
|
||||
|| lower == MINIO_INTERNAL_ENCRYPTED_MULTIPART
|
||||
}
|
||||
|
||||
/// Returns true when a metadata key proves that object data is encrypted.
|
||||
///
|
||||
/// Original-size metadata alone is not proof: older plaintext objects can
|
||||
/// retain that compatibility field after metadata migration.
|
||||
pub fn is_object_encryption_marker(key: &str) -> bool {
|
||||
(is_encryption_metadata_key(key)
|
||||
&& !key.eq_ignore_ascii_case(RUSTFS_ENCRYPTION_ORIGINAL_SIZE)
|
||||
&& !key.eq_ignore_ascii_case(MINIO_ENCRYPTION_ORIGINAL_SIZE))
|
||||
|| super::is_sse_header(key)
|
||||
}
|
||||
|
||||
/// Reads the logical object size recorded by encryption metadata.
|
||||
pub fn get_object_encryption_original_size(metadata: &std::collections::HashMap<String, String>) -> std::io::Result<Option<i64>> {
|
||||
let actual_size = super::get_str(metadata, super::SUFFIX_ACTUAL_SIZE);
|
||||
let size = get_case_insensitive(metadata, RUSTFS_ENCRYPTION_ORIGINAL_SIZE)
|
||||
.or_else(|| get_case_insensitive(metadata, SSEC_ORIGINAL_SIZE))
|
||||
.or(actual_size.as_deref());
|
||||
|
||||
let Some(size) = size.filter(|size| !size.is_empty()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
size.parse::<i64>()
|
||||
.map(Some)
|
||||
.map_err(|error| std::io::Error::other(format!("Failed to parse encryption original size: {error}")))
|
||||
}
|
||||
|
||||
fn get_case_insensitive<'a>(metadata: &'a std::collections::HashMap<String, String>, key: &str) -> Option<&'a str> {
|
||||
metadata.get(key).map(String::as_str).or_else(|| {
|
||||
metadata
|
||||
.iter()
|
||||
.find(|(candidate, _)| candidate.eq_ignore_ascii_case(key))
|
||||
.map(|(_, value)| value.as_str())
|
||||
})
|
||||
}
|
||||
|
||||
fn rustfs_key(suffix: &str) -> String {
|
||||
@@ -106,10 +149,37 @@ mod tests {
|
||||
assert!(is_encryption_metadata_key("x-rustfs-encryption-iv"));
|
||||
assert!(is_encryption_metadata_key("X-Rustfs-Encryption-Key"));
|
||||
assert!(is_encryption_metadata_key("x-minio-encryption-iv"));
|
||||
assert!(is_encryption_metadata_key("X-Minio-Internal-Server-Side-Encryption-Sealed-Key"));
|
||||
assert!(is_encryption_metadata_key("X-Minio-Internal-Encrypted-Multipart"));
|
||||
assert!(!is_encryption_metadata_key("x-amz-meta-custom"));
|
||||
assert!(!is_encryption_metadata_key("x-rustfs-internal-healing"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_encryption_marker_excludes_size_only_metadata() {
|
||||
assert!(!is_object_encryption_marker(RUSTFS_ENCRYPTION_ORIGINAL_SIZE));
|
||||
assert!(is_object_encryption_marker("X-Minio-Internal-Server-Side-Encryption-Sealed-Key"));
|
||||
assert!(is_object_encryption_marker("x-amz-server-side-encryption"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_encryption_original_size_is_case_insensitive() {
|
||||
let metadata = std::collections::HashMap::from([(
|
||||
"X-Amz-Server-Side-Encryption-Customer-Original-Size".to_string(),
|
||||
"42".to_string(),
|
||||
)]);
|
||||
assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_encryption_original_size_prefers_rustfs_metadata() {
|
||||
let metadata = std::collections::HashMap::from([
|
||||
(SSEC_ORIGINAL_SIZE.to_string(), "21".to_string()),
|
||||
(RUSTFS_ENCRYPTION_ORIGINAL_SIZE.to_string(), "42".to_string()),
|
||||
]);
|
||||
assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_header() {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
@@ -37,10 +37,8 @@ pub const SUFFIX_CRC: &str = "crc";
|
||||
pub const SUFFIX_TRANSITION_STATUS: &str = "transition-status";
|
||||
pub const SUFFIX_TRANSITIONED_OBJECTNAME: &str = "transitioned-object";
|
||||
pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
|
||||
pub const SUFFIX_TRANSITIONED_VERSION_STATE: &str = "transitioned-version-state";
|
||||
pub const SUFFIX_TRANSITION_TIER: &str = "transition-tier";
|
||||
pub const SUFFIX_TRANSITION_TIER_DESTINATION_ID: &str = "transition-tier-destination-id";
|
||||
pub const SUFFIX_TRANSITION_TRANSACTION_ID: &str = "transition-transaction-id";
|
||||
pub const SUFFIX_RESTORE_OPERATION_ID: &str = "restore-operation-id";
|
||||
pub const SUFFIX_FREE_VERSION: &str = "free-version";
|
||||
pub const SUFFIX_PURGESTATUS: &str = "purgestatus";
|
||||
|
||||
@@ -98,39 +98,6 @@ rc admin ilm transition run local/mybucket --prefix logs/ --tier cold --max-obje
|
||||
|
||||
Inspect the aggregate counters before widening scope. Full object-key lists are intentionally not returned by the admin response. If `RUSTFS_RPC_SECRET` or other credentials were pasted into an issue, chat, log, or ticket while debugging tiering, rotate them on every node, restart the cluster with the new value, and redact the exposed copy before sharing more diagnostics.
|
||||
|
||||
## Reconcile an unknown transition upload
|
||||
|
||||
Historical transition transactions in `upload_outcome_unknown` state can use an explicit two-stage operator workflow when the tier probe is ambiguous and the provider supports exact version deletion. The endpoint refuses transactions that are still inside their ownership window or are in any other state.
|
||||
|
||||
First inspect the transaction without changing it:
|
||||
|
||||
```text
|
||||
GET /rustfs/admin/v3/ilm/transition/reconcile/<transaction-id>
|
||||
```
|
||||
|
||||
If independent provider evidence identifies the exact remote version to remove, submit that opaque version identifier with explicit confirmation:
|
||||
|
||||
```json
|
||||
POST /rustfs/admin/v3/ilm/transition/reconcile/<transaction-id>
|
||||
{
|
||||
"action": "delete_candidate",
|
||||
"confirm": true,
|
||||
"remote_version_id": "<exact-provider-version>"
|
||||
}
|
||||
```
|
||||
|
||||
This operation performs only an exact version delete. Its response reports whether the transaction journal was still observed after the delete; background recovery may have finalized the same transaction concurrently. If the journal remains, inspect the transaction again and finalize it only after the live provider probe proves that the candidate is missing:
|
||||
|
||||
```json
|
||||
POST /rustfs/admin/v3/ilm/transition/reconcile/<transaction-id>
|
||||
{
|
||||
"action": "finalize_missing",
|
||||
"confirm": true
|
||||
}
|
||||
```
|
||||
|
||||
`finalize_missing` re-runs the provider probe and fails closed for `unversioned_present`, `versioned_present`, `ambiguous`, `unsupported`, or probe errors. It never accepts an operator assertion in place of a live `missing` result. Providers without an authoritative probe or exact version deletion remain pending; this endpoint does not infer provider capabilities, accept external absence assertions, or select a candidate automatically.
|
||||
|
||||
## Historical fixes (for context, already merged)
|
||||
|
||||
- Expire/GET race (`NoSuchVersion` during expiry of a tiered object):
|
||||
|
||||
@@ -358,7 +358,7 @@ Fixture-backed tests should run when the fixture path is present:
|
||||
|
||||
```bash
|
||||
cargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture
|
||||
cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture
|
||||
cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture
|
||||
```
|
||||
|
||||
## Multi-Expert Adversarial Review Summary
|
||||
|
||||
@@ -20,10 +20,8 @@ use crate::admin::storage_api::error::StorageError;
|
||||
use crate::admin::storage_api::lifecycle::{
|
||||
ManualTransitionCancelCheck, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink,
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission,
|
||||
ManualTransitionScopeAdmissionClaim, TransitionOperatorDeleteResult, TransitionOperatorError,
|
||||
claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
|
||||
delete_transition_candidate_for_operator, enqueue_transition_for_existing_objects_scoped,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
|
||||
ManualTransitionScopeAdmissionClaim, claim_manual_transition_scope_admission,
|
||||
delete_manual_transition_scope_admission_if_current, enqueue_transition_for_existing_objects_scoped,
|
||||
load_manual_transition_job_record, load_manual_transition_job_record_with_etag, load_manual_transition_scope_admission,
|
||||
manual_transition_job_lease_expired, manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired,
|
||||
persist_manual_transition_job_progress, renew_manual_transition_job_lease, request_manual_transition_job_cancel,
|
||||
@@ -35,7 +33,6 @@ use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
use rustfs_utils::{
|
||||
MaskedAccessKey,
|
||||
@@ -59,7 +56,6 @@ const MAX_MANUAL_TRANSITION_DURATION_SECONDS: u64 = 3600;
|
||||
const LOG_COMPONENT_ADMIN: &str = "admin";
|
||||
const LOG_SUBSYSTEM_ILM_TRANSITION: &str = "ilm_transition";
|
||||
const EVENT_ADMIN_ILM_TRANSITION_STATE: &str = "admin_ilm_transition_state";
|
||||
const EVENT_ADMIN_ILM_TRANSITION_RECONCILE: &str = "admin_ilm_transition_reconcile";
|
||||
|
||||
static ACTIVE_MANUAL_TRANSITION_SCOPES: OnceLock<Mutex<Vec<ManualTransitionRunScope>>> = OnceLock::new();
|
||||
static ACTIVE_MANUAL_TRANSITION_JOBS: OnceLock<Mutex<HashMap<Uuid, CancellationToken>>> = OnceLock::new();
|
||||
@@ -220,16 +216,6 @@ pub fn register_ilm_transition_route(r: &mut S3Router<AdminOperation>) -> std::i
|
||||
format!("{ADMIN_PREFIX}/v3/ilm/transition/jobs/{{job_id}}").as_str(),
|
||||
AdminOperation(&ManualTransitionJobCancelHandler {}),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::GET,
|
||||
format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(),
|
||||
AdminOperation(&TransitionReconcileInspectHandler {}),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(),
|
||||
AdminOperation(&TransitionReconcileApplyHandler {}),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -401,10 +387,6 @@ fn log_manual_transition_completed(
|
||||
}
|
||||
|
||||
async fn authorize_manual_transition_request(req: &S3Request<Body>) -> S3Result<String> {
|
||||
authorize_transition_admin_request(req, AdminAction::SetTierAction).await
|
||||
}
|
||||
|
||||
async fn authorize_transition_admin_request(req: &S3Request<Body>, action: AdminAction) -> S3Result<String> {
|
||||
let Some(input_cred) = req.credentials.as_ref() else {
|
||||
return Err(s3_error!(InvalidRequest, "authentication required"));
|
||||
};
|
||||
@@ -417,119 +399,19 @@ async fn authorize_transition_admin_request(req: &S3Request<Body>, action: Admin
|
||||
.get::<Option<RemoteAddr>>()
|
||||
.and_then(|opt| opt.map(|addr| addr.0));
|
||||
|
||||
validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await?;
|
||||
validate_admin_request(
|
||||
&req.headers,
|
||||
&cred,
|
||||
owner,
|
||||
false,
|
||||
vec![Action::AdminAction(AdminAction::SetTierAction)],
|
||||
remote_addr,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(actor)
|
||||
}
|
||||
|
||||
fn transition_transaction_id_from_params(params: &Params<'_, '_>) -> S3Result<Uuid> {
|
||||
Uuid::parse_str(params.get("transaction_id").unwrap_or(""))
|
||||
.map_err(|_| s3_error!(InvalidArgument, "invalid transition transaction id"))
|
||||
}
|
||||
|
||||
fn map_transition_operator_error(err: TransitionOperatorError) -> S3Error {
|
||||
match err {
|
||||
TransitionOperatorError::NotFound => s3_error!(NoSuchKey, "transition transaction not found"),
|
||||
TransitionOperatorError::NotExpired => {
|
||||
s3_error!(OperationAborted, "transition transaction is still inside its active ownership window")
|
||||
}
|
||||
TransitionOperatorError::InvalidState(_) => {
|
||||
s3_error!(OperationAborted, "transition transaction is not eligible for operator reconciliation")
|
||||
}
|
||||
TransitionOperatorError::RemoteVersionRequired => {
|
||||
s3_error!(InvalidArgument, "an exact non-empty remote version is required")
|
||||
}
|
||||
TransitionOperatorError::CandidateNotMissing(_) => {
|
||||
s3_error!(OperationAborted, "remote candidate is not proven missing")
|
||||
}
|
||||
TransitionOperatorError::Store(_) | TransitionOperatorError::Remote(_) => {
|
||||
s3_error!(InternalError, "transition reconciliation failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum TransitionReconcileAction {
|
||||
DeleteCandidate,
|
||||
FinalizeMissing,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct TransitionReconcileRequest {
|
||||
action: TransitionReconcileAction,
|
||||
confirm: bool,
|
||||
#[serde(default)]
|
||||
remote_version_id: Option<String>,
|
||||
}
|
||||
|
||||
enum ValidatedTransitionReconcileAction<'a> {
|
||||
DeleteCandidate(&'a str),
|
||||
FinalizeMissing,
|
||||
}
|
||||
|
||||
fn validate_transition_reconcile_request(
|
||||
request: &TransitionReconcileRequest,
|
||||
) -> S3Result<ValidatedTransitionReconcileAction<'_>> {
|
||||
if !request.confirm {
|
||||
return Err(s3_error!(
|
||||
InvalidRequest,
|
||||
"transition reconciliation requires confirm=true; use GET to inspect without changes"
|
||||
));
|
||||
}
|
||||
match request.action {
|
||||
TransitionReconcileAction::DeleteCandidate => request
|
||||
.remote_version_id
|
||||
.as_deref()
|
||||
.filter(|version_id| !version_id.is_empty())
|
||||
.map(ValidatedTransitionReconcileAction::DeleteCandidate)
|
||||
.ok_or_else(|| s3_error!(InvalidArgument, "delete_candidate requires remote_version_id")),
|
||||
TransitionReconcileAction::FinalizeMissing if request.remote_version_id.is_none() => {
|
||||
Ok(ValidatedTransitionReconcileAction::FinalizeMissing)
|
||||
}
|
||||
TransitionReconcileAction::FinalizeMissing => {
|
||||
Err(s3_error!(InvalidArgument, "finalize_missing must not include remote_version_id"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TransitionCandidateDeleteResponse {
|
||||
outcome: &'static str,
|
||||
result: TransitionOperatorDeleteResult,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TransitionFinalizeMissingResponse {
|
||||
outcome: &'static str,
|
||||
journal_retained: bool,
|
||||
transaction_id: Uuid,
|
||||
}
|
||||
|
||||
fn log_transition_reconcile_applied(
|
||||
transaction_id: Uuid,
|
||||
action: &str,
|
||||
outcome: &str,
|
||||
request_id: &str,
|
||||
actor: &str,
|
||||
remote_addr: &str,
|
||||
) {
|
||||
info!(
|
||||
event = EVENT_ADMIN_ILM_TRANSITION_RECONCILE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_ILM_TRANSITION,
|
||||
operation = "transition_operator_reconcile",
|
||||
transaction_id = %transaction_id,
|
||||
action,
|
||||
outcome,
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
"admin transition reconciliation applied"
|
||||
);
|
||||
}
|
||||
|
||||
fn response_state(report: &ManualTransitionRunReport) -> &'static str {
|
||||
if report.was_truncated() || report.has_partial_enqueue() || report.tier_failure > 0 || report.transition_failed > 0 {
|
||||
"partial"
|
||||
@@ -1014,81 +896,6 @@ impl Operation for ManualTransitionJobCancelHandler {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TransitionReconcileInspectHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for TransitionReconcileInspectHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?;
|
||||
let transaction_id = transition_transaction_id_from_params(¶ms)?;
|
||||
let Some(store) = object_store_from_extensions(&req.extensions) else {
|
||||
return Err(s3_error!(InternalError, "object store is not initialized"));
|
||||
};
|
||||
let status = inspect_transition_transaction_for_operator(store, transaction_id)
|
||||
.await
|
||||
.map_err(map_transition_operator_error)?;
|
||||
json_response(&status, StatusCode::OK)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TransitionReconcileApplyHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for TransitionReconcileApplyHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
|
||||
let remote_addr = admin_remote_addr(&req).unwrap_or_default();
|
||||
let actor = authorize_transition_admin_request(&req, AdminAction::SetTierAction).await?;
|
||||
let transaction_id = transition_transaction_id_from_params(¶ms)?;
|
||||
let Some(store) = object_store_from_extensions(&req.extensions) else {
|
||||
return Err(s3_error!(InternalError, "object store is not initialized"));
|
||||
};
|
||||
let mut input = req.input;
|
||||
let body = input
|
||||
.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE)
|
||||
.await
|
||||
.map_err(|_| s3_error!(InvalidRequest, "transition reconciliation body is too large or unreadable"))?;
|
||||
let request: TransitionReconcileRequest = serde_json::from_slice(&body)
|
||||
.map_err(|_| s3_error!(InvalidRequest, "transition reconciliation request must be valid JSON"))?;
|
||||
|
||||
match validate_transition_reconcile_request(&request)? {
|
||||
ValidatedTransitionReconcileAction::DeleteCandidate(remote_version_id) => {
|
||||
let result = delete_transition_candidate_for_operator(store, transaction_id, remote_version_id)
|
||||
.await
|
||||
.map_err(map_transition_operator_error)?;
|
||||
let outcome = if result.journal_observed_after_delete {
|
||||
"exact_delete_completed_journal_observed"
|
||||
} else {
|
||||
"exact_delete_completed_journal_already_finalized"
|
||||
};
|
||||
log_transition_reconcile_applied(transaction_id, "delete_candidate", outcome, &request_id, &actor, &remote_addr);
|
||||
json_response(&TransitionCandidateDeleteResponse { outcome, result }, StatusCode::OK)
|
||||
}
|
||||
ValidatedTransitionReconcileAction::FinalizeMissing => {
|
||||
finalize_missing_transition_transaction_for_operator(store, transaction_id)
|
||||
.await
|
||||
.map_err(map_transition_operator_error)?;
|
||||
log_transition_reconcile_applied(
|
||||
transaction_id,
|
||||
"finalize_missing",
|
||||
"journal_deleted_after_missing_probe",
|
||||
&request_id,
|
||||
&actor,
|
||||
&remote_addr,
|
||||
);
|
||||
json_response(
|
||||
&TransitionFinalizeMissingResponse {
|
||||
outcome: "journal_finalized",
|
||||
journal_retained: false,
|
||||
transaction_id,
|
||||
},
|
||||
StatusCode::OK,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -1118,65 +925,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_reconcile_request_is_explicit_and_fail_closed() {
|
||||
let unconfirmed: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"delete_candidate","confirm":false,"remote_version_id":"v1"}"#)
|
||||
.expect("request should decode");
|
||||
assert!(validate_transition_reconcile_request(&unconfirmed).is_err());
|
||||
|
||||
let missing_version: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"delete_candidate","confirm":true}"#).expect("request should decode");
|
||||
assert!(validate_transition_reconcile_request(&missing_version).is_err());
|
||||
|
||||
let unsafe_finalize: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"finalize_missing","confirm":true,"remote_version_id":"v1"}"#)
|
||||
.expect("request should decode");
|
||||
assert!(validate_transition_reconcile_request(&unsafe_finalize).is_err());
|
||||
|
||||
let delete: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"delete_candidate","confirm":true,"remote_version_id":"opaque-v1"}"#)
|
||||
.expect("request should decode");
|
||||
assert!(matches!(
|
||||
validate_transition_reconcile_request(&delete),
|
||||
Ok(ValidatedTransitionReconcileAction::DeleteCandidate("opaque-v1"))
|
||||
));
|
||||
|
||||
let finalize: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"finalize_missing","confirm":true}"#).expect("request should decode");
|
||||
assert!(matches!(
|
||||
validate_transition_reconcile_request(&finalize),
|
||||
Ok(ValidatedTransitionReconcileAction::FinalizeMissing)
|
||||
));
|
||||
|
||||
assert!(
|
||||
serde_json::from_slice::<TransitionReconcileRequest>(
|
||||
br#"{"action":"finalize_missing","confirm":true,"unexpected":true}"#
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_reconcile_routes_use_read_and_write_tier_actions() {
|
||||
let src = include_str!("ilm_transition.rs");
|
||||
let inspect = src
|
||||
.split("impl Operation for TransitionReconcileInspectHandler")
|
||||
.nth(1)
|
||||
.and_then(|block| block.split("impl Operation for TransitionReconcileApplyHandler").next())
|
||||
.expect("inspect handler block");
|
||||
assert!(inspect.contains("AdminAction::ListTierAction"));
|
||||
assert!(!inspect.contains("AdminAction::SetTierAction"));
|
||||
|
||||
let apply = src
|
||||
.split("impl Operation for TransitionReconcileApplyHandler")
|
||||
.nth(1)
|
||||
.and_then(|block| block.split("#[cfg(test)]").next())
|
||||
.expect("apply handler block");
|
||||
assert!(apply.contains("AdminAction::SetTierAction"));
|
||||
assert!(!apply.contains("AdminAction::ListTierAction"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_query_defaults_to_bounded_run() {
|
||||
let (bucket, options, run_mode) =
|
||||
|
||||
@@ -363,36 +363,27 @@ impl Operation for ConfigureKmsHandler {
|
||||
// Convert request to KmsConfig
|
||||
let kms_config = configure_request.to_kms_config();
|
||||
|
||||
// Configure the service
|
||||
let (success, message, status) = match service_manager.configure(kms_config.clone()).await {
|
||||
let persisted_config = kms_config.clone();
|
||||
let (success, message, status) = match service_manager
|
||||
.configure_with_persistence(kms_config, || async move {
|
||||
save_kms_config(&persisted_config)
|
||||
.await
|
||||
.map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}")))
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
// Persist the configuration to cluster storage
|
||||
if let Err(e) = save_kms_config(&kms_config).await {
|
||||
let error_msg = format!("KMS configured in memory but failed to persist: {e}");
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "configure",
|
||||
state = "persist_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
} else {
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "configure",
|
||||
state = "configured",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS configured successfully".to_string(), status)
|
||||
}
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "configure",
|
||||
state = "configured",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS configured successfully".to_string(), status)
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to configure KMS: {e}");
|
||||
@@ -499,125 +490,61 @@ impl Operation for StartKmsHandler {
|
||||
);
|
||||
|
||||
let service_manager = kms_service_manager_from_context();
|
||||
|
||||
// Check if already running and force flag
|
||||
let current_status = service_manager.get_status().await;
|
||||
if matches!(current_status, KmsServiceStatus::Running) && !start_request.force.unwrap_or(false) {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "start",
|
||||
state = "already_running",
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let response = StartKmsResponse {
|
||||
success: false,
|
||||
message: "KMS service is already running. Use force=true to restart.".to_string(),
|
||||
status: current_status,
|
||||
};
|
||||
let json_response = match serde_json::to_string(&response) {
|
||||
Ok(json) => json,
|
||||
Err(e) => {
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
|
||||
operation = "start",
|
||||
result = "response_serialize_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
return Ok(S3Response::new((
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Body::from("Serialization error".to_string()),
|
||||
)));
|
||||
}
|
||||
};
|
||||
return Ok(S3Response::new((StatusCode::OK, Body::from(json_response))));
|
||||
}
|
||||
|
||||
// Start the service (or restart if force=true)
|
||||
let (success, message, status) =
|
||||
if start_request.force.unwrap_or(false) && matches!(current_status, KmsServiceStatus::Running) {
|
||||
// Force restart
|
||||
match service_manager.stop().await {
|
||||
Ok(()) => match service_manager.start().await {
|
||||
Ok(()) => {
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "restart",
|
||||
state = "running",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS service restarted successfully".to_string(), status)
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to restart KMS service: {e}");
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "restart",
|
||||
state = "start_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to stop KMS service for restart: {e}");
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "restart",
|
||||
state = "stop_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Normal start
|
||||
match service_manager.start().await {
|
||||
Ok(()) => {
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "start",
|
||||
state = "running",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS service started successfully".to_string(), status)
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to start KMS service: {e}");
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "start",
|
||||
state = "start_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
}
|
||||
}
|
||||
};
|
||||
let force = start_request.force.unwrap_or(false);
|
||||
let (success, message, status) = match service_manager.start_or_restart(force).await {
|
||||
Ok(rustfs_kms::KmsStartOutcome::Started) => {
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "start",
|
||||
state = "running",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS service started successfully".to_string(), status)
|
||||
}
|
||||
Ok(rustfs_kms::KmsStartOutcome::Restarted) => {
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "restart",
|
||||
state = "running",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS service restarted successfully".to_string(), status)
|
||||
}
|
||||
Ok(rustfs_kms::KmsStartOutcome::AlreadyRunning) => {
|
||||
let status = service_manager.get_status().await;
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "start",
|
||||
state = "already_running",
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(false, "KMS service is already running. Use force=true to restart.".to_string(), status)
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to start or restart KMS service: {e}");
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "start",
|
||||
state = "start_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
}
|
||||
};
|
||||
|
||||
let response = StartKmsResponse {
|
||||
success,
|
||||
@@ -774,8 +701,7 @@ impl Operation for GetKmsStatusHandler {
|
||||
|
||||
let service_manager = kms_service_manager_from_context();
|
||||
|
||||
let status = service_manager.get_status().await;
|
||||
let config = service_manager.get_redacted_config().await;
|
||||
let (status, config) = service_manager.get_redacted_state().await;
|
||||
|
||||
// Get backend type and health status
|
||||
let backend_type = config.as_ref().map(|c| c.backend.clone());
|
||||
@@ -908,36 +834,27 @@ impl Operation for ReconfigureKmsHandler {
|
||||
// Convert request to KmsConfig
|
||||
let kms_config = configure_request.to_kms_config();
|
||||
|
||||
// Reconfigure the service (stops, reconfigures, and starts)
|
||||
let (success, message, status) = match service_manager.reconfigure(kms_config.clone()).await {
|
||||
let persisted_config = kms_config.clone();
|
||||
let (success, message, status) = match service_manager
|
||||
.reconfigure_with_persistence(kms_config, || async move {
|
||||
save_kms_config(&persisted_config)
|
||||
.await
|
||||
.map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}")))
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
// Persist the configuration to cluster storage
|
||||
if let Err(e) = save_kms_config(&kms_config).await {
|
||||
let error_msg = format!("KMS reconfigured in memory but failed to persist: {e}");
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "reconfigure",
|
||||
state = "persist_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
} else {
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "reconfigure",
|
||||
state = "reconfigured",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS reconfigured and restarted successfully".to_string(), status)
|
||||
}
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "reconfigure",
|
||||
state = "reconfigured",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS reconfigured and restarted successfully".to_string(), status)
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to reconfigure KMS: {e}");
|
||||
|
||||
@@ -432,18 +432,6 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
SET_TIER,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}",
|
||||
LIST_TIER,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}",
|
||||
SET_TIER,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/rustfs/admin/v3/audit/target/list",
|
||||
@@ -1837,17 +1825,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn route_policy_uses_tier_actions_for_transition_routes() {
|
||||
fn route_policy_requires_set_tier_for_manual_transition_routes() {
|
||||
assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SET_TIER);
|
||||
assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER);
|
||||
assert_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER);
|
||||
assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", LIST_TIER);
|
||||
assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", SET_TIER);
|
||||
assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SERVER_INFO);
|
||||
assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SERVER_INFO);
|
||||
assert_not_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SERVER_INFO);
|
||||
assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", SET_TIER);
|
||||
assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", LIST_TIER);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -202,16 +202,6 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
"/v3/ilm/transition/jobs/{job_id}",
|
||||
"/v3/ilm/transition/jobs/11111111-1111-4111-8111-111111111111",
|
||||
),
|
||||
admin_route_sample(
|
||||
Method::GET,
|
||||
"/v3/ilm/transition/reconcile/{transaction_id}",
|
||||
"/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111",
|
||||
),
|
||||
admin_route_sample(
|
||||
Method::POST,
|
||||
"/v3/ilm/transition/reconcile/{transaction_id}",
|
||||
"/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111",
|
||||
),
|
||||
admin_route_sample(
|
||||
Method::DELETE,
|
||||
"/v3/ilm/transition/jobs/{job_id}",
|
||||
@@ -866,16 +856,6 @@ fn test_register_routes_cover_representative_admin_paths() {
|
||||
Method::DELETE,
|
||||
&admin_path("/v3/ilm/transition/jobs/11111111-1111-4111-8111-111111111111"),
|
||||
);
|
||||
assert_route(
|
||||
&router,
|
||||
Method::GET,
|
||||
&admin_path("/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111"),
|
||||
);
|
||||
assert_route(
|
||||
&router,
|
||||
Method::POST,
|
||||
&admin_path("/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111"),
|
||||
);
|
||||
|
||||
assert_route(&router, Method::GET, &table_catalog_path("/config"));
|
||||
assert_route(&router, Method::PUT, &table_catalog_path("/buckets/analytics"));
|
||||
|
||||
@@ -210,10 +210,6 @@ pub(crate) mod lifecycle {
|
||||
pub(crate) type ManualTransitionRunOptions =
|
||||
super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunOptions;
|
||||
pub(crate) type ManualTransitionRunReport = super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunReport;
|
||||
pub(crate) use super::ecstore_bucket::lifecycle::transition_transaction::{
|
||||
TransitionOperatorDeleteResult, TransitionOperatorError, delete_transition_candidate_for_operator,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
|
||||
};
|
||||
|
||||
pub(crate) async fn enqueue_transition_for_existing_objects_scoped(
|
||||
api: std::sync::Arc<super::ECStore>,
|
||||
|
||||
@@ -18,13 +18,12 @@ use super::handles::{
|
||||
IamHandle, KmsHandle, default_action_credential_interface, default_boot_time_interface, default_bucket_metadata_interface,
|
||||
default_bucket_monitor_interface, default_buffer_config_interface, default_deployment_id_interface,
|
||||
default_endpoints_interface, default_expiry_state_interface, default_federated_identity_interface,
|
||||
default_internode_metrics_interface, default_kms_runtime_interface, default_local_node_name_interface,
|
||||
default_lock_client_interface, default_lock_clients_interface, default_notification_system_interface,
|
||||
default_notify_interface, default_outbound_tls_runtime_interface, default_performance_metrics_interface,
|
||||
default_region_interface, default_replication_pool_interface, default_replication_stats_interface,
|
||||
default_runtime_port_interface, default_s3select_db_interface, default_scanner_metrics_interface,
|
||||
default_server_config_interface, default_storage_class_interface, default_tier_config_interface,
|
||||
default_transition_state_interface,
|
||||
default_internode_metrics_interface, default_local_node_name_interface, default_lock_client_interface,
|
||||
default_lock_clients_interface, default_notification_system_interface, default_notify_interface,
|
||||
default_outbound_tls_runtime_interface, default_performance_metrics_interface, default_region_interface,
|
||||
default_replication_pool_interface, default_replication_stats_interface, default_runtime_port_interface,
|
||||
default_s3select_db_interface, default_scanner_metrics_interface, default_server_config_interface,
|
||||
default_storage_class_interface, default_tier_config_interface, default_transition_state_interface,
|
||||
};
|
||||
use super::interfaces::{
|
||||
ActionCredentialInterface, BootTimeInterface, BucketMetadataInterface, BucketMonitorInterface, BufferConfigInterface,
|
||||
@@ -80,6 +79,7 @@ pub struct AppContext {
|
||||
impl AppContext {
|
||||
pub fn new(object_store: Arc<ECStore>, iam: Arc<dyn IamInterface>, kms: Arc<dyn KmsInterface>) -> Self {
|
||||
let object_data_cache = ObjectDataCacheAdapter::from_env_or_disabled();
|
||||
let kms_runtime = Arc::new(crate::app::context::handles::KmsRuntimeHandle::new(kms.handle()));
|
||||
// Let ecstore probe this cache inside get_object_reader, after
|
||||
// metadata resolution but before the erasure data read (backlog#802).
|
||||
crate::app::object_data_cache::register_object_data_cache_body_hook(Arc::clone(&object_data_cache));
|
||||
@@ -94,7 +94,7 @@ impl AppContext {
|
||||
iam,
|
||||
federated_identity: default_federated_identity_interface(),
|
||||
kms,
|
||||
kms_runtime: default_kms_runtime_interface(),
|
||||
kms_runtime,
|
||||
outbound_tls_runtime: default_outbound_tls_runtime_interface(),
|
||||
notify: default_notify_interface(),
|
||||
notification_system: default_notification_system_interface(),
|
||||
|
||||
@@ -128,12 +128,19 @@ impl KmsInterface for KmsHandle {
|
||||
}
|
||||
|
||||
/// Default KMS runtime interface adapter.
|
||||
#[derive(Default)]
|
||||
pub struct KmsRuntimeHandle;
|
||||
pub struct KmsRuntimeHandle {
|
||||
kms: Option<Arc<KmsServiceManager>>,
|
||||
}
|
||||
|
||||
impl KmsRuntimeHandle {
|
||||
pub fn new(kms: Arc<KmsServiceManager>) -> Self {
|
||||
Self { kms: Some(kms) }
|
||||
}
|
||||
}
|
||||
|
||||
impl KmsRuntimeInterface for KmsRuntimeHandle {
|
||||
fn service_manager(&self) -> Option<Arc<KmsServiceManager>> {
|
||||
runtime_sources::kms_service_manager()
|
||||
self.kms.clone()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -485,7 +492,9 @@ pub fn default_notification_system_interface() -> Arc<dyn NotificationSystemInte
|
||||
}
|
||||
|
||||
pub fn default_kms_runtime_interface() -> Arc<dyn KmsRuntimeInterface> {
|
||||
Arc::new(KmsRuntimeHandle)
|
||||
Arc::new(KmsRuntimeHandle {
|
||||
kms: runtime_sources::kms_service_manager(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn default_outbound_tls_runtime_interface() -> Arc<dyn OutboundTlsRuntimeInterface> {
|
||||
@@ -607,10 +616,10 @@ pub fn default_buffer_config_interface() -> Arc<dyn BufferConfigInterface> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
ServerConfigHandle, default_federated_identity_interface, federated_identity_interface,
|
||||
publish_default_federated_identity_service, runtime_sources,
|
||||
KmsRuntimeHandle, KmsServiceManager, ServerConfigHandle, default_federated_identity_interface,
|
||||
federated_identity_interface, publish_default_federated_identity_service, runtime_sources,
|
||||
};
|
||||
use crate::app::context::interfaces::ServerConfigInterface;
|
||||
use crate::app::context::interfaces::{KmsRuntimeInterface, ServerConfigInterface};
|
||||
use rustfs_config::server_config::Config;
|
||||
use rustfs_iam::{
|
||||
federation::{FederatedIdentityRegistry, FederatedIdentityService, oidc::StandardOidcAdapter},
|
||||
@@ -733,4 +742,19 @@ mod tests {
|
||||
"handle B must serve its own credentials"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kms_runtime_handles_keep_injected_managers_isolated() {
|
||||
let manager_a = Arc::new(KmsServiceManager::new());
|
||||
let manager_b = Arc::new(KmsServiceManager::new());
|
||||
let handle_a = KmsRuntimeHandle::new(manager_a.clone());
|
||||
let handle_b = KmsRuntimeHandle::new(manager_b.clone());
|
||||
|
||||
assert!(Arc::ptr_eq(&handle_a.service_manager().expect("manager A"), &manager_a));
|
||||
assert!(Arc::ptr_eq(&handle_b.service_manager().expect("manager B"), &manager_b));
|
||||
assert!(!Arc::ptr_eq(
|
||||
&handle_a.service_manager().expect("manager A"),
|
||||
&handle_b.service_manager().expect("manager B")
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,9 +198,13 @@ async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
|
||||
first_info.total_free_capacity
|
||||
);
|
||||
assert_eq!(
|
||||
second_info.total_used_capacity,
|
||||
second_info.total_capacity.saturating_sub(second_info.total_free_capacity),
|
||||
"server used capacity must stay internally consistent on repeated requests"
|
||||
(first_info.total_capacity, first_info.total_free_capacity, first_info.total_used_capacity,),
|
||||
(
|
||||
second_info.total_capacity,
|
||||
second_info.total_free_capacity,
|
||||
second_info.total_used_capacity,
|
||||
),
|
||||
"repeated data usage requests must report stable capacity values"
|
||||
);
|
||||
|
||||
// The endpoint must serve the seeded snapshot numbers, not recomputed ones.
|
||||
|
||||
@@ -60,7 +60,6 @@ use uuid::Uuid;
|
||||
static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
|
||||
static INIT: Once = Once::new();
|
||||
const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
const RESTORE_COPY_BACK_WAIT_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const ENV_GET_CODEC_STREAMING_ENABLE: &str = "RUSTFS_GET_CODEC_STREAMING_ENABLE";
|
||||
const ENV_GET_CODEC_STREAMING_ROLLOUT: &str = "RUSTFS_GET_CODEC_STREAMING_ROLLOUT";
|
||||
const ENV_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED: &str = "RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED";
|
||||
@@ -170,34 +169,6 @@ async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str,
|
||||
.expect("Failed to upload test object")
|
||||
}
|
||||
|
||||
async fn transition_uploaded_object_directly(
|
||||
ecstore: &Arc<ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
tier_name: &str,
|
||||
uploaded: &ObjectInfo,
|
||||
) -> ObjectInfo {
|
||||
let transition_opts = ObjectOptions {
|
||||
transition: lifecycle::lifecycle_contract::TransitionOptions {
|
||||
status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: uploaded.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: uploaded.version_id.map(|version| version.to_string()),
|
||||
versioned: uploaded.version_id.is_some(),
|
||||
mod_time: uploaded.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
ecstore
|
||||
.transition_object(bucket, object, &transition_opts)
|
||||
.await
|
||||
.expect("Failed to transition object directly");
|
||||
wait_for_transition(ecstore, bucket, object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should transition before restore assertions")
|
||||
}
|
||||
|
||||
async fn set_bucket_lifecycle_transition_with_tier(
|
||||
bucket_name: &str,
|
||||
storage_class: &str,
|
||||
@@ -339,45 +310,6 @@ async fn wait_for_transition(ecstore: &Arc<ECStore>, bucket: &str, object: &str,
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_restore_completion(
|
||||
ecstore: &Arc<ECStore>,
|
||||
backend: &MockWarmBackend,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: Duration,
|
||||
) -> Result<ObjectInfo, String> {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
let mut last_state = None;
|
||||
|
||||
loop {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
let tier_gets = backend.get_count().await;
|
||||
let op_log = backend.op_log().await;
|
||||
return Err(format!(
|
||||
"restore copy-back should complete within {timeout:?}; tier_gets={tier_gets}, op_log={op_log:?}; last observed state: {}",
|
||||
last_state.unwrap_or_else(|| "no object info observed".to_string())
|
||||
));
|
||||
}
|
||||
|
||||
match (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await {
|
||||
Ok(info) => {
|
||||
if !info.restore_ongoing && info.restore_expires.is_some() {
|
||||
return Ok(info);
|
||||
}
|
||||
last_state = Some(format!(
|
||||
"restore_ongoing={}, restore_expires={:?}, transitioned_status={}",
|
||||
info.restore_ongoing, info.restore_expires, info.transitioned_object.status
|
||||
));
|
||||
}
|
||||
Err(err) => {
|
||||
last_state = Some(format!("get_object_info failed: {err}"));
|
||||
}
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
// SAFETY: this helper is used only by `#[serial]` tests and runs under the single-threaded Tokio
|
||||
// runtime (`worker_threads = 1`), so no concurrent test can mutate process environment during the
|
||||
// `env::set_var` / `env::remove_var` window.
|
||||
@@ -2094,10 +2026,10 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
|
||||
|
||||
/// backlog#1148 ilm-8: the RestoreObject API surface on a transitioned object.
|
||||
///
|
||||
/// POST restore(days=1) is accepted and flips the object to
|
||||
/// `x-amz-restore: ongoing-request="true"` while the mock tier GET barrier
|
||||
/// proves the background copy-back has reached the remote read; a second POST
|
||||
/// during that window is rejected with 409 `RestoreAlreadyInProgress`; once the
|
||||
/// POST restore(days=1) is accepted and immediately flips the object to
|
||||
/// `x-amz-restore: ongoing-request="true"` (the mock tier's injected GET
|
||||
/// latency keeps the background copy-back in flight); a second POST during
|
||||
/// that window is rejected with 409 `RestoreAlreadyInProgress`; once the
|
||||
/// copy-back completes the object reports `ongoing-request="false"` with a
|
||||
/// future expiry-date; and a full GET is then served from the local restored
|
||||
/// copy (the mock tier records no further `get` calls).
|
||||
@@ -2118,17 +2050,29 @@ async fn restore_object_usecase_reports_ongoing_conflict_and_completion() {
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-api-restore-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
// Keep the object under the shared ILM test prefix even though this setup
|
||||
// transitions it directly; it keeps diagnostics aligned with sibling tests.
|
||||
// Must live under the `test/` prefix: `set_bucket_lifecycle_transition_with_tier`
|
||||
// scopes the transition rule to `<Filter><Prefix>test/</Prefix>`, so an object
|
||||
// outside it never matches, is never enqueued, and never transitions — the
|
||||
// setup `wait_for_transition` would then time out before the restore assertions.
|
||||
let object = "test/restore/api-object.bin";
|
||||
let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
|
||||
backend.clear_op_log().await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
|
||||
let get_barrier = backend.arm_get_barrier().await;
|
||||
lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(ecstore.clone(), bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned object");
|
||||
let _ = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should transition before the restore API runs");
|
||||
|
||||
// Slow the tier GET so the background copy-back stays in flight long
|
||||
// enough to observe the ongoing state and the conflict rejection.
|
||||
backend.set_latency(Some(Duration::from_millis(1500))).await;
|
||||
|
||||
let restore_request = || RestoreRequest {
|
||||
days: Some(1),
|
||||
@@ -2152,18 +2096,15 @@ async fn restore_object_usecase_reports_ongoing_conflict_and_completion() {
|
||||
.await
|
||||
.expect("restore request should be accepted");
|
||||
|
||||
get_barrier.wait_until_paused().await;
|
||||
|
||||
// The barrier proves the detached copy-back reached the tier GET and is
|
||||
// still paused, so the ongoing state and conflict rejection are not timing
|
||||
// assumptions about task scheduling.
|
||||
// The accepted restore is immediately visible as ongoing (the metadata is
|
||||
// written synchronously before the copy-back is spawned).
|
||||
let ongoing = ecstore
|
||||
.get_object_info(bucket.as_str(), object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to load object info during restore");
|
||||
assert!(
|
||||
ongoing.restore_ongoing,
|
||||
"x-amz-restore must report ongoing-request=true while the copy-back tier GET is paused"
|
||||
"x-amz-restore must report ongoing-request=true right after the restore is accepted"
|
||||
);
|
||||
|
||||
// A second restore while one is in flight is rejected.
|
||||
@@ -2176,12 +2117,21 @@ async fn restore_object_usecase_reports_ongoing_conflict_and_completion() {
|
||||
"unexpected rejection for a repeated restore: {err:?}"
|
||||
);
|
||||
|
||||
get_barrier.release();
|
||||
|
||||
// Completion: ongoing flips to false and a future expiry-date appears.
|
||||
let completed =
|
||||
wait_for_restore_completion(&ecstore, &backend, bucket.as_str(), object, RESTORE_COPY_BACK_WAIT_TIMEOUT).await;
|
||||
let completed = completed.unwrap_or_else(|err| panic!("{err}"));
|
||||
let mut completed = None;
|
||||
for _ in 0..40 {
|
||||
let info = ecstore
|
||||
.get_object_info(bucket.as_str(), object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to poll object info for restore completion");
|
||||
if !info.restore_ongoing && info.restore_expires.is_some() {
|
||||
completed = Some(info);
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
let completed = completed.expect("restore copy-back should complete within the poll window");
|
||||
backend.clear_faults().await;
|
||||
|
||||
let now_secs = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
@@ -2226,18 +2176,27 @@ async fn restore_object_usecase_accepts_exactly_one_of_two_concurrent_restores()
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-api-restore-cas-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
// Keep the object under the shared ILM test prefix for diagnostics parity.
|
||||
// Must live under the `test/` prefix — the shared transition rule filters on it.
|
||||
let object = "test/restore/cas-object.bin";
|
||||
let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
|
||||
backend.clear_op_log().await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
|
||||
// Hold the accepted copy-back at the tier GET until both accept attempts
|
||||
// return, so the loser cannot observe an already-restored object.
|
||||
let get_barrier = backend.arm_get_barrier().await;
|
||||
lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(ecstore.clone(), bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned object");
|
||||
let _ = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should transition before the concurrent restores run");
|
||||
|
||||
// Keep the winner's copy-back in flight while the loser's accept runs, so
|
||||
// the loser cannot slip into the already-restored path after a completed
|
||||
// copy-back.
|
||||
backend.set_latency(Some(Duration::from_millis(1500))).await;
|
||||
|
||||
let tier_gets_before_restore = backend.get_count().await;
|
||||
|
||||
@@ -2282,34 +2241,27 @@ async fn restore_object_usecase_accepts_exactly_one_of_two_concurrent_restores()
|
||||
"the losing concurrent restore must be rejected as already in progress: {rejection:?}"
|
||||
);
|
||||
|
||||
get_barrier.wait_until_paused().await;
|
||||
get_barrier.release();
|
||||
|
||||
// This test is scoped to the accept CAS: completion, expiry metadata, and
|
||||
// local restored GET service are covered by the single-request restore test
|
||||
// above. Here it is enough to prove exactly one copy-back was admitted.
|
||||
let expected_tier_gets = tier_gets_before_restore + 1;
|
||||
let deadline = tokio::time::Instant::now() + TRANSITION_WAIT_TIMEOUT;
|
||||
loop {
|
||||
let actual_tier_gets = backend.get_count().await;
|
||||
if actual_tier_gets >= expected_tier_gets {
|
||||
assert_eq!(
|
||||
actual_tier_gets - tier_gets_before_restore,
|
||||
1,
|
||||
"two concurrent restore requests must trigger exactly one tier copy-back GET"
|
||||
);
|
||||
// Let the single accepted copy-back complete, then verify the tier saw
|
||||
// exactly one restore read — a second GET means a double copy-back.
|
||||
let mut completed = false;
|
||||
for _ in 0..40 {
|
||||
let info = ecstore
|
||||
.get_object_info(bucket.as_str(), object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to poll object info for restore completion");
|
||||
if !info.restore_ongoing && info.restore_expires.is_some() {
|
||||
completed = true;
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
let op_log = backend.op_log().await;
|
||||
panic!(
|
||||
"mock tier should record exactly one restore GET within {TRANSITION_WAIT_TIMEOUT:?}; \
|
||||
tier_gets_before_restore={tier_gets_before_restore}, actual_tier_gets={actual_tier_gets}, op_log={op_log:?}"
|
||||
);
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
backend.clear_faults().await;
|
||||
assert!(completed, "the accepted restore copy-back should complete within the poll window");
|
||||
assert_eq!(
|
||||
backend.get_count().await - tier_gets_before_restore,
|
||||
1,
|
||||
"two concurrent restore requests must trigger exactly one tier copy-back GET"
|
||||
);
|
||||
}
|
||||
|
||||
/// rustfs/backlog#1320: a single PUT must compute the replication decision
|
||||
|
||||
@@ -763,7 +763,7 @@ async fn enqueue_transitioned_delete_cleanup(
|
||||
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup);
|
||||
|
||||
let je = if opts.delete_prefix {
|
||||
tier_sweeper::transitioned_force_delete_journal_entry(&existing.transitioned_object, existing.transition_version_state)
|
||||
tier_sweeper::transitioned_force_delete_journal_entry(&existing.transitioned_object)
|
||||
} else {
|
||||
let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok());
|
||||
tier_sweeper::transitioned_delete_journal_entry(
|
||||
@@ -771,7 +771,6 @@ async fn enqueue_transitioned_delete_cleanup(
|
||||
opts.versioned,
|
||||
opts.version_suspended,
|
||||
&existing.transitioned_object,
|
||||
existing.transition_version_state,
|
||||
)
|
||||
};
|
||||
let Some(mut je) = je else {
|
||||
@@ -9684,7 +9683,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn transitioned_delete_cleanup_persists_known_state_and_rejects_unknown_state() {
|
||||
async fn transitioned_delete_cleanup_persists_identity_bound_and_legacy_journals() {
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
if current_app_context().is_none() {
|
||||
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
|
||||
@@ -9704,9 +9703,8 @@ mod tests {
|
||||
current.transitioned_object.tier = "WARM".to_string();
|
||||
current.transitioned_object.name = "remote/identity-bound".to_string();
|
||||
current.transitioned_object.version_id = "remote-version".to_string();
|
||||
current.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
|
||||
|
||||
let journal_name = |remote_object: &str, backend_identity: Option<[u8; 32]>, version_id_exact: bool| {
|
||||
let journal_name = |remote_object: &str, backend_identity: Option<[u8; 32]>| {
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
let mut hasher = Sha256::new();
|
||||
@@ -9719,10 +9717,6 @@ mod tests {
|
||||
hasher.update([0]);
|
||||
hasher.update(backend_identity);
|
||||
}
|
||||
if version_id_exact {
|
||||
hasher.update([0]);
|
||||
hasher.update(b"exact-version-id");
|
||||
}
|
||||
format!("ilm/tier-delete-journal/{}.json", rustfs_utils::crypto::hex(hasher.finalize().as_slice()))
|
||||
};
|
||||
|
||||
@@ -9732,7 +9726,7 @@ mod tests {
|
||||
let mut identity_bound = store
|
||||
.get_object_reader(
|
||||
".rustfs.sys",
|
||||
&journal_name("remote/identity-bound", Some(identity), true),
|
||||
&journal_name("remote/identity-bound", Some(identity)),
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions::default(),
|
||||
@@ -9745,14 +9739,11 @@ mod tests {
|
||||
.expect("identity-bound journal body should be readable");
|
||||
let identity_bound: serde_json::Value =
|
||||
serde_json::from_slice(&identity_bound_data).expect("identity-bound journal should decode as JSON");
|
||||
assert_eq!(identity_bound["version"], serde_json::json!(4));
|
||||
assert_eq!(identity_bound["version"], serde_json::json!(2));
|
||||
assert_eq!(identity_bound["backend_identity"], serde_json::json!(identity));
|
||||
assert_eq!(identity_bound["version_id_exact"], serde_json::json!(true));
|
||||
assert_eq!(identity_bound["version_state"], serde_json::json!("exact"));
|
||||
|
||||
current.user_defined = Arc::new(HashMap::new());
|
||||
current.transitioned_object.name = "remote/legacy".to_string();
|
||||
current.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
enqueue_transitioned_delete_cleanup(
|
||||
store.clone(),
|
||||
"bucket",
|
||||
@@ -9764,31 +9755,24 @@ mod tests {
|
||||
Some(¤t),
|
||||
)
|
||||
.await
|
||||
.expect("unknown force-delete cleanup should fail closed without a journal");
|
||||
let legacy_err = match store
|
||||
.expect("legacy force-delete cleanup should persist a fail-closed v1 journal");
|
||||
let mut legacy = store
|
||||
.get_object_reader(
|
||||
".rustfs.sys",
|
||||
&journal_name("remote/legacy", None, false),
|
||||
&journal_name("remote/legacy", None),
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("unknown remote version state must not persist a delete journal"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(
|
||||
matches!(
|
||||
&legacy_err,
|
||||
StorageError::FileNotFound
|
||||
| StorageError::ObjectNotFound(_, _)
|
||||
| StorageError::FileVersionNotFound
|
||||
| StorageError::VersionNotFound(_, _, _)
|
||||
| StorageError::VolumeNotFound
|
||||
),
|
||||
"unknown remote version state must leave no journal, got {legacy_err:?}"
|
||||
);
|
||||
.expect("legacy journal should be readable");
|
||||
let mut legacy_data = Vec::new();
|
||||
tokio::io::AsyncReadExt::read_to_end(&mut legacy.stream, &mut legacy_data)
|
||||
.await
|
||||
.expect("legacy journal body should be readable");
|
||||
let legacy: serde_json::Value = serde_json::from_slice(&legacy_data).expect("legacy journal should decode as JSON");
|
||||
assert_eq!(legacy["version"], serde_json::json!(1));
|
||||
assert_eq!(legacy["backend_identity"], serde_json::Value::Null);
|
||||
}
|
||||
|
||||
async fn put_real_cold_fill_object(store: &Arc<ECStore>, bucket: &str, object: &str, body: &[u8]) -> ObjectInfo {
|
||||
|
||||
@@ -409,24 +409,20 @@ pub(crate) mod bucket {
|
||||
versioned: bool,
|
||||
suspended: bool,
|
||||
transitioned: &super::super::super::storage_contracts::TransitionedObject,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
) -> Option<Jentry> {
|
||||
crate::storage::storage_api::ecstore_bucket::lifecycle::tier_sweeper::transitioned_delete_journal_entry(
|
||||
version_id,
|
||||
versioned,
|
||||
suspended,
|
||||
transitioned,
|
||||
transition_version_state,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn transitioned_force_delete_journal_entry(
|
||||
transitioned: &super::super::super::storage_contracts::TransitionedObject,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
) -> Option<Jentry> {
|
||||
crate::storage::storage_api::ecstore_bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(
|
||||
transitioned,
|
||||
transition_version_state,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
+18
-106
@@ -14,7 +14,7 @@
|
||||
|
||||
use super::{
|
||||
module_switch::{resolve_notify_module_state, validate_notify_module_env, with_refreshed_notify_module_state_from},
|
||||
refresh_persisted_module_switches_from, runtime_sources,
|
||||
refresh_persisted_module_switches_from_store, runtime_sources,
|
||||
};
|
||||
use crate::storage_api::server::event::{
|
||||
EventArgs as EcstoreEventArgs, StorageObjectInfo, read_existing_server_config_no_lock, register_event_dispatch_hook,
|
||||
@@ -309,36 +309,30 @@ pub async fn shutdown_event_notifier() -> Result<(), NotificationError> {
|
||||
|
||||
#[instrument]
|
||||
pub async fn init_event_notifier() -> Result<(), NotificationError> {
|
||||
init_event_notifier_with_store(runtime_sources::current_object_store_handle).await
|
||||
}
|
||||
|
||||
async fn init_event_notifier_with_store<CurrentStore>(current_store: CurrentStore) -> Result<(), NotificationError>
|
||||
where
|
||||
CurrentStore: FnOnce() -> Option<std::sync::Arc<rustfs_notify::NotifyStore>>,
|
||||
{
|
||||
mark_event_notifier_unreconciled();
|
||||
validate_notify_module_env().map_err(NotificationError::Initialization)?;
|
||||
let system = ensure_live_events_initialized();
|
||||
|
||||
let Some(store) = current_store() else {
|
||||
let enabled = refresh_notify_module_enabled();
|
||||
if enabled {
|
||||
return Err(NotificationError::Initialization(
|
||||
"failed to refresh notify module switch: storage layer not initialized".to_string(),
|
||||
));
|
||||
}
|
||||
initialize_live_event_support(&system, None).await?;
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
refresh_persisted_module_switches_from(store.clone())
|
||||
refresh_persisted_module_switches_from_store()
|
||||
.await
|
||||
.map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))?;
|
||||
|
||||
let enabled = refresh_notify_module_enabled();
|
||||
|
||||
if !enabled {
|
||||
initialize_live_event_support(&system, Some(store)).await?;
|
||||
info!(
|
||||
target: "rustfs::main::init_event_notifier",
|
||||
"Notify module is disabled, initializing live event stream support only. Set {}=true to enable notification targets.",
|
||||
rustfs_config::ENV_NOTIFY_ENABLE
|
||||
);
|
||||
if system.runtime_lifecycle_state() != NotificationRuntimeState::LiveOnly {
|
||||
system.set_targets_enabled(false, None).await?;
|
||||
}
|
||||
system.reload_persisted_config().await?;
|
||||
info!(
|
||||
target: "rustfs::main::init_event_notifier",
|
||||
"Live event stream support initialized successfully."
|
||||
);
|
||||
ensure_event_notifier_converged(&system)?;
|
||||
mark_event_notifier_reconciled();
|
||||
return Ok(());
|
||||
}
|
||||
@@ -353,7 +347,7 @@ where
|
||||
"Event notifier configuration found, proceeding with initialization."
|
||||
);
|
||||
|
||||
system.reload_persisted_config_from_store(store).await?;
|
||||
system.reload_persisted_config().await?;
|
||||
let runtime_state = system.runtime_lifecycle_state();
|
||||
if !matches!(runtime_state, NotificationRuntimeState::TargetsEnabled { .. }) {
|
||||
system.set_targets_enabled(true, None).await?;
|
||||
@@ -367,53 +361,13 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize_live_event_support(
|
||||
system: &NotificationSystem,
|
||||
store: Option<std::sync::Arc<rustfs_notify::NotifyStore>>,
|
||||
) -> Result<(), NotificationError> {
|
||||
if store.is_some() {
|
||||
info!(
|
||||
target: "rustfs::main::init_event_notifier",
|
||||
"Notify module is disabled, initializing live event stream support only. Set {}=true to enable notification targets.",
|
||||
rustfs_config::ENV_NOTIFY_ENABLE
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
target: "rustfs::main::init_event_notifier",
|
||||
"Notify module is disabled, initializing live event stream support only. Persisted notification target reconciliation is deferred until storage is initialized. Set {}=true to enable notification targets.",
|
||||
rustfs_config::ENV_NOTIFY_ENABLE
|
||||
);
|
||||
}
|
||||
|
||||
if system.runtime_lifecycle_state() != NotificationRuntimeState::LiveOnly {
|
||||
system.set_targets_enabled(false, None).await?;
|
||||
}
|
||||
if let Some(store) = store {
|
||||
system.reload_persisted_config_from_store(store).await?;
|
||||
}
|
||||
info!(
|
||||
target: "rustfs::main::init_event_notifier",
|
||||
"Live event stream support initialized successfully."
|
||||
);
|
||||
ensure_event_notifier_converged(system)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::module_switch::{
|
||||
PersistedModuleSwitches, current_persisted_module_switches, persisted_module_switches_configured,
|
||||
set_persisted_module_switches,
|
||||
};
|
||||
use super::{
|
||||
convert_ecstore_object_info, init_event_notifier_with_store, parse_host_and_port, run_persisted_event_notifier_reconciler,
|
||||
};
|
||||
use crate::server::is_event_notifier_reconciled;
|
||||
use super::{convert_ecstore_object_info, parse_host_and_port, run_persisted_event_notifier_reconciler};
|
||||
use crate::storage_api::server::event::StorageObjectInfo;
|
||||
use crate::storage_api::server::event::contract::lifecycle::TransitionedObject;
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_notify::NotificationError;
|
||||
use rustfs_notify::NotificationRuntimeState;
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
future::pending,
|
||||
@@ -427,48 +381,6 @@ mod tests {
|
||||
use tokio::sync::Notify;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disabled_notify_without_storage_initializes_live_events_without_reconcile() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, Some("false"))], async {
|
||||
let previous = current_persisted_module_switches();
|
||||
let previous_configured = persisted_module_switches_configured();
|
||||
set_persisted_module_switches(PersistedModuleSwitches::default(), false);
|
||||
|
||||
init_event_notifier_with_store(|| None)
|
||||
.await
|
||||
.expect("disabled notify should not require storage during early bootstrap");
|
||||
|
||||
let system = rustfs_notify::notification_system().expect("live event container should be initialized");
|
||||
assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::LiveOnly);
|
||||
assert!(
|
||||
!is_event_notifier_reconciled(),
|
||||
"early live-only bootstrap must not claim persisted notify config is reconciled"
|
||||
);
|
||||
|
||||
set_persisted_module_switches(previous, previous_configured);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn enabled_notify_without_storage_still_fails_visible() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"))], async {
|
||||
let previous = current_persisted_module_switches();
|
||||
let previous_configured = persisted_module_switches_configured();
|
||||
set_persisted_module_switches(PersistedModuleSwitches::default(), false);
|
||||
|
||||
let err = init_event_notifier_with_store(|| None)
|
||||
.await
|
||||
.expect_err("enabled notify should still fail when storage is unavailable");
|
||||
assert!(err.to_string().contains("storage layer not initialized"));
|
||||
|
||||
set_persisted_module_switches(previous, previous_configured);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_and_port_with_ipv4_and_port() {
|
||||
let (host, port) = parse_host_and_port("127.0.0.1:9000".to_string());
|
||||
|
||||
@@ -80,7 +80,7 @@ pub(crate) fn current_persisted_module_switches() -> PersistedModuleSwitches {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn persisted_module_switches_configured() -> bool {
|
||||
fn persisted_module_switches_configured() -> bool {
|
||||
PERSISTED_MODULE_SWITCH_CONFIGURED.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
|
||||
+11
-9
@@ -4,14 +4,13 @@ use std::fs;
|
||||
use std::io::Cursor;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use super::sse::SseObjectEncryptionResolver;
|
||||
use super::storage_api::ecstore_test_support::{
|
||||
DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
|
||||
};
|
||||
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
use storage_api::minio_generated_read::{
|
||||
DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
|
||||
};
|
||||
use temp_env::async_with_vars;
|
||||
use tokio::io::{AsyncReadExt, AsyncWrite};
|
||||
|
||||
@@ -49,7 +48,7 @@ impl AsyncWrite for VecAsyncWriter {
|
||||
fn fixture_root() -> PathBuf {
|
||||
std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/minio-generated"))
|
||||
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../crates/rio-v2/tests/fixtures/minio-generated"))
|
||||
}
|
||||
|
||||
fn case_dir(case_id: &str) -> PathBuf {
|
||||
@@ -137,12 +136,14 @@ async fn read_fixture_plaintext(encrypted: Vec<u8>, object_info: ObjectInfo, kms
|
||||
("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
|
||||
],
|
||||
async move {
|
||||
let (mut reader, offset, length) = GetObjectReader::new(
|
||||
let resolver = SseObjectEncryptionResolver;
|
||||
let (mut reader, offset, length) = GetObjectReader::new_with_resolver(
|
||||
Box::new(Cursor::new(encrypted)),
|
||||
None,
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&http::HeaderMap::new(),
|
||||
Some(&resolver),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?;
|
||||
@@ -219,11 +220,12 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil
|
||||
readers.push(reader);
|
||||
}
|
||||
|
||||
let erasure = Erasure::new(
|
||||
let erasure = Erasure::try_new(
|
||||
file_info.erasure.data_blocks,
|
||||
file_info.erasure.parity_blocks,
|
||||
file_info.erasure.block_size,
|
||||
);
|
||||
)
|
||||
.expect("fixture erasure geometry");
|
||||
let mut writer = VecAsyncWriter::default();
|
||||
let (written, err) = erasure.decode(&mut writer, readers, 0, part.size, part.size).await;
|
||||
if let Some(err) = err {
|
||||
@@ -36,6 +36,8 @@ mod ecfs_extend;
|
||||
mod ecfs_test;
|
||||
pub(crate) mod head_prefix;
|
||||
#[cfg(test)]
|
||||
mod minio_generated_read_test;
|
||||
#[cfg(test)]
|
||||
mod multi_factor_scheduler_integration_test;
|
||||
pub(crate) mod runtime_sources;
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -49,7 +49,7 @@ use std::{
|
||||
collections::HashMap,
|
||||
io::Cursor,
|
||||
pin::Pin,
|
||||
sync::{Arc, LazyLock, OnceLock},
|
||||
sync::{Arc, OnceLock},
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::spawn;
|
||||
@@ -116,7 +116,6 @@ fn remove_heal_control_replay(
|
||||
}
|
||||
|
||||
static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new();
|
||||
static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
|
||||
fn admit_heal_control_replay(
|
||||
replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>,
|
||||
@@ -449,19 +448,6 @@ impl HealControlRpcService {
|
||||
pub(crate) async fn initialize_heal_topology_fingerprint(
|
||||
cache: Arc<tokio::sync::OnceCell<String>>,
|
||||
endpoint_pools: EndpointServerPools,
|
||||
) -> Result<(), String> {
|
||||
initialize_heal_topology_fingerprint_with_probe(
|
||||
cache,
|
||||
endpoint_pools,
|
||||
crate::storage::storage_api::start_remote_version_state_fleet_probe,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn initialize_heal_topology_fingerprint_with_probe(
|
||||
cache: Arc<tokio::sync::OnceCell<String>>,
|
||||
endpoint_pools: EndpointServerPools,
|
||||
start_probe: impl FnOnce(String),
|
||||
) -> Result<(), String> {
|
||||
if cache.get().is_some() {
|
||||
return Ok(());
|
||||
@@ -469,8 +455,7 @@ async fn initialize_heal_topology_fingerprint_with_probe(
|
||||
let fingerprint = tokio::task::spawn_blocking(move || heal::heal_topology_fingerprint(&endpoint_pools))
|
||||
.await
|
||||
.map_err(|_| "heal control topology calculation task failed".to_string())??;
|
||||
let _ = cache.set(fingerprint.clone());
|
||||
start_probe(fingerprint);
|
||||
let _ = cache.set(fingerprint);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -804,35 +789,6 @@ impl heal_control_service_server::HealControlService for HealControlRpcService {
|
||||
response_proof: Bytes::new(),
|
||||
}));
|
||||
}
|
||||
if rustfs_protos::is_remote_version_state_capability_probe(&request.get_ref().command) {
|
||||
let topology_member = self
|
||||
.endpoint_pools()
|
||||
.await
|
||||
.ok_or_else(|| Status::failed_precondition("heal control topology is not initialized"))?
|
||||
.peers()
|
||||
.1;
|
||||
if topology_member.is_empty() {
|
||||
return Err(Status::failed_precondition("local topology member identity is unavailable"));
|
||||
}
|
||||
let result =
|
||||
rustfs_protos::encode_remote_version_state_capability(&topology_member, NODE_CAPABILITY_SERVER_EPOCH.as_bytes())
|
||||
.map_err(|_| Status::internal("remote version state capability length cannot be represented"))?;
|
||||
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
|
||||
request.get_ref().version,
|
||||
&request.get_ref().topology_fingerprint,
|
||||
&request.get_ref().command,
|
||||
&result,
|
||||
)
|
||||
.map_err(|_| Status::internal("heal control response length cannot be represented"))?;
|
||||
let response_proof = sign_tonic_rpc_response_proof(&canonical_response)
|
||||
.map_err(|_| Status::internal("heal control response proof is unavailable"))?;
|
||||
return Ok(Response::new(HealControlResponse {
|
||||
success: true,
|
||||
result: result.into(),
|
||||
error_info: None,
|
||||
response_proof: response_proof.into(),
|
||||
}));
|
||||
}
|
||||
let endpoints = self
|
||||
.endpoint_pools()
|
||||
.await
|
||||
@@ -2071,10 +2027,10 @@ mod tests {
|
||||
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message,
|
||||
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint,
|
||||
initialize_heal_topology_fingerprint_with_probe, legacy_scanner_activity_response, make_heal_control_server,
|
||||
make_heal_control_server_with_cache, make_server, make_server_for_context, make_tier_mutation_control_server_for_context,
|
||||
previous_scanner_activity_response, remove_heal_control_replay, scanner_activity_response, stop_rebalance_response,
|
||||
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, legacy_scanner_activity_response,
|
||||
make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context,
|
||||
make_tier_mutation_control_server_for_context, previous_scanner_activity_response, remove_heal_control_replay,
|
||||
scanner_activity_response, stop_rebalance_response,
|
||||
};
|
||||
use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
|
||||
use crate::storage::storage_api::rpc_consumer::node_service::{HealBucketInfo, HealEndpoint};
|
||||
@@ -2122,7 +2078,6 @@ mod tests {
|
||||
use tokio::time::Duration;
|
||||
use tokio_stream::wrappers::TcpListenerStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
use uuid::Uuid;
|
||||
|
||||
struct HealControlMockStorage;
|
||||
|
||||
@@ -3003,60 +2958,6 @@ mod tests {
|
||||
assert_eq!(non_coordinator.code(), tonic::Code::FailedPrecondition);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_version_state_probe_authenticates_topology_challenge_and_process_epoch() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("remote-version-state-node-service-test-secret".to_string());
|
||||
let endpoints = heal_control_test_endpoints_with_coordinator("node-d", true);
|
||||
let fingerprint = heal_topology_fingerprint(&endpoints).expect("test topology should hash");
|
||||
let (service, source) = super::make_heal_control_server_for_source();
|
||||
*source.write().await = Some(endpoints);
|
||||
let probe_command = rustfs_protos::remote_version_state_capability_probe(&[7; 16]);
|
||||
let mut request = Request::new(HealControlRequest {
|
||||
version: rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
|
||||
topology_fingerprint: fingerprint.clone(),
|
||||
command: Bytes::from(probe_command.clone()),
|
||||
});
|
||||
let body = rustfs_protos::canonical_heal_control_request_body(
|
||||
request.get_ref().version,
|
||||
&request.get_ref().topology_fingerprint,
|
||||
&request.get_ref().command,
|
||||
)
|
||||
.expect("probe should encode");
|
||||
set_tonic_canonical_body_digest(&mut request, &body).expect("digest metadata should encode");
|
||||
mark_v2_authenticated(&mut request);
|
||||
let response = service
|
||||
.heal_control(request)
|
||||
.await
|
||||
.expect("matching topology should be acknowledged")
|
||||
.into_inner();
|
||||
|
||||
let (topology_member, process_epoch) =
|
||||
rustfs_protos::decode_remote_version_state_capability(&response.result).expect("capability response should decode");
|
||||
assert_eq!(topology_member, "node-a:9000");
|
||||
let server_epoch = Uuid::from_slice(process_epoch).expect("server epoch should be a UUID");
|
||||
assert!(!server_epoch.is_nil());
|
||||
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
|
||||
rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
|
||||
&fingerprint,
|
||||
&probe_command,
|
||||
&response.result,
|
||||
)
|
||||
.expect("response should encode");
|
||||
crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
|
||||
.expect("outer proof should bind the response to the request");
|
||||
|
||||
let different_probe = rustfs_protos::remote_version_state_capability_probe(&[8; 16]);
|
||||
let different_response = rustfs_protos::canonical_heal_control_response_body(
|
||||
rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
|
||||
&fingerprint,
|
||||
&different_probe,
|
||||
&response.result,
|
||||
)
|
||||
.expect("different response should encode");
|
||||
crate::storage::storage_api::verify_tonic_rpc_response_proof(&different_response, &response.response_proof)
|
||||
.expect_err("proof from one challenge must not be reusable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_control_coordinator_rejects_expired_and_non_admin_starts() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("heal-control-node-service-test-secret".to_string());
|
||||
@@ -3119,15 +3020,10 @@ mod tests {
|
||||
let topology = heal_control_test_endpoints("node-d");
|
||||
let expected = heal_topology_fingerprint(&topology).expect("test topology should hash");
|
||||
let cache = Arc::new(tokio::sync::OnceCell::new());
|
||||
let started_probe = Arc::new(std::sync::Mutex::new(None));
|
||||
let started_probe_capture = Arc::clone(&started_probe);
|
||||
initialize_heal_topology_fingerprint_with_probe(Arc::clone(&cache), topology, move |fingerprint| {
|
||||
*started_probe_capture.lock().expect("probe capture should not poison") = Some(fingerprint);
|
||||
})
|
||||
.await
|
||||
.expect("valid topology should initialize");
|
||||
initialize_heal_topology_fingerprint(Arc::clone(&cache), topology)
|
||||
.await
|
||||
.expect("valid topology should initialize");
|
||||
assert_eq!(cache.get(), Some(&expected));
|
||||
assert_eq!(started_probe.lock().expect("probe capture should not poison").as_ref(), Some(&expected));
|
||||
|
||||
let mut invalid = heal_control_test_endpoints("node-d");
|
||||
invalid.as_mut()[0].endpoints.as_mut()[0].pool_idx = -1;
|
||||
|
||||
+270
-13
@@ -70,6 +70,10 @@
|
||||
//! ```
|
||||
|
||||
use super::StorageError;
|
||||
use super::storage_api::ecstore_object::{
|
||||
EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial,
|
||||
ReadEncryptionMode, ReadEncryptionRequest,
|
||||
};
|
||||
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use aes_gcm::aead::Payload;
|
||||
@@ -144,6 +148,7 @@ use rustfs_utils::http::headers::{
|
||||
};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId};
|
||||
use std::borrow::Cow;
|
||||
|
||||
// ============================================================================
|
||||
// High-Level SSE Configuration
|
||||
@@ -641,6 +646,23 @@ pub(crate) fn validate_sse_headers_for_read(metadata: &HashMap<String, String>,
|
||||
}
|
||||
|
||||
pub(crate) fn map_get_object_reader_error(err: StorageError) -> ApiError {
|
||||
if let StorageError::Io(io_error) = &err
|
||||
&& let Some(resolution_error) = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<EncryptionResolutionError>())
|
||||
{
|
||||
let code = match resolution_error.kind() {
|
||||
EncryptionResolutionErrorKind::InvalidRequest => S3ErrorCode::InvalidRequest,
|
||||
EncryptionResolutionErrorKind::ServiceUnavailable => S3ErrorCode::ServiceUnavailable,
|
||||
_ => S3ErrorCode::InternalError,
|
||||
};
|
||||
return ApiError {
|
||||
code,
|
||||
message: resolution_error.to_string(),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
if let Some(message) = map_ssec_get_object_reader_error_message(&err) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::InvalidRequest,
|
||||
@@ -763,6 +785,104 @@ pub enum EncryptionKeyKind {
|
||||
Object,
|
||||
}
|
||||
|
||||
pub(crate) struct SseObjectEncryptionResolver;
|
||||
|
||||
#[async_trait]
|
||||
impl ObjectEncryptionResolver for SseObjectEncryptionResolver {
|
||||
async fn resolve_read_material(
|
||||
&self,
|
||||
request: ReadEncryptionRequest<'_>,
|
||||
) -> Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
|
||||
let metadata = normalize_encryption_metadata_case(request.metadata)?;
|
||||
let (_, customer_key, customer_key_md5) =
|
||||
extract_ssec_params_from_headers(request.headers).map_err(map_encryption_resolution_error)?;
|
||||
let material = sse_decryption(DecryptionRequest {
|
||||
bucket: request.bucket,
|
||||
key: request.object,
|
||||
metadata: &metadata,
|
||||
sse_customer_key: customer_key.as_ref(),
|
||||
sse_customer_key_md5: customer_key_md5.as_ref(),
|
||||
})
|
||||
.await
|
||||
.map_err(map_encryption_resolution_error)?;
|
||||
|
||||
Ok(material.map(|material| ReadEncryptionMaterial {
|
||||
key_bytes: material.key_bytes,
|
||||
mode: match material.key_kind {
|
||||
EncryptionKeyKind::Direct => ReadEncryptionMode::Direct {
|
||||
base_nonce: material.base_nonce,
|
||||
},
|
||||
EncryptionKeyKind::Object => ReadEncryptionMode::Object,
|
||||
},
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_encryption_metadata_case(
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<Cow<'_, HashMap<String, String>>, EncryptionResolutionError> {
|
||||
const CANONICAL_KEYS: &[&str] = &[
|
||||
"x-amz-server-side-encryption",
|
||||
"x-amz-server-side-encryption-aws-kms-key-id",
|
||||
"x-amz-server-side-encryption-customer-algorithm",
|
||||
"x-amz-server-side-encryption-customer-key-md5",
|
||||
SSEC_ORIGINAL_SIZE_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
"x-rustfs-encryption-context",
|
||||
"x-rustfs-encryption-tag",
|
||||
INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
|
||||
];
|
||||
|
||||
let needs_normalization = metadata.keys().any(|key| {
|
||||
CANONICAL_KEYS
|
||||
.iter()
|
||||
.any(|canonical| key != canonical && key.eq_ignore_ascii_case(canonical))
|
||||
});
|
||||
if !needs_normalization {
|
||||
return Ok(Cow::Borrowed(metadata));
|
||||
}
|
||||
|
||||
let mut normalized = metadata.clone();
|
||||
for canonical in CANONICAL_KEYS {
|
||||
let mut matching_values = metadata
|
||||
.iter()
|
||||
.filter_map(|(key, value)| key.eq_ignore_ascii_case(canonical).then_some(value));
|
||||
let Some(value) = matching_values.next() else {
|
||||
continue;
|
||||
};
|
||||
if matching_values.any(|candidate| candidate != value) {
|
||||
return Err(EncryptionResolutionError::new(
|
||||
EncryptionResolutionErrorKind::InvalidMetadata,
|
||||
format!("conflicting object encryption metadata for {canonical}"),
|
||||
));
|
||||
}
|
||||
if !normalized.contains_key(*canonical) {
|
||||
normalized.insert((*canonical).to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
Ok(Cow::Owned(normalized))
|
||||
}
|
||||
|
||||
fn map_encryption_resolution_error(error: ApiError) -> EncryptionResolutionError {
|
||||
let kind = match error.code {
|
||||
S3ErrorCode::InvalidArgument | S3ErrorCode::InvalidRequest => EncryptionResolutionErrorKind::InvalidRequest,
|
||||
S3ErrorCode::ServiceUnavailable => EncryptionResolutionErrorKind::ServiceUnavailable,
|
||||
_ => EncryptionResolutionErrorKind::DecryptionFailed,
|
||||
};
|
||||
EncryptionResolutionError::new(kind, error.message)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ManagedSealedKey {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
@@ -2631,19 +2751,20 @@ fn ssec_invalid_request(message: &str) -> ApiError {
|
||||
mod tests {
|
||||
use super::{
|
||||
ApiError, DataKey, DecryptionRequest, EncryptionKeyKind, EncryptionMaterial, EncryptionRequest,
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, KmsUnavailableError, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, PrepareEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER, SSEType,
|
||||
SseDekProvider, SsecParams, StorageError, TestSseDekProvider, apply_managed_decryption_material,
|
||||
apply_managed_encryption_material, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse,
|
||||
kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, normalize_managed_metadata,
|
||||
reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption, sse_prepare_encryption,
|
||||
strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
|
||||
validate_ssec_params, verify_ssec_key_match,
|
||||
EncryptionResolutionErrorKind, INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, KmsUnavailableError,
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
|
||||
ObjectEncryptionResolver, PrepareEncryptionRequest, ReadEncryptionMode, ReadEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER,
|
||||
SSEType, SseDekProvider, SseObjectEncryptionResolver, SsecParams, StorageError, TestSseDekProvider,
|
||||
apply_managed_decryption_material, apply_managed_encryption_material, encryption_material_to_metadata,
|
||||
extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
|
||||
generate_ssec_nonce, is_managed_sse, kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata,
|
||||
normalize_managed_metadata, reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption,
|
||||
sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write,
|
||||
validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match,
|
||||
};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use super::{
|
||||
@@ -2703,6 +2824,97 @@ mod tests {
|
||||
SSE_TEST_LOCK.get_or_init(|| Mutex::new(())).lock().await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_encryption_resolver_returns_ssec_read_material() {
|
||||
let key = [0x31; 32];
|
||||
let key_b64 = BASE64_STANDARD.encode(key);
|
||||
let key_md5 = BASE64_STANDARD.encode(md5::compute(key).0);
|
||||
let nonce = [0x42; 12];
|
||||
let metadata = HashMap::from([
|
||||
("X-Amz-Server-Side-Encryption-Customer-Algorithm".to_string(), "AES256".to_string()),
|
||||
("X-Amz-Server-Side-Encryption-Customer-Key-Md5".to_string(), key_md5.clone()),
|
||||
("X-Rustfs-Encryption-Iv".to_string(), BASE64_STANDARD.encode(nonce)),
|
||||
]);
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
|
||||
headers.insert(
|
||||
"x-amz-server-side-encryption-customer-key",
|
||||
HeaderValue::from_str(&key_b64).expect("base64 key is a valid header"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-amz-server-side-encryption-customer-key-md5",
|
||||
HeaderValue::from_str(&key_md5).expect("base64 MD5 is a valid header"),
|
||||
);
|
||||
|
||||
let material = SseObjectEncryptionResolver
|
||||
.resolve_read_material(ReadEncryptionRequest {
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
metadata: &metadata,
|
||||
headers: &headers,
|
||||
})
|
||||
.await
|
||||
.expect("SSE-C material should resolve")
|
||||
.expect("SSE-C metadata should produce material");
|
||||
|
||||
assert_eq!(material.key_bytes, key);
|
||||
assert_eq!(material.mode, ReadEncryptionMode::Direct { base_nonce: nonce });
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_encryption_resolver_classifies_missing_ssec_key_as_invalid_request() {
|
||||
let metadata = HashMap::from([("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string())]);
|
||||
let result = SseObjectEncryptionResolver
|
||||
.resolve_read_material(ReadEncryptionRequest {
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
metadata: &metadata,
|
||||
headers: &HeaderMap::new(),
|
||||
})
|
||||
.await;
|
||||
let error = match result {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("missing SSE-C key must fail closed"),
|
||||
};
|
||||
|
||||
assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidRequest);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_encryption_resolver_rejects_conflicting_metadata_case_variants() {
|
||||
let metadata = HashMap::from([
|
||||
("x-rustfs-encryption-key".to_string(), "first".to_string()),
|
||||
("X-Rustfs-Encryption-Key".to_string(), "second".to_string()),
|
||||
]);
|
||||
let result = SseObjectEncryptionResolver
|
||||
.resolve_read_material(ReadEncryptionRequest {
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
metadata: &metadata,
|
||||
headers: &HeaderMap::new(),
|
||||
})
|
||||
.await;
|
||||
let error = match result {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("conflicting metadata aliases must fail closed"),
|
||||
};
|
||||
|
||||
assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidMetadata);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_encryption_metadata_case_accepts_lowercase_minio_internal_keys() {
|
||||
let lowercase_key = MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_ascii_lowercase();
|
||||
let metadata = HashMap::from([(lowercase_key, "sealed-key".to_string())]);
|
||||
|
||||
let normalized = super::normalize_encryption_metadata_case(&metadata).expect("metadata aliases should normalize");
|
||||
|
||||
assert_eq!(
|
||||
normalized.get(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER),
|
||||
Some(&"sealed-key".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
struct UnavailableSseDekProvider;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -3725,6 +3937,19 @@ mod tests {
|
||||
|
||||
assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object);
|
||||
assert_eq!(decrypted.key_bytes, material.key_bytes);
|
||||
|
||||
let resolved = SseObjectEncryptionResolver
|
||||
.resolve_read_material(ReadEncryptionRequest {
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
metadata: &metadata,
|
||||
headers: &HeaderMap::new(),
|
||||
})
|
||||
.await
|
||||
.expect("managed resolver")
|
||||
.expect("managed material");
|
||||
assert_eq!(resolved.mode, ReadEncryptionMode::Object);
|
||||
assert_eq!(resolved.key_bytes, material.key_bytes);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
@@ -3785,6 +4010,29 @@ mod tests {
|
||||
|
||||
assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object);
|
||||
assert_eq!(decrypted.key_bytes, material.key_bytes);
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
|
||||
headers.insert(
|
||||
"x-amz-server-side-encryption-customer-key",
|
||||
HeaderValue::from_str(&customer_key).expect("customer key header"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-amz-server-side-encryption-customer-key-md5",
|
||||
HeaderValue::from_str(&customer_key_md5).expect("customer key MD5 header"),
|
||||
);
|
||||
let resolved = SseObjectEncryptionResolver
|
||||
.resolve_read_material(ReadEncryptionRequest {
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
metadata: &metadata,
|
||||
headers: &headers,
|
||||
})
|
||||
.await
|
||||
.expect("SSE-C resolver")
|
||||
.expect("SSE-C material");
|
||||
assert_eq!(resolved.mode, ReadEncryptionMode::Object);
|
||||
assert_eq!(resolved.key_bytes, material.key_bytes);
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
@@ -4715,6 +4963,15 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_map_get_object_reader_error_preserves_typed_service_unavailable() {
|
||||
let resolution_error =
|
||||
super::EncryptionResolutionError::new(EncryptionResolutionErrorKind::ServiceUnavailable, "KMS unavailable");
|
||||
let err = map_get_object_reader_error(StorageError::other(resolution_error));
|
||||
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(err.message, "KMS unavailable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_map_get_object_reader_error_leaves_non_ssec_errors_unchanged() {
|
||||
let err = map_get_object_reader_error(StorageError::other("plain io failure"));
|
||||
|
||||
@@ -472,7 +472,7 @@ pub(crate) mod ecstore_metrics {
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod ecstore_notification {
|
||||
pub(crate) use rustfs_ecstore::api::notification::{
|
||||
NotificationSys, get_global_notification_sys, new_global_notification_sys, start_remote_version_state_fleet_probe,
|
||||
NotificationSys, get_global_notification_sys, new_global_notification_sys,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -510,12 +510,21 @@ pub(crate) mod ecstore_object {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::object::GetObjectBodySource;
|
||||
pub(crate) use rustfs_ecstore::api::object::{
|
||||
GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, ObjectMutationHook, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup,
|
||||
ObjectEncryptionResolver, ObjectMutationHook, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "rio-v2"))]
|
||||
pub(crate) mod ecstore_test_support {
|
||||
pub(crate) use rustfs_ecstore::api::bitrot::create_bitrot_reader;
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, endpoint::Endpoint, new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::erasure::Erasure;
|
||||
pub(crate) use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_set_disk {
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::{DEFAULT_READ_BUFFER_SIZE, get_lock_acquire_timeout, is_valid_storage_class};
|
||||
}
|
||||
@@ -945,13 +954,21 @@ pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Res
|
||||
/// The process-level bootstrap instance context that single-instance startup
|
||||
/// threads through the storage foundation (Phase 5 follow-up, backlog#1052).
|
||||
pub(crate) fn bootstrap_instance_ctx() -> Arc<InstanceContext> {
|
||||
ecstore_runtime::bootstrap_ctx()
|
||||
let context = ecstore_runtime::bootstrap_ctx();
|
||||
configure_object_encryption_resolver(&context);
|
||||
context
|
||||
}
|
||||
|
||||
/// Construct a fresh per-server instance context (backlog#1052 S5): a second
|
||||
/// embedded server owns its own erasure/region/endpoint/deployment id cells.
|
||||
pub(crate) fn new_instance_ctx() -> Arc<InstanceContext> {
|
||||
Arc::new(InstanceContext::new())
|
||||
let context = Arc::new(InstanceContext::new());
|
||||
configure_object_encryption_resolver(&context);
|
||||
context
|
||||
}
|
||||
|
||||
fn configure_object_encryption_resolver(context: &InstanceContext) {
|
||||
let _ = context.set_object_encryption_resolver(Arc::new(super::sse::SseObjectEncryptionResolver));
|
||||
}
|
||||
|
||||
pub(crate) fn init_lock_clients(endpoint_pools: EndpointServerPools) {
|
||||
@@ -962,10 +979,6 @@ pub(crate) async fn new_global_notification_sys(endpoint_pools: EndpointServerPo
|
||||
ecstore_notification::new_global_notification_sys(endpoint_pools).await
|
||||
}
|
||||
|
||||
pub(crate) fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
ecstore_notification::start_remote_version_state_fleet_probe(topology_fingerprint);
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config(api: Arc<ECStore>, file: &str) -> Result<Vec<u8>> {
|
||||
ecstore_config::com::read_config(api, file).await
|
||||
}
|
||||
@@ -1717,7 +1730,7 @@ pub(crate) async fn init_compression_total_memory_from_backend(store: Arc<ECStor
|
||||
mod tests {
|
||||
use super::{
|
||||
apply_active_resync_intents, bucket_targets_metadata_lock_shard, ecstore_bucket, lock_bucket_targets_metadata,
|
||||
scanner_maintenance_config_file,
|
||||
new_instance_ctx, scanner_maintenance_config_file,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -1748,6 +1761,16 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fresh_instance_context_installs_object_encryption_resolver() {
|
||||
assert!(new_instance_ctx().object_encryption_resolver().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_instance_context_installs_object_encryption_resolver() {
|
||||
assert!(super::bootstrap_instance_ctx().object_encryption_resolver().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_maintenance_config_only_includes_scanner_owned_work() {
|
||||
assert!(scanner_maintenance_config_file(ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG));
|
||||
|
||||
@@ -317,7 +317,7 @@ write_blackbox_matrix() {
|
||||
printf 'quick\theal degraded erasure disk rebuild\tblack-box\tcargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture\tnone\t%s\n' "$e2e_status"
|
||||
printf 'quick\tnamespace lock quorum under EC ops\tblack-box\tcargo test --package e2e_test namespace_lock_quorum_test -- --nocapture\tnone\t%s\n' "$e2e_status"
|
||||
printf 'full\tlegacy bitrot read fixture restore\tfixture\tcargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture\tRUSTFS_LEGACY_TEST_ROOT,RUSTFS_LEGACY_TEST_DISK\t%s\n' "$legacy_status"
|
||||
printf 'full\tMinIO generated encrypted read and negative restore fixture\tfixture\tcargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture\tRUSTFS_MINIO_FIXTURE_ROOT,RUSTFS_MINIO_STATIC_KMS_KEY_B64\t%s\n' "$minio_status"
|
||||
printf 'full\tMinIO generated encrypted read and negative restore fixture\tfixture\tcargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture\tRUSTFS_MINIO_FIXTURE_ROOT,RUSTFS_MINIO_STATIC_KMS_KEY_B64\t%s\n' "$minio_status"
|
||||
printf 'full\tS3 multipart range versioning delete subset\tblack-box\tenv TESTEXPR=\"multipart or range or versioning or delete\" DEPLOY_MODE=build MAXFAIL=0 ./scripts/s3-tests/run.sh\tnone\t%s\n' "$s3_status"
|
||||
printf 'destructive\tdistributed cluster concurrency\tblack-box\tcargo test --package e2e_test cluster_concurrency_test -- --nocapture\tnone\t%s\n' "$destructive_status"
|
||||
printf 'destructive\tstale multipart cleanup cluster\tblack-box\tcargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture\tnone\t%s\n' "$destructive_status"
|
||||
@@ -377,7 +377,7 @@ run_fixture_steps() {
|
||||
|
||||
if fixture_available; then
|
||||
run_step "ecstore-minio-generated-read-fixture" \
|
||||
cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture
|
||||
cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture
|
||||
elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then
|
||||
echo "ERROR: $(minio_fixture_missing_reason)" >&2
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user