Compare commits

..

6 Commits

213 changed files with 2797 additions and 9459 deletions
Generated
+390 -443
View File
File diff suppressed because it is too large Load Diff
+33 -33
View File
@@ -151,16 +151,16 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
# the s3s-facing edge of the rustfs binary.
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.42.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.2.0", default-features = false }
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.42.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.2.0", default-features = false }
# The gateway framework itself, consumed for its extension traits by the ring-2
# adapter. `default-features = false` keeps the `server` feature, and with it the
# server crate, out of the build.
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.66.0", default-features = false }
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.66.0", default-features = false }
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", default-features = false }
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.63.0", default-features = false }
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.13.0", default-features = false }
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", default-features = false }
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.63.0", default-features = false }
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.13.0", default-features = false }
# Async Runtime and Networking
async-channel = "2.5.0"
@@ -168,13 +168,13 @@ async_zip = { default-features = false, version = "0.0.19" }
mysql_async = { default-features = false, version = "0.37.1" }
async-compression = { version = "0.4.52" }
async-trait = "0.1.92"
async-nats = { version = "0.53.0", default-features = false }
async-nats = { version = "0.52.0", default-features = false }
axum = "0.8.9"
futures = "0.3.34"
futures-core = "0.3.34"
futures-lite = "2.6.1"
futures-util = "0.3.34"
pollster = "1.0.2"
pollster = "1.0.1"
pulsar = { default-features = false, version = "=6.9.0" }
lapin = { default-features = false, version = "4.12.3" }
hyper = { version = "1.12.0" }
@@ -209,8 +209,8 @@ quinn = { version = "0.11.12", default-features = false, features = [
] }
# Serialization and Data Formats
apache-avro = { version = "0.23.0", features = ["snappy", "zstandard"] }
bytes = { version = "1.12.2" }
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
bytes = { version = "1.12.1" }
csv-core = "0.1.13"
bytesize = "2.7.0"
byteorder = "1.5.0"
@@ -242,7 +242,7 @@ jsonwebtoken = { version = "12.0.0" }
openidconnect = { default-features = false, version = "4.1.0" }
pbkdf2 = "0.13.0"
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
rsa = { version = "0.10.0", features = ["pkcs1v15"] }
rsa = { version = "=0.10.0-rc.18" }
rustls = { default-features = false, version = "0.23.46" }
rustls-native-certs = "0.8"
rustls-pki-types = "1.16.0"
@@ -250,13 +250,13 @@ x509-parser = "0.18.1"
sha1 = "0.11.0"
sha2 = "0.11.1"
subtle = "2.6"
zeroize = { version = "1.9.2" }
zeroize = { version = "1.9.1" }
proptest = "1"
# Time and Date
chrono = { version = "0.4.45" }
humantime = "2.4.0"
jiff = { version = "1.0.1" }
jiff = { version = "0.2.43" }
time = { version = "0.3.55" }
# Database
@@ -273,18 +273,18 @@ tar-codec = "0.0.16"
tar-framing = "0.0.16"
atoi = "3.1.1"
atomic_enum = "0.3.0"
aws-config = { version = "1.13.3" }
aws-config = { version = "1.13.2" }
aws-credential-types = { version = "1.3.1" }
aws-sdk-kms = { default-features = false, version = "1.133.0" }
aws-sdk-s3 = { default-features = false, version = "1.160.0" }
aws-sdk-sts = { default-features = false, version = "1.127.0" }
aws-sdk-kms = { default-features = false, version = "1.131.0" }
aws-sdk-s3 = { default-features = false, version = "1.159.0" }
aws-sdk-sts = { default-features = false, version = "1.126.0" }
aws-smithy-async = { version = "1.3.1" }
aws-smithy-http-client = { default-features = false, version = "1.5.3" }
aws-smithy-runtime-api = { version = "1.22.0" }
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
aws-smithy-runtime-api = { version = "1.21.0" }
aws-smithy-types = { version = "1.9.4" }
base64-simd = "0.8.0"
brotli = "9.1.3"
clap = { version = "4.6.8" }
brotli = "9.1.1"
clap = { version = "4.6.7" }
const-str = { version = "1.1.0" }
convert_case = "0.12.0"
criterion = { version = "0.8" }
@@ -292,7 +292,7 @@ crossbeam-queue = "0.3.14"
crossbeam-channel = "0.5.17"
crossbeam-deque = "0.8.8"
crossbeam-utils = "0.8.23"
datafusion = { default-features = false, version = "55.3.0" }
datafusion = { default-features = false, version = "55.2.0" }
derive_builder = "0.20.2"
ed25519-dalek = "3.0.0"
enumset = "1.1.15"
@@ -318,7 +318,7 @@ log = "0.4"
memmap2 = "0.9.12"
nixl-sys = { git = "https://github.com/houseme/nixl", rev = "cc636c83f13d692e85b67e55aed092413d16fb41" }
lz4 = "1.28.1"
lz4_flex = { version = "0.14.2", default-features = false, features = ["std", "frame"] }
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
matchit = "0.9.2"
md-5 = "0.11.0"
mime_guess = "2.0.5"
@@ -337,14 +337,14 @@ percent-encoding = "2.3.2"
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
pin-project-lite = "0.2.17"
pretty_assertions = "1.4.1"
rand = { version = "0.10.4" }
rand = { version = "0.10.3" }
ratelimit = "2.0.0"
rayon = "1.12.0"
rustfs-erasure-codec = { version = "9.1.0" }
reed-solomon-simd = "3.1.0"
regex = { version = "1.13.2" }
regex = { version = "1.13.1" }
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
redis = { version = "1.8.2" }
redis = { version = "1.8.1" }
rustify = { version = "0.7", default-features = false }
rustix = { version = "1.1.6" }
rust-embed = { version = "8.14.0" }
@@ -364,7 +364,7 @@ snap = "1.1.2"
starshard = { version = "2.4.1" }
strum = { version = "0.28.0" }
sync_wrapper = "1.0.2"
sysinfo = "0.40.4"
sysinfo = "0.40.3"
temp-env = "0.3.6"
tempfile = "3.27.0"
test-case = "3.4.1"
@@ -378,15 +378,15 @@ tracing-subscriber = { version = "0.3.23" }
transform-stream = "0.3.1"
url = "2.5.8"
urlencoding = "2.1.3"
uuid = { version = "1.29.3" }
uuid = { version = "1.29.2" }
vaultrs = { version = "0.8.0" }
tar = "0.4.46"
walkdir = "2.5.0"
winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.21" }
zip = "9.7.0"
xxhash-rust = { version = "0.8.19" }
zip = "9.5.1"
zstd = "0.14.4"
# Observability and Metrics
@@ -405,9 +405,9 @@ pyroscope = { version = "2.1.5" }
# FTP and SFTP
libunftp = { version = "0.25.0" }
unftp-core = "0.1.1"
suppaftp = { version = "13.0.0" }
suppaftp = { version = "12.2.1" }
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.65.2" }
russh = { version = "0.65.1" }
russh-sftp = "3.0.1"
# WebDAV
@@ -416,7 +416,7 @@ dav-server = "0.12.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.7.4" }
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
hotpath = { version = "0.32.1", default-features = false }
hotpath = { version = "0.31.2", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.49" }
+1 -1
View File
@@ -36,7 +36,7 @@ jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
base64-simd = { workspace = true }
pbkdf2 = { workspace = true, optional = true }
rand = { workspace = true, features = ["serde"] }
rsa = { workspace = true }
rsa = { workspace = true, features = ["sha2"] }
serde = { workspace = true, features = ["derive"] }
sha2 = { workspace = true, optional = true }
thiserror.workspace = true
+16 -16
View File
@@ -13,9 +13,9 @@
// limitations under the License.
use rsa::{
Pkcs1v15Encrypt, PrivateKey, PublicKey,
Pkcs1v15Encrypt, RsaPrivateKey, RsaPublicKey,
pkcs8::{DecodePrivateKey, DecodePublicKey},
pss::{Signature, SigningKey, VerifyingKey},
pss::{BlindedSigningKey, Signature, VerifyingKey},
sha2::Sha256,
signature::{RandomizedSigner, Verifier},
traits::PublicKeyParts,
@@ -37,7 +37,7 @@ pub struct Token {
pub fn gencode(token: &Token, key: &str) -> Result<String> {
let data = serde_json::to_vec(token)?;
let mut rng = rand::rng();
let public_key = PublicKey::from_public_key_pem(key).map_err(Error::other)?;
let public_key = RsaPublicKey::from_public_key_pem(key).map_err(Error::other)?;
let encrypted_data = public_key.encrypt(&mut rng, Pkcs1v15Encrypt, &data).map_err(Error::other)?;
Ok(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&encrypted_data))
}
@@ -51,7 +51,7 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
let encrypted_data = base64_simd::URL_SAFE_NO_PAD
.decode_to_vec(token.as_bytes())
.map_err(Error::other)?;
let private_key = PrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
let private_key = RsaPrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
let decrypted_data = private_key.decrypt(Pkcs1v15Encrypt, &encrypted_data).map_err(Error::other)?;
serde_json::from_slice(&decrypted_data).map_err(Error::other)
}
@@ -63,8 +63,8 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
pub fn sign_license_token(token: &Token, private_key_pem: &str) -> Result<String> {
let payload = serde_json::to_vec(token)?;
let mut rng = rand::rng();
let private_key = PrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
let signing_key = SigningKey::<Sha256>::new(private_key);
let private_key = RsaPrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
let signing_key = BlindedSigningKey::<Sha256>::new(private_key);
let signature: Signature = signing_key.try_sign_with_rng(&mut rng, &payload).map_err(Error::other)?;
let signature: Box<[u8]> = signature.into();
@@ -80,7 +80,7 @@ pub fn parse_signed_license_token(token: &str, public_key_pem: &str) -> Result<T
let signed_payload = base64_simd::URL_SAFE_NO_PAD
.decode_to_vec(token.as_bytes())
.map_err(Error::other)?;
let public_key = PublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
let public_key = RsaPublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
let signature_len = public_key.size();
if signed_payload.len() <= signature_len {
@@ -103,7 +103,7 @@ pub fn parse_license_with_public_key(license: &str, public_key: &str) -> Result<
mod tests {
use super::*;
use rsa::{
PrivateKey,
RsaPrivateKey,
pkcs8::{EncodePrivateKey, EncodePublicKey, LineEnding},
};
use std::time::{SystemTime, UNIX_EPOCH};
@@ -112,8 +112,8 @@ mod tests {
fn test_sign_license_token_and_parse_signed_license_token() {
let mut rng = rand::rng();
let bits = 2048;
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
let public_key = PublicKey::from(&private_key);
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
let public_key = RsaPublicKey::from(&private_key);
let private_key_pem = private_key
.to_pkcs8_pem(LineEnding::LF)
@@ -144,8 +144,8 @@ mod tests {
fn test_legacy_gencode_and_parse_roundtrip() {
let mut rng = rand::rng();
let bits = 2048;
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
let public_key = PublicKey::from(&private_key);
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
let public_key = RsaPublicKey::from(&private_key);
let private_key_pem = private_key
.to_pkcs8_pem(LineEnding::LF)
@@ -173,8 +173,8 @@ mod tests {
#[test]
fn test_parse_signed_license_token_rejects_tampered_payload() {
let mut rng = rand::rng();
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
let public_key = PublicKey::from(&private_key);
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
let public_key = RsaPublicKey::from(&private_key);
let private_key_pem = private_key
.to_pkcs8_pem(LineEnding::LF)
.expect("failed to encode private key pem");
@@ -206,8 +206,8 @@ mod tests {
#[test]
fn test_parse_signed_license_token_rejects_invalid_token() {
let mut rng = rand::rng();
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
let public_key = PublicKey::from(&private_key);
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
let public_key = RsaPublicKey::from(&private_key);
let public_key_pem = public_key
.to_public_key_pem(LineEnding::LF)
.expect("failed to encode public key pem");
+2 -2
View File
@@ -98,6 +98,7 @@ rustfs-madmin.workspace = true
rustfs-filemeta.workspace = true
bytes = { workspace = true, features = ["serde"] }
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
aws-smithy-types.workspace = true
async-trait = { workspace = true }
@@ -123,7 +124,7 @@ urlencoding.workspace = true
walkdir.workspace = true
base64-simd = { workspace = true }
rand = { workspace = true, features = ["serde"] }
jiff = { workspace = true, features = ["serde"] }
chrono = { workspace = true, features = ["serde"] }
hex-simd = { workspace = true }
md-5 = { workspace = true }
prost.workspace = true
@@ -141,7 +142,6 @@ clap = { workspace = true, features = ["derive", "env"] }
[dev-dependencies]
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
aws-config = { workspace = true }
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
opentelemetry-proto = { workspace = true }
russh = { workspace = true, features = ["serde"] }
russh-sftp = { workspace = true }
+1 -16
View File
@@ -170,15 +170,6 @@ impl DiskFaultHarness {
/// Start (or restart after [`Self::kill_server`]) the RustFS server with
/// all volume directories and the same address.
pub async fn start_server(&mut self) -> ChaosResult<()> {
self.start_server_with_readiness(false).await
}
/// Start a process once its node read-readiness probe passes.
pub async fn start_server_at_read_quorum(&mut self) -> ChaosResult<()> {
self.start_server_with_readiness(true).await
}
async fn start_server_with_readiness(&mut self, read_quorum_only: bool) -> ChaosResult<()> {
// The common helper always appends `env.temp_dir` as the final storage
// path: point it at the last disk for startup and pass the remaining
// disks explicitly, then restore it to the root so Drop cleans up
@@ -190,13 +181,7 @@ impl DiskFaultHarness {
let arg_refs: Vec<&str> = disk_args.iter().map(String::as_str).collect();
let env_refs: Vec<(&str, &str)> = self.extra_env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
let result = if read_quorum_only {
self.env
.start_rustfs_server_with_env_at_read_quorum(arg_refs, &env_refs)
.await
} else {
self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await
};
let result = self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await;
self.env.temp_dir = self.root.to_string_lossy().to_string();
result
}
+5 -35
View File
@@ -722,7 +722,7 @@ impl RustFSTestEnvironment {
cleanup_existing: bool,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let binary_path = rustfs_binary_path();
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing, false)
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing)
.await
}
@@ -732,7 +732,6 @@ impl RustFSTestEnvironment {
extra_args: Vec<&str>,
extra_env: &[(&str, &str)],
cleanup_existing: bool,
read_quorum_only: bool,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
if cleanup_existing {
self.cleanup_existing_processes().await?;
@@ -757,7 +756,7 @@ impl RustFSTestEnvironment {
self.process = Some(process);
// Wait for server to be ready
self.wait_for_server_ready_with_probe(read_quorum_only).await?;
self.wait_for_server_ready().await?;
Ok(())
}
@@ -771,7 +770,7 @@ impl RustFSTestEnvironment {
extra_args: Vec<&str>,
extra_env: &[(&str, &str)],
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true, false)
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true)
.await
}
@@ -783,7 +782,7 @@ impl RustFSTestEnvironment {
extra_args: Vec<&str>,
extra_env: &[(&str, &str)],
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false, false)
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
.await
}
@@ -801,18 +800,6 @@ impl RustFSTestEnvironment {
self.start_rustfs_server_inner(extra_args, extra_env, true).await
}
/// Start a degraded read-serving process using the node readiness probe.
/// ListBuckets retains its namespace write-quorum requirement.
pub async fn start_rustfs_server_with_env_at_read_quorum(
&mut self,
extra_args: Vec<&str>,
extra_env: &[(&str, &str)],
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let binary_path = rustfs_binary_path();
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, true, true)
.await
}
/// Start RustFS server without cleaning up other running RustFS processes.
///
/// This is useful for tests that need multiple independent RustFS instances
@@ -841,16 +828,8 @@ impl RustFSTestEnvironment {
/// Fails immediately if the spawned server process has already exited
/// (e.g. a port bind failure) instead of polling out the full timeout.
pub async fn wait_for_server_ready(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
self.wait_for_server_ready_with_probe(false).await
}
async fn wait_for_server_ready_with_probe(
&mut self,
read_quorum_only: bool,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
info!("Waiting for RustFS server to be ready on {}", self.address);
let client = self.create_s3_client();
let health_client = HttpClient::builder().no_proxy().timeout(Duration::from_secs(2)).build()?;
for i in 0..60 {
if let Some(process) = self.process.as_mut()
@@ -859,16 +838,7 @@ impl RustFSTestEnvironment {
return Err(format!("RustFS server process exited before becoming ready: {status}").into());
}
let ready = if read_quorum_only {
health_client
.get(format!("{}/minio/health/ready", self.url))
.send()
.await
.is_ok_and(|response| response.status().is_success())
} else {
TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok()
};
if ready {
if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
info!("✅ RustFS server is ready after {} attempts", i + 1);
return Ok(());
}
+16 -36
View File
@@ -13,8 +13,8 @@
// limitations under the License.
//! Over-the-wire smoke net for the embedded console listener (backlog#1154
//! peri-4). The console is a second HTTP listener for the control-plane and
//! unauthenticated console routes under `/rustfs/console`;
//! peri-4). The console is a second HTTP listener that serves the full S3/admin
//! surface plus the unauthenticated console routes under `/rustfs/console`;
//! before this test its behaviour was only covered by in-process router tests
//! (`rustfs/src/admin/console.rs`), so a misconfiguration that exposed
//! credentials through the public console endpoints or fail-opened the
@@ -26,13 +26,13 @@
//! * `/rustfs/console/license` exposes only the coarse `licensed` flag.
//! * the console SPA prefix dispatches to the static handler, never to the
//! S3 API (no S3 error XML), whether or not console assets are embedded.
//! * unauthenticated admin API requests are denied with 403 AccessDenied.
//! * S3 root/object requests on the console listener return an empty 404,
//! including authenticated requests, before S3 dispatch.
//! * unauthenticated requests to the admin API and the S3 root on the console
//! listener are denied (403 AccessDenied) — the extra listener does not
//! fail-open the protected surface.
//! * a listener with the console disabled (the main S3 port here) does not
//! serve the unauthenticated console endpoints at all.
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, signed_request_with_client};
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
use std::error::Error;
use tokio::time::{Duration, sleep};
@@ -166,38 +166,18 @@ async fn test_console_over_the_wire_smoke() -> TestResult {
let admin_body = admin_response.text().await?;
assert!(admin_body.contains("AccessDenied"), "admin denial must be AccessDenied: {admin_body}");
let signed_data_root = signed_request_with_client(
&client,
http::Method::GET,
&format!("{}/", env.url),
&env.access_key,
&env.secret_key,
None,
None,
)
.await?;
// The console listener serves no S3 surface at all: an object route is not found there,
// signed or not, instead of reaching the S3 service and being denied by it.
let s3_root_response = client.get(format!("{console_base}/")).send().await?;
assert_eq!(
signed_data_root.status(),
reqwest::StatusCode::OK,
"the same credentials must reach S3 on the data listener"
s3_root_response.status(),
reqwest::StatusCode::NOT_FOUND,
"the S3 root must not be served on the console listener"
);
assert!(
s3_root_response.text().await?.is_empty(),
"the console listener must not answer an object route with an S3 document"
);
for path in ["/", "/console-object-bucket/key"] {
let url = format!("{console_base}{path}");
for signed in [false, true] {
let response = if signed {
signed_request_with_client(&client, http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?
} else {
client.get(&url).send().await?
};
assert_eq!(
response.status(),
reqwest::StatusCode::NOT_FOUND,
"S3 route {path}, signed={signed} must be isolated from the console listener"
);
assert!(response.bytes().await?.is_empty(), "isolated S3 routes must not reach a handler");
}
}
// --- console disabled on a listener means no console surface at all ------
// The main S3 listener of this same process runs with the console disabled,
@@ -26,7 +26,7 @@ use aws_sdk_s3::types::{
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
ObjectLockRule,
};
use jiff::{SignedDuration, Timestamp};
use chrono::{Duration as ChronoDuration, Utc};
fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
@@ -66,7 +66,7 @@ async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
.send()
.await?;
let retain_until = Timestamp::now() + SignedDuration::from_hours(24);
let retain_until = Utc::now() + ChronoDuration::days(1);
let compliance_key = "compliance.bin";
let compliance_version = put_object_with_retention(
@@ -176,10 +176,7 @@ async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_conf
let retain_until = retention
.retain_until_date()
.ok_or("default retention omitted retain-until date")?;
assert!(
retain_until.secs() > Timestamp::now().as_second(),
"default retention is not in the future"
);
assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
@@ -38,8 +38,8 @@ use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
VersioningConfiguration,
};
use chrono::{DateTime, Utc};
use http::{Method, StatusCode};
use jiff::Timestamp;
use serde_json::Value;
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
@@ -526,8 +526,8 @@ async fn put_multipart(client: &Client, bucket: &str, key: &str) -> TestResult<V
Ok(body)
}
fn retention_deadline() -> TestResult<Timestamp> {
Ok(Timestamp::now() + jiff::SignedDuration::from_hours(24))
fn retention_deadline() -> TestResult<DateTime<Utc>> {
DateTime::<Utc>::from_timestamp(Utc::now().timestamp() + 24 * 3600, 0).ok_or_else(|| "retention deadline out of range".into())
}
/// Seed the deterministic inventory into the seed pool and record it.
@@ -654,7 +654,7 @@ pub(crate) async fn seed_manifest(run: &TopologyRun) -> TestResult<TopologyManif
let mut entry = ManifestEntry::body(&locked, &key, "compliance", Some(version_id.clone()), Some(version_id), &body);
entry.retention = Some(RetentionRecord {
mode: "COMPLIANCE".to_string(),
retain_until_unix: retain_until.as_second(),
retain_until_unix: retain_until.timestamp(),
});
entries.push(entry);
}
+1 -2
View File
@@ -29,7 +29,6 @@ use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ServerSideEncryption;
use base64_simd::STANDARD as BASE64;
use http::header::{CONTENT_TYPE, HOST};
use jiff::Timestamp;
use md5::{Digest as Md5Digest, Md5};
use rustfs_s3_types::Body;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
@@ -387,7 +386,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
"usage": "EncryptDecrypt",
"status": "Active",
"metadata": HashMap::<String, String>::new(),
"created_at": format!("{}[UTC]", Timestamp::now().strftime("%Y-%m-%dT%H:%M:%S%.fZ")),
"created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
"rotated_at": serde_json::Value::Null,
"created_by": "e2e-test",
"encrypted_key_material": BASE64.encode_to_string(key_data),
+3 -3
View File
@@ -23,13 +23,13 @@ use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
};
use chrono::{Duration as ChronoDuration, Utc};
use flate2::{
Compression,
write::{GzEncoder, ZlibEncoder},
};
use http::HeaderValue;
use http::header::{CONTENT_TYPE, HOST};
use jiff::{SignedDuration, Timestamp};
use md5::{Digest as Md5Digest, Md5};
use rustfs_s3_types::Body;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
@@ -42,8 +42,8 @@ use tokio::io::AsyncWriteExt;
use uuid::Uuid;
fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
let expiration = (Timestamp::now() + SignedDuration::from_hours(1))
.strftime("%Y-%m-%dT%H:%M:%S.000Z")
let expiration = (Utc::now() + ChronoDuration::hours(1))
.format("%Y-%m-%dT%H:%M:%S.000Z")
.to_string();
let policy = serde_json::json!({
"expiration": expiration,
@@ -283,7 +283,6 @@ async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> Te
#[tokio::test]
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
crate::common::init_logging();
assert_single_node_cold_start_with_missing_volume().await?;
for (node_count, parity) in [(3, 1), (4, 2), (6, 3), (6, 2), (4, 0)] {
let read_quorum = node_count - parity;
@@ -431,138 +430,6 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul
Ok(())
}
async fn assert_single_node_cold_start_with_missing_volume() -> TestResult {
for drive_count in [2, 3, 4] {
let mut server = crate::chaos::DiskFaultHarness::new(drive_count).await?;
server.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{}", drive_count / 2));
server.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
server.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
server.set_env("RUSTFS_HEALTH_ENDPOINT_ENABLE", "true");
server.set_env("RUST_LOG", "warn");
server.start_server_at_read_quorum().await?;
let client = server.env.create_s3_client();
let client = Client::from_conf(
client
.config()
.to_builder()
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
.build(),
);
let bucket = format!("missing-volume-{drive_count}");
client.create_bucket().bucket(&bucket).send().await?;
let small = vec![17; 4096];
let large = vec![73; 8 * 1024 * 1024];
for (key, body) in [("small", &small), ("large", &large)] {
client
.put_object()
.bucket(&bucket)
.key(key)
.body(Bytes::copy_from_slice(body).into())
.send()
.await?;
for disk in 0..drive_count {
let census = crate::chaos::census_object_version_on_disk(server.disk_path(disk), &bucket, key, None)?;
assert!(census.is_complete(), "seed shard must exist before restart: {census:?}");
}
}
server.kill_server();
let missing = drive_count - 1;
server.take_disk_offline(missing)?;
server.start_server_at_read_quorum().await?;
for (key, body) in [("small", &small), ("large", &large)] {
assert_quorum_object_body(&client, &bucket, key, body).await?;
let head = client.head_object().bucket(&bucket).key(key).send().await?;
assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
}
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
let keys: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
assert_eq!(keys, ["large", "small"]);
let put = client
.put_object()
.bucket(&bucket)
.key("degraded-write")
.body(Bytes::from_static(b"degraded write").into())
.send()
.await;
if drive_count == 2 {
let error = put.expect_err("one of two drives cannot commit a new object");
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
let error = client
.delete_object()
.bucket(&bucket)
.key("large")
.send()
.await
.expect_err("one of two drives cannot commit an object deletion");
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
assert_quorum_object_body(&client, &bucket, "large", &large).await?;
} else {
put?;
assert_quorum_object_body(&client, &bucket, "degraded-write", b"degraded write").await?;
client.delete_object().bucket(&bucket).key("degraded-write").send().await?;
}
assert!(
!server.disk_path(missing).exists(),
"degraded startup and S3 operations must leave the root absent"
);
if drive_count == 2 {
server.kill_server();
std::fs::create_dir(server.disk_path(missing))?;
server.start_server_at_read_quorum().await?;
for (key, body) in [("small", &small), ("large", &large)] {
assert_quorum_object_body(&client, &bucket, key, body).await?;
client.head_object().bucket(&bucket).key(key).send().await?;
}
assert_eq!(client.list_objects_v2().bucket(&bucket).send().await?.contents().len(), 2);
let error = client
.put_object()
.bucket(&bucket)
.key("empty-root-write")
.body(Bytes::from_static(b"must remain unavailable").into())
.send()
.await
.expect_err("an empty replacement cannot restore write quorum");
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
assert!(
!server
.disk_path(missing)
.join(crate::storage_api::RUSTFS_META_BUCKET)
.join("format.json")
.exists(),
"read admission must not format the empty replacement"
);
}
server.kill_server();
server.bring_disk_online(missing)?;
server.start_server_at_read_quorum().await?;
for (key, body) in [("small", &small), ("large", &large)] {
assert_quorum_object_body(&client, &bucket, key, body).await?;
}
for key in ["degraded-write", "empty-root-write"] {
let error = client
.head_object()
.bucket(&bucket)
.key(key)
.send()
.await
.expect_err("rejected or deleted degraded write must stay absent after recovery");
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(404));
}
client
.put_object()
.bucket(&bucket)
.key("recovery-write")
.body(Bytes::from_static(b"recovered write").into())
.send()
.await?;
assert_quorum_object_body(&client, &bucket, "recovery-write", b"recovered write").await?;
}
Ok(())
}
#[tokio::test]
async fn test_cold_start_read_quorum_follows_erasure_layout() -> TestResult {
crate::common::init_logging();
@@ -864,6 +864,70 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
Ok(())
}
#[tokio::test]
async fn test_webhook_subscriptions_follow_bucket_incarnation() -> TestResult {
init_logging();
let (endpoint, mut rx, collector) = spawn_event_collector().await?;
let allowed_origin = endpoint_origin(&endpoint)?;
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_OUTBOUND_ALLOW_ORIGINS, allowed_origin.as_str())])
.await?;
enable_notify_module(&env).await?;
let target = "incarnation";
configure_webhook_target(&env, target, &endpoint).await?;
wait_for_target_registered(&env, target).await?;
let client = env.create_s3_client();
let bucket = "notify-incarnation";
let control_bucket = "notify-incarnation-control";
for name in [bucket, control_bucket] {
client.create_bucket().bucket(name).send().await?;
put_notification_config(&client, name, target, "events/", ".txt").await?;
}
let put = |bucket, key| {
client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"event fixture"))
.send()
};
let wait = Duration::from_secs(20);
let original_key = "events/original.txt";
put(bucket, original_key).await?;
wait_for_event(&mut rx, original_key, "s3:ObjectCreated:", wait).await?;
assert!(client.delete_bucket().bucket(bucket).send().await.is_err());
let after_failed_delete = "events/retained.txt";
put(bucket, after_failed_delete).await?;
wait_for_event(&mut rx, after_failed_delete, "s3:ObjectCreated:", wait).await?;
for key in [original_key, after_failed_delete] {
client.delete_object().bucket(bucket).key(key).send().await?;
}
client.delete_bucket().bucket(bucket).send().await?;
client.create_bucket().bucket(bucket).send().await?;
let config = client.get_bucket_notification_configuration().bucket(bucket).send().await?;
assert!(config.queue_configurations().is_empty());
let new_key = "events/fresh.txt";
let sentinel_key = "events/sentinel.txt";
put(bucket, new_key).await?;
put(control_bucket, sentinel_key).await?;
let events = collect_until(&mut rx, sentinel_key, wait, Duration::from_secs(2)).await;
assert!(events.iter().any(|event| event_key(event).as_deref() == Some(sentinel_key)));
assert!(events.iter().all(|event| event_key(event).as_deref() != Some(new_key)));
put_notification_config(&client, bucket, target, "events/", ".txt").await?;
let subscribed_key = "events/subscribed.txt";
put(bucket, subscribed_key).await?;
wait_for_event(&mut rx, subscribed_key, "s3:ObjectCreated:", wait).await?;
env.stop_server();
collector.abort();
Ok(())
}
/// An event queued while the target endpoint rejects delivery survives on the
/// durable store and is redelivered once the endpoint comes back.
#[tokio::test]
+7 -7
View File
@@ -29,7 +29,7 @@ use aws_sdk_s3::types::{
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockMode,
ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule,
};
use jiff::{SignedDuration, Timestamp};
use chrono::{DateTime, Duration, Utc};
use tracing::info;
use crate::common::RustFSTestEnvironment;
@@ -126,10 +126,10 @@ pub async fn put_object_with_retention(
key: &str,
data: &[u8],
mode: ObjectLockRetentionMode,
retain_until: Timestamp,
retain_until: DateTime<Utc>,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
// AWS SDK requires UTC time without timezone offset (e.g., "2026-01-24T11:20:14Z")
let retain_until_str = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
let retain_until_str = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
let retain_until_datetime =
aws_sdk_s3::primitives::DateTime::from_str(&retain_until_str, aws_sdk_s3::primitives::DateTimeFormat::DateTime)?;
@@ -181,10 +181,10 @@ pub async fn put_object_retention(
key: &str,
version_id: Option<&str>,
mode: ObjectLockRetentionMode,
retain_until: Timestamp,
retain_until: DateTime<Utc>,
bypass_governance: bool,
) -> Result<(), Box<SdkError<PutObjectRetentionError>>> {
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.as_second());
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.timestamp());
let retention = ObjectLockRetention::builder()
.mode(mode.clone())
@@ -253,6 +253,6 @@ pub async fn delete_object_with_bypass(
}
/// Calculate a future retain_until date
pub fn future_retain_until(days: i64) -> Timestamp {
Timestamp::now() + SignedDuration::from_hours(days * 24)
pub fn future_retain_until(days: i64) -> DateTime<Utc> {
Utc::now() + Duration::days(days)
}
@@ -34,7 +34,7 @@ use aws_sdk_s3::types::{
CompletedMultipartUpload, CompletedPart, Delete, MetadataDirective, ObjectIdentifier, ObjectLockLegalHoldStatus,
ObjectLockMode, ObjectLockRetentionMode,
};
use jiff::{SignedDuration, Timestamp};
use chrono::{DateTime, Duration, Utc};
use tracing::info;
/// Initialize test logging
@@ -67,7 +67,7 @@ async fn put_bucket_deny_policy(
}
fn retention_timestamp(days: i64) -> aws_sdk_s3::primitives::DateTime {
let retain_until = future_retain_until(days).strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
let retain_until = future_retain_until(days).format("%Y-%m-%dT%H:%M:%SZ").to_string();
aws_sdk_s3::primitives::DateTime::from_str(&retain_until, aws_sdk_s3::primitives::DateTimeFormat::DateTime)
.expect("retention timestamp should parse")
}
@@ -118,14 +118,13 @@ where
);
}
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> Timestamp {
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> DateTime<Utc> {
let formatted = value
.fmt(DateTimeFormat::DateTime)
.expect("S3 timestamp should format as RFC3339");
let normalized = formatted
.strip_suffix('Z')
.map_or(formatted.clone(), |prefix| format!("{prefix}+00:00"));
Timestamp::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", normalized).expect("S3 timestamp should parse as RFC3339")
DateTime::parse_from_rfc3339(&formatted)
.expect("S3 timestamp should parse as RFC3339")
.with_timezone(&Utc)
}
// ============================================================================
@@ -471,7 +470,7 @@ async fn test_get_object_retention_returns_configured_values() {
let bucket = "test-get-retention";
let key = "retained-object";
let retain_until = future_retain_until(30);
let retain_until_expected = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
let retain_until_expected = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
env.create_object_lock_bucket(bucket).await.unwrap();
@@ -1925,7 +1924,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
.await
.unwrap();
let copy_started = Timestamp::now();
let copy_started = Utc::now();
client
.copy_object()
.copy_source(format!("{src_bucket}/{src_key}"))
@@ -1948,7 +1947,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
assert!(
retain_until < copy_started + SignedDuration::from_hours(3 * 24),
retain_until < copy_started + Duration::days(3),
"destination default retention should not inherit the source's longer retention"
);
@@ -2009,7 +2008,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
.expect("explicit retain-until date should be present"),
);
assert!(
explicit_retain_until > Timestamp::now() + SignedDuration::from_hours(20 * 24),
explicit_retain_until > Utc::now() + Duration::days(20),
"explicit retention should override the shorter bucket default"
);
@@ -2089,7 +2088,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
.await
.unwrap();
let create_started = Timestamp::now();
let create_started = Utc::now();
let create_output = client.create_multipart_upload().bucket(bucket).key(key).send().await.unwrap();
let upload_id = create_output.upload_id().unwrap();
@@ -2132,7 +2131,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
assert!(
retain_until < create_started + SignedDuration::from_hours(3 * 24),
retain_until < create_started + Duration::days(3),
"CompleteMultipartUpload should keep the default retention calculated at create time"
);
}
@@ -573,15 +573,6 @@ impl NodeService for MinimalLockNodeService {
Err(Status::unimplemented("lock-only test server"))
}
type RenameDataBatchStream = ResponseStream<rustfs_protos::proto_gen::node_service::RenameDataBatchResult>;
async fn rename_data_batch(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataBatchRequest>,
) -> Result<Response<Self::RenameDataBatchStream>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn rename_multipart_data(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
+1
View File
@@ -154,6 +154,7 @@ arc-swap.workspace = true
async-trait.workspace = true
bytes = { workspace = true, features = ["serde"] }
byteorder = { workspace = true }
chrono = { workspace = true, features = ["serde"] }
jiff = { workspace = true, features = ["serde"] }
thiserror.workspace = true
flatbuffers.workspace = true
@@ -90,9 +90,7 @@ fn remote_credentials(credentials: &Credentials, account_id: &str) -> RemoteCred
access_key: credentials.access_key.clone(),
secret_key: credentials.secret_key.clone(),
session_token: credentials.effective_session_token().map(str::to_string),
expiration: credentials
.effective_expiration()
.and_then(|timestamp| SystemTime::try_from(timestamp).ok()),
expiration: credentials.effective_expiration().map(SystemTime::from),
account_id: account_id.to_string(),
}
}
@@ -51,15 +51,20 @@ impl LcAuditEvent {
}
}
pub(crate) fn emit_transition_failed_event(object: ObjectInfo) {
emit_lifecycle_event(EventName::ObjectTransitionFailed, object, LIFECYCLE_TRANSITION_USER_AGENT);
pub(crate) fn emit_transition_failed_event(object: ObjectInfo, incarnation_id: Option<uuid::Uuid>) {
emit_lifecycle_event(EventName::ObjectTransitionFailed, object, LIFECYCLE_TRANSITION_USER_AGENT, incarnation_id);
}
pub(crate) fn emit_transition_complete_event(object: ObjectInfo) {
emit_lifecycle_event(EventName::ObjectTransitionComplete, object, LIFECYCLE_TRANSITION_USER_AGENT);
pub(crate) fn emit_transition_complete_event(object: ObjectInfo, incarnation_id: Option<uuid::Uuid>) {
emit_lifecycle_event(
EventName::ObjectTransitionComplete,
object,
LIFECYCLE_TRANSITION_USER_AGENT,
incarnation_id,
);
}
pub(crate) fn emit_transitioned_expiration_event(source: &ObjectInfo, deleted: &ObjectInfo) {
pub(crate) fn emit_transitioned_expiration_event(source: &ObjectInfo, deleted: &ObjectInfo, incarnation_id: uuid::Uuid) {
let event_name = transitioned_expiration_event_name(source.delete_marker, deleted.delete_marker);
let object = ObjectInfo {
bucket: source.bucket.clone(),
@@ -69,12 +74,17 @@ pub(crate) fn emit_transitioned_expiration_event(source: &ObjectInfo, deleted: &
delete_marker: source.delete_marker,
..Default::default()
};
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT);
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT, Some(incarnation_id));
}
pub(crate) fn emit_non_transitioned_expiration_event(action: IlmAction, source: &ObjectInfo, deleted: ObjectInfo) {
pub(crate) fn emit_non_transitioned_expiration_event(
action: IlmAction,
source: &ObjectInfo,
deleted: ObjectInfo,
incarnation_id: uuid::Uuid,
) {
let event_name = non_transitioned_expiration_event_name(action, source.delete_marker, deleted.delete_marker);
emit_lifecycle_event(event_name, deleted, LIFECYCLE_EXPIRY_USER_AGENT);
emit_lifecycle_event(event_name, deleted, LIFECYCLE_EXPIRY_USER_AGENT, Some(incarnation_id));
}
/// Emit the lifecycle expiration event for one version removed by the batch
@@ -83,9 +93,15 @@ pub(crate) fn emit_non_transitioned_expiration_event(action: IlmAction, source:
/// That path never sent events, so a successful noncurrent-version expiry was
/// invisible to notification subscribers even though the equivalent
/// current-version path emits one (backlog#2202).
pub(crate) fn emit_noncurrent_expiration_event(bucket: &str, target: &ObjectToDelete, deleted: &DeletedObject, failed: bool) {
pub(crate) fn emit_noncurrent_expiration_event(
bucket: &str,
target: &ObjectToDelete,
deleted: &DeletedObject,
failed: bool,
incarnation_id: uuid::Uuid,
) {
if let Some((event_name, object)) = noncurrent_expiration_event(bucket, target, deleted, failed) {
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT);
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT, Some(incarnation_id));
}
}
@@ -131,9 +147,10 @@ fn noncurrent_expiration_event(
Some((event_name, object))
}
fn emit_lifecycle_event(event_name: EventName, object: ObjectInfo, user_agent: &str) {
fn emit_lifecycle_event(event_name: EventName, object: ObjectInfo, user_agent: &str, incarnation_id: Option<uuid::Uuid>) {
send_event(EventArgs {
event_name: event_name.to_string(),
bucket_incarnation_id: incarnation_id,
bucket_name: object.bucket.clone(),
object,
user_agent: user_agent.to_string(),
@@ -2166,9 +2166,10 @@ impl TransitionState {
..Default::default()
};
if let Err(err) =
let (incarnation_id, transition_result) =
transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone()))
.await
.await;
if let Err(err) = transition_result
{
if let (Some(job_id), Some(result_key)) = (task.manual_job_id, task.manual_result_key.as_deref()) {
record_manual_transition_worker_result_for_task_with_reason(
@@ -2195,7 +2196,7 @@ impl TransitionState {
"Lifecycle tier operation failed"
);
}
emit_transition_failed_event(obj_info_for_event);
emit_transition_failed_event(obj_info_for_event, incarnation_id);
} else {
if let (Some(job_id), Some(result_key)) = (task.manual_job_id, task.manual_result_key.as_deref()) {
record_manual_transition_worker_result_for_task(
@@ -2217,7 +2218,7 @@ impl TransitionState {
}
transition_state.add_lastday_stats(&task.event.storage_class, ts);
emit_transition_complete_event(obj_info_for_event);
emit_transition_complete_event(obj_info_for_event, incarnation_id);
}
// Release the dedup claim so a later lifecycle pass can
// re-transition this object if needed (rustfs/backlog#1268).
@@ -5121,7 +5122,7 @@ async fn expire_transitioned_object_with_lock_lost_signal(
}
};
schedule_lifecycle_replication_delete_if_needed(oi, &dobj).await;
schedule_lifecycle_replication_delete_if_needed(oi, &dobj, bucket_incarnation_id).await;
// The transitioned version is gone; evict any cached body for this object
// so it does not linger until TTL (ODC-26).
@@ -5134,7 +5135,7 @@ async fn expire_transitioned_object_with_lock_lost_signal(
//audit_log_lifecycle(oi, ILMExpiry, tags);
emit_transitioned_expiration_event(oi, &dobj);
emit_transitioned_expiration_event(oi, &dobj, bucket_incarnation_id);
Ok(dobj)
}
@@ -5152,12 +5153,12 @@ pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
Ok(obj)
}
pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> {
async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> (Option<Uuid>, Result<(), Error>) {
let time_ilm = Metrics::time_ilm(lae.event.action);
let (publication_guard, bucket_incarnation_id) = lifecycle_transition_publication_guard(&api, oi)
.await
.ok_or_else(|| Error::other("lifecycle transition is not allowed for this bucket"))?;
let Some((publication_guard, bucket_incarnation_id)) = lifecycle_transition_publication_guard(&api, oi).await else {
return (None, Err(Error::other("lifecycle transition is not allowed for this bucket")));
};
let etag = if let Some(etag) = &oi.etag { etag } else { "" };
let etag = etag.to_string();
@@ -5180,7 +5181,7 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
opts.add_namespace_lock_guard(&publication_guard);
let result = api.transition_object(&oi.bucket, &oi.name, &opts).await;
time_ilm(1)();
result
(Some(bucket_incarnation_id), result)
}
#[allow(
@@ -6034,7 +6035,7 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
{
accounting.commit(size).await;
}
schedule_lifecycle_replication_delete_if_needed(oi, &dobj).await;
schedule_lifecycle_replication_delete_if_needed(oi, &dobj, bucket_incarnation_id).await;
// The object (or all its versions, for delete_all) was expired; evict any
// cached body so dead bytes do not sit resident until TTL (ODC-26). The
@@ -6050,7 +6051,7 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
//let tags = LcAuditEvent::new(lc_event.clone(), src.clone()).tags();
//tags["version-id"] = dobj.version_id;
emit_non_transitioned_expiration_event(lc_event.action, oi, dobj);
emit_non_transitioned_expiration_event(lc_event.action, oi, dobj, bucket_incarnation_id);
if lc_event.action != IlmAction::NoneAction {
let mut num_versions = 1_u64;
@@ -6202,12 +6203,12 @@ fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> DeletedObject
}
}
async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo, dobj: &ObjectInfo) {
async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo, dobj: &ObjectInfo, bucket_incarnation_id: Uuid) {
let delete_object = lifecycle_deleted_object(oi, dobj);
if delete_object.replication_state.is_none() {
return;
}
replication_sink::schedule_delete(oi.bucket.clone(), delete_object).await;
replication_sink::schedule_delete(oi.bucket.clone(), delete_object, Some(bucket_incarnation_id)).await;
}
async fn lifecycle_delete_config_snapshot(api: &ECStore, oi: &ObjectInfo) -> Result<DeleteReplicationConfigSnapshot, Error> {
@@ -128,7 +128,7 @@ pub async fn delete_object_versions(
// order and admission; the event is derived from the committed
// result, and a send failure never rolls back a delete that
// already happened (backlog#2202).
emit_noncurrent_expiration_event(bucket, target, deleted_obj, false);
emit_noncurrent_expiration_event(bucket, target, deleted_obj, false, bucket_incarnation_id);
}
if !deleted_obj
.replication_state
@@ -137,7 +137,7 @@ pub async fn delete_object_versions(
{
continue;
}
replication_sink::schedule_delete(bucket.to_string(), deleted_obj.clone()).await;
replication_sink::schedule_delete(bucket.to_string(), deleted_obj.clone(), Some(bucket_incarnation_id)).await;
}
for (i, err) in errors.iter().enumerate() {
@@ -66,8 +66,12 @@ pub(crate) fn lifecycle_action_waits_for_replication(action: IlmAction) -> bool
)
}
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
ReplicationLifecycleBridge::schedule_delete(bucket, delete_object).await;
pub(crate) async fn schedule_delete(
bucket: String,
delete_object: DeletedObject,
notification_bucket_incarnation: Option<uuid::Uuid>,
) {
ReplicationLifecycleBridge::schedule_delete(bucket, delete_object, notification_bucket_incarnation).await;
}
#[cfg(test)]
+94 -62
View File
@@ -460,7 +460,7 @@ fn clear_bucket_overrides(bucket: &str) {
}
/// Publish application-owned bytes on every cache install path.
fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
fn sync_application_bucket_configs(bucket: &str, bm: &BucketMetadata) {
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
hook(
bucket,
@@ -468,12 +468,22 @@ fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
bm.on_demand_migration_config()
.map(|(bytes, stamp)| (bytes, stamp, bm.bucket_incarnation_id)),
);
hook(
bucket,
super::metadata::BUCKET_NOTIFICATION_CONFIG,
(!bm.notification_config_xml.is_empty()).then_some((
bm.notification_config_xml.as_slice(),
bm.notification_config_updated_at,
bm.bucket_incarnation_id,
)),
);
}
}
fn clear_on_demand_migration(bucket: &str) {
fn clear_application_bucket_configs(bucket: &str) {
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
hook(bucket, super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, None);
hook(bucket, super::metadata::BUCKET_NOTIFICATION_CONFIG, None);
}
}
@@ -2253,6 +2263,7 @@ impl BucketMetadataSys {
.await?;
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, persisted, &guard)
.await
.map(|_| ())
}
#[cfg(test)]
@@ -2486,7 +2497,7 @@ impl BucketMetadataSys {
if removed {
BucketTargetSys::get().delete(bucket).await;
clear_bucket_overrides(bucket);
clear_on_demand_migration(bucket);
clear_application_bucket_configs(bucket);
}
}
return Ok(());
@@ -2524,7 +2535,7 @@ impl BucketMetadataSys {
self.missing_buckets.invalidate(bucket).await;
sync_bucket_target_sys(bucket, &bm).await;
sync_bucket_overrides(bucket, &bm);
sync_on_demand_migration(bucket, &bm);
sync_application_bucket_configs(bucket, &bm);
}
MetadataLoadMode::Initial => {
let _publish_guard = self
@@ -2550,7 +2561,7 @@ impl BucketMetadataSys {
metadata: BucketMetadata,
persisted: bool,
namespace_guard: &rustfs_lock::NamespaceLockGuard,
) -> Result<()> {
) -> Result<bool> {
self.revoke_pending_bucket_creation(bucket).await;
if !persisted {
let _publish_guard = self
@@ -2563,7 +2574,7 @@ impl BucketMetadataSys {
_ => false,
};
if !unchanged {
return Ok(());
return Ok(false);
}
let removed = map.remove(bucket).is_some();
drop(map);
@@ -2572,9 +2583,9 @@ impl BucketMetadataSys {
if removed {
BucketTargetSys::get().delete(bucket).await;
clear_bucket_overrides(bucket);
clear_on_demand_migration(bucket);
clear_application_bucket_configs(bucket);
}
return Ok(());
return Ok(true);
}
let _publish_guard = self
.lock_metadata_publish(bucket, namespace_guard, "refreshed bucket metadata publish")
@@ -2587,7 +2598,7 @@ impl BucketMetadataSys {
_ => false,
};
if !unchanged {
return Ok(());
return Ok(false);
}
map.insert(bucket.to_string(), Arc::clone(&metadata));
drop(map);
@@ -2595,8 +2606,8 @@ impl BucketMetadataSys {
self.missing_buckets.invalidate(bucket).await;
sync_bucket_target_sys(bucket, &metadata).await;
sync_bucket_overrides(bucket, &metadata);
sync_on_demand_migration(bucket, &metadata);
Ok(())
sync_application_bucket_configs(bucket, &metadata);
Ok(true)
}
pub async fn get(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
@@ -2625,7 +2636,7 @@ impl BucketMetadataSys {
self.missing_buckets.invalidate(&bucket).await;
sync_bucket_target_sys(&bucket, &bm).await;
sync_bucket_overrides(&bucket, &bm);
sync_on_demand_migration(&bucket, &bm);
sync_application_bucket_configs(&bucket, &bm);
}
}
@@ -2649,7 +2660,7 @@ impl BucketMetadataSys {
if removed {
BucketTargetSys::get().delete(bucket).await;
clear_bucket_overrides(bucket);
clear_on_demand_migration(bucket);
clear_application_bucket_configs(bucket);
}
removed || removed_fabricated
}
@@ -2876,25 +2887,35 @@ impl BucketMetadataSys {
namespace_guard: &rustfs_lock::NamespaceLockGuard,
) -> Result<()> {
self.revoke_pending_bucket_creation(bucket).await;
let expected = self.metadata_map.read().await.get(bucket).cloned();
if !self
.bucket_exists(bucket, namespace_guard, "peer bucket metadata existence check")
.await?
{
return Err(Error::other(PEER_METADATA_NOT_PERSISTED));
}
let (metadata, persisted) = await_bucket_namespace_operation(
Some(namespace_guard),
bucket,
"peer bucket metadata load",
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
)
.await?;
if !persisted {
return Err(Error::other(PEER_METADATA_NOT_PERSISTED));
// A concurrent publication can reject the snapshot we read. Re-read
// from storage so a newer committed config does not lose its reload to
// an older one that happened to publish first.
for _ in 0..3 {
let expected = self.metadata_map.read().await.get(bucket).cloned();
let (metadata, persisted) = await_bucket_namespace_operation(
Some(namespace_guard),
bucket,
"peer bucket metadata load",
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
)
.await?;
if !persisted {
return Err(Error::other(PEER_METADATA_NOT_PERSISTED));
}
if self
.publish_if_unchanged(bucket, expected.as_ref(), metadata, true, namespace_guard)
.await?
{
return Ok(());
}
}
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, true, namespace_guard)
.await
Err(Error::other("bucket metadata changed repeatedly during peer reload"))
}
pub async fn get_config(&self, bucket: &str) -> Result<(Arc<BucketMetadata>, bool)> {
@@ -2992,7 +3013,7 @@ impl BucketMetadataSys {
self.missing_buckets.invalidate(bucket).await;
sync_bucket_target_sys(bucket, &bm).await;
sync_bucket_overrides(bucket, &bm);
sync_on_demand_migration(bucket, &bm);
sync_application_bucket_configs(bucket, &bm);
return Ok((bm, true));
} else {
let exists = self
@@ -3443,7 +3464,7 @@ impl BucketMetadataSys {
self.missing_buckets.invalidate(bucket).await;
sync_bucket_target_sys(bucket, &metadata).await;
sync_bucket_overrides(bucket, &metadata);
sync_on_demand_migration(bucket, &metadata);
sync_application_bucket_configs(bucket, &metadata);
Ok(BucketMetadataAuthority::Authoritative(metadata))
}
@@ -6792,31 +6813,33 @@ mod tests {
const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
type RecordedOdmConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
type RecordedOdmHookCall = (String, RecordedOdmConfig);
const NOTIFY_XML: &[u8] = br#"<NotificationConfiguration><QueueConfiguration><Id>test</Id><Queue>arn:rustfs:sqs:us-east-1:primary:webhook</Queue><Event>s3:ObjectCreated:*</Event></QueueConfiguration></NotificationConfiguration>"#;
const PUBLISH_TEST_BUCKET: &str = "application-config-hook-paths";
/// Every `(bucket, config)` the recording hook has seen. Tests filter by
/// their own bucket name; the hook is process-wide and set once.
static ODM_HOOK_CALLS: std::sync::Mutex<Vec<RecordedOdmHookCall>> = std::sync::Mutex::new(Vec::new());
type RecordedConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
type RecordedConfigCall = (String, RecordedConfig);
fn install_recording_odm_hook() {
static CONFIG_HOOK_CALLS: std::sync::Mutex<Vec<RecordedConfigCall>> = std::sync::Mutex::new(Vec::new());
fn install_recording_config_hook() {
BUCKET_CONFIG_PUBLISH_HOOK.get_or_init(|| {
Box::new(|bucket, config_file, config| {
assert_eq!(config_file, super::super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG);
ODM_HOOK_CALLS.lock().unwrap().push((
bucket.to_string(),
config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
));
if bucket == PUBLISH_TEST_BUCKET {
CONFIG_HOOK_CALLS.lock().unwrap().push((
config_file.to_string(),
config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
));
}
})
});
}
fn odm_hook_calls(bucket: &str) -> Vec<RecordedOdmConfig> {
ODM_HOOK_CALLS
fn config_hook_calls(config_file: &str) -> Vec<RecordedConfig> {
CONFIG_HOOK_CALLS
.lock()
.unwrap()
.iter()
.filter(|(name, _)| name == bucket)
.filter(|(file, _)| file == config_file)
.map(|(_, config)| config.clone())
.collect()
}
@@ -6826,51 +6849,56 @@ mod tests {
/// reload, refresh loop, lazy load) and withdraws on removal, mirroring
/// `sync_bucket_durability`.
#[tokio::test]
async fn on_demand_migration_hook_fires_on_every_cache_install_path() {
install_recording_odm_hook();
async fn application_config_hook_fires_on_every_cache_install_path() {
use crate::bucket::metadata::{BUCKET_NOTIFICATION_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
install_recording_config_hook();
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
let bucket = "odm-hook-paths";
let bucket = PUBLISH_TEST_BUCKET;
for dir in &dirs {
std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
}
let incarnation = Uuid::new_v4();
let expect_publish = |before: usize, label: &str| {
let calls = odm_hook_calls(bucket);
assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
assert_eq!(
calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
Some(ODM_JSON),
"{label} must publish the stored bytes"
);
assert_eq!(calls.last().unwrap().as_ref().map(|(_, _, id)| *id), Some(incarnation));
for (file, bytes) in [
(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON),
(BUCKET_NOTIFICATION_CONFIG, NOTIFY_XML),
] {
let calls = config_hook_calls(file);
assert_eq!(calls.len(), before + 1, "{label} must publish {file} exactly once");
let (stored, _, id) = calls.last().unwrap().as_ref().expect("stored configuration");
assert_eq!(stored, bytes, "{label} must publish the stored config");
assert_eq!(*id, incarnation);
}
};
// set (via persist_new_and_set, which installs through `set`).
let mut bm = BucketMetadata::new(bucket);
bm.bucket_incarnation_id = incarnation;
bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
.unwrap();
bm.update_config(BUCKET_NOTIFICATION_CONFIG, NOTIFY_XML.to_vec()).unwrap();
let writer = BucketMetadataSys::new(ecstore.clone());
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
writer.persist_new_and_set(bm).await.expect("metadata should persist");
expect_publish(before, "set");
// init (initial load on a cold system).
let mut cold = BucketMetadataSys::new(ecstore.clone());
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
cold.init(vec![bucket.to_string()]).await;
assert!(cold.get(bucket).await.is_ok(), "initial load must cache the bucket");
expect_publish(before, "init");
// peer reload.
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
cold.reload_from_store(bucket).await.expect("peer reload should publish");
expect_publish(before, "peer reload");
// refresh loop.
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
let mut failed = HashSet::new();
cold.concurrent_load(&[bucket.to_string()], &mut failed, MetadataLoadMode::Refresh)
.await;
@@ -6879,24 +6907,28 @@ mod tests {
// lazy load on another cold system.
let lazy = BucketMetadataSys::new(ecstore);
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
let (_, loaded) = lazy.get_config(bucket).await.expect("lazy load should publish");
assert!(loaded, "the lazy path must have gone to disk");
expect_publish(before, "lazy load");
// Removal withdraws the config.
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
assert!(lazy.remove(bucket).await);
let calls = odm_hook_calls(bucket);
let calls = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG);
assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
assert_eq!(calls.last().unwrap(), &None);
let notify_calls = config_hook_calls(BUCKET_NOTIFICATION_CONFIG);
assert_eq!(notify_calls.len(), before + 1);
assert_eq!(notify_calls.last(), Some(&None));
// Opaque bytes reach the application even if they are not valid JSON.
let mut corrupt = BucketMetadata::new(bucket);
corrupt.on_demand_migration_config_json = b"not-json".to_vec();
let before = odm_hook_calls(bucket).len();
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
let calls = odm_hook_calls(bucket);
let calls = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG);
assert_eq!(calls.len(), before + 1);
assert_eq!(
calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
@@ -1070,8 +1070,8 @@ mod tests {
.unwrap_or_else(|| panic!("signed request must contain {name}"))
}
fn signing_time(headers: &[(String, String)]) -> jiff::Timestamp {
jiff::Timestamp::strptime("%Y%m%dT%H%M%SZ", recorded_header(headers, "x-amz-date"))
fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
.expect("SDK signing timestamp must use the SigV4 format")
}
@@ -1086,8 +1086,8 @@ mod tests {
.map(|(key, value)| (key.to_string(), value.to_string()))
.collect(),
);
let server_time = jiff::Timestamp::try_from(self.clock.now()).expect("clock timestamp")
+ jiff::SignedDuration::from_secs(self.skew_seconds);
let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
+ chrono::Duration::seconds(self.skew_seconds);
let (status, body) = if headers.len() == 1 {
(
403,
@@ -1098,7 +1098,7 @@ mod tests {
};
let response = http::Response::builder()
.status(status)
.header("date", server_time.strftime("%a, %d %b %Y %H:%M:%S GMT").to_string())
.header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
.header("content-type", "application/xml")
.header("content-length", body.len())
.body(SdkBody::from(body))
@@ -1136,7 +1136,7 @@ mod tests {
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
for skew_seconds in [-600, 600] {
let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
let initial = jiff::Timestamp::try_from(clock.now()).expect("clock timestamp");
let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
client
.get_object()
.bucket("bucket")
@@ -1163,12 +1163,12 @@ mod tests {
assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
assert_eq!(
signing_time(&headers[1]),
initial + jiff::SignedDuration::from_secs(skew_seconds),
initial + chrono::Duration::seconds(skew_seconds),
"{error_code}: retry must apply the measured offset exactly"
);
assert_eq!(
signing_time(&headers[2]),
initial + jiff::SignedDuration::from_secs(skew_seconds + 17),
initial + chrono::Duration::seconds(skew_seconds + 17),
"{error_code}: the next operation must apply cached skew to the advanced signing clock"
);
let signature = |index: usize| {
@@ -13,6 +13,8 @@
// limitations under the License.
pub(crate) use crate::services::event_notification::EventArgs;
#[cfg(test)]
pub(crate) use crate::services::event_notification::test_recorder;
use super::runtime_boundary;
@@ -89,7 +89,11 @@ impl ReplicationLifecycleBridge {
}
}
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
pub(crate) async fn schedule_delete(
bucket: String,
delete_object: DeletedObject,
notification_bucket_incarnation: Option<uuid::Uuid>,
) {
#[cfg(test)]
{
SCHEDULED_DELETE_OBJECTS
@@ -98,6 +102,7 @@ impl ReplicationLifecycleBridge {
.push(delete_object.clone());
}
super::replication_pool::schedule_replication_delete(DeletedObjectReplicationInfo {
notification_bucket_incarnation,
delete_object: deleted_object_for_replication(delete_object),
bucket,
event_type: REPLICATE_INCOMING_DELETE.to_string(),
@@ -22,6 +22,10 @@ use rustfs_utils::path::path_join_buf;
use time::OffsetDateTime;
use super::replication_error_boundary::{Error, Result};
#[cfg(test)]
pub(crate) use crate::bucket::metadata_sys::{
bucket_metadata_sys_of, init_bucket_metadata_sys, set_bucket_metadata_in, test_support,
};
pub(crate) type ReplicationInstanceContext = InstanceContext;
@@ -110,13 +110,16 @@ impl ReplicationObjectBridge {
check_replicate_delete_with_snapshot(object, source, opts, source_error, snapshot)
}
/// Admits replication with the bucket identity verified by the originating mutation.
/// An absent identity suppresses external notifications; workers never infer one by name.
pub async fn schedule_object<S: ReplicationStorage>(
object: ObjectInfo,
storage: Arc<S>,
decision: ReplicateDecision,
op_type: ReplicationType,
notification_bucket_incarnation: Option<uuid::Uuid>,
) {
schedule_replication(object, storage, decision, op_type).await;
schedule_replication(object, storage, decision, op_type, notification_bucket_incarnation).await;
}
pub async fn schedule_delete(delete_object: DeletedObjectReplicationInfo) {
@@ -148,8 +151,16 @@ impl ReplicationObjectBridge {
}
}
pub async fn schedule_storage_delete(delete_object: DeletedObject, bucket: String, event_type: String) {
/// Schedules a committed delete with its originating bucket identity for notifications.
/// An absent identity leaves external notifications unbound.
pub async fn schedule_storage_delete(
delete_object: DeletedObject,
bucket: String,
event_type: String,
notification_bucket_incarnation: Option<uuid::Uuid>,
) {
Self::schedule_delete(DeletedObjectReplicationInfo {
notification_bucket_incarnation,
delete_object: deleted_object_for_replication(delete_object),
bucket,
event_type,
@@ -158,10 +169,18 @@ impl ReplicationObjectBridge {
.await;
}
pub async fn schedule_storage_deletes(delete_objects: Vec<DeletedObject>, bucket: String, event_type: String) {
/// Schedules committed deletes from one bucket instance, retaining that notification identity.
/// An absent identity leaves external notifications unbound.
pub async fn schedule_storage_deletes(
delete_objects: Vec<DeletedObject>,
bucket: String,
event_type: String,
notification_bucket_incarnation: Option<uuid::Uuid>,
) {
let delete_objects = delete_objects
.into_iter()
.map(|delete_object| DeletedObjectReplicationInfo {
notification_bucket_incarnation,
delete_object: deleted_object_for_replication(delete_object),
bucket: bucket.clone(),
event_type: event_type.clone(),
@@ -1016,7 +1016,9 @@ async fn replay_mrf_metadata_entry<S: ReplicationStorage>(
/// admitted targets, carrying over the entry's retry count.
fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_type: ReplicationType) -> ReplicateObjectInfo {
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type);
// The journal does not bind this task to a bucket instance. Keep external
// notifications unbound instead of attaching an old task to a reused name.
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type, None);
roi.retry_count = entry.retry_count.max(0) as u32;
roi
}
@@ -3171,12 +3173,13 @@ pub(crate) async fn schedule_replication<S: ReplicationStorage>(
o: Arc<S>,
dsc: ReplicateDecision,
op_type: ReplicationType,
notification_bucket_incarnation: Option<uuid::Uuid>,
) {
let (synchronous, asynchronous) = dsc.partition_by_sync();
let mut async_oi = oi;
if synchronous.replicate_any() {
let ri = replicate_object_info_from_object_info(async_oi.clone(), synchronous, op_type);
let ri = replicate_object_info_from_object_info(async_oi.clone(), synchronous, op_type, notification_bucket_incarnation);
let state = replicate_object(ri, o.clone()).await;
async_oi.replication_status_internal = state.replication_status_internal;
async_oi.version_purge_status_internal = state.version_purge_status_internal;
@@ -3185,7 +3188,7 @@ pub(crate) async fn schedule_replication<S: ReplicationStorage>(
if asynchronous.replicate_any()
&& let Some(pool) = runtime_sources::replication_pool()
{
let ri = replicate_object_info_from_object_info(async_oi, asynchronous, op_type);
let ri = replicate_object_info_from_object_info(async_oi, asynchronous, op_type, notification_bucket_incarnation);
let _ = pool.queue_replica_task(ri).await;
}
}
@@ -3194,6 +3197,7 @@ fn replicate_object_info_from_object_info(
oi: ObjectInfo,
dsc: ReplicateDecision,
op_type: ReplicationType,
notification_bucket_incarnation: Option<uuid::Uuid>,
) -> ReplicateObjectInfo {
let tgt_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default());
let purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default());
@@ -3209,6 +3213,7 @@ fn replicate_object_info_from_object_info(
let checksum = if ssec { oi.checksum.clone() } else { None };
ReplicateObjectInfo {
notification_bucket_incarnation,
name: oi.name,
size: oi.size,
actual_size: asz,
@@ -3350,7 +3355,7 @@ pub async fn queue_replication_metadata(bucket: &str, oi: ObjectInfo, retry_coun
return ReplicationQueueAdmission::Skipped;
}
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata);
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata, None);
roi.retry_count = retry_count;
if let Some(pool) = runtime_sources::replication_pool() {
pool.queue_replica_task(roi).await
@@ -5205,6 +5210,117 @@ mod tests {
}
}
#[test]
fn replication_admission_keeps_original_incarnation_and_mrf_replay_does_not_guess() {
let incarnation = Uuid::new_v4();
let target = "arn:rustfs:replication:target-a";
for op_type in [ReplicationType::Object, ReplicationType::Metadata] {
let object = ObjectInfo {
bucket: "source".to_string(),
name: "object".to_string(),
version_id: Some(Uuid::new_v4()),
..Default::default()
};
let admitted = replicate_object_info_from_object_info(
object.clone(),
test_replicate_decision(&[target]),
op_type,
Some(incarnation),
);
assert_eq!(admitted.notification_bucket_incarnation, Some(incarnation));
let encoded = encode_mrf_file(&[admitted.to_mrf_entry()]).expect("encode queued replication");
let entries = decode_mrf_file(&encoded).expect("decode queued replication");
let replayed = admitted_mrf_replicate_object(object, &entries[0], op_type);
assert_eq!(
replayed.notification_bucket_incarnation, None,
"MRF replay must not infer a bucket instance from a reused name"
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial_test::serial]
async fn queued_delete_notifications_keep_origin_across_bucket_recreation() {
use super::super::replication_event_sink::test_recorder;
use super::super::replication_metadata_boundary::{
bucket_metadata_sys_of, init_bucket_metadata_sys, set_bucket_metadata_in,
test_support::isolated_store_over_temp_disks,
};
use super::super::replication_storage_boundary::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
runtime_sources::set_test_object_layer(store.clone()).await;
test_recorder::install();
let bucket = "queued-delete-notification-origin";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create original bucket");
let original = store.bucket_incarnation_id(bucket).await.expect("original incarnation");
let queued = DeletedObjectReplicationInfo {
notification_bucket_incarnation: Some(original),
bucket: bucket.to_string(),
target_arn: "arn:rustfs:replication:target-a".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "original-object".to_string(),
version_id: Some(Uuid::new_v4()),
replication_state: Some(ReplicationState {
replicate_decision_str: "malformed-decision".to_string(),
..Default::default()
}),
..Default::default()
},
..Default::default()
};
let encoded = encode_mrf_file(&[queued.to_mrf_entry()]).expect("persist old delete task");
let entries = decode_mrf_file(&encoded).expect("recover old delete task");
let mut replayed = reconstructed_heal_delete_info(
&entries[0],
&ObjectInfo::default(),
&replicate_decision_for_admitted_targets(&entries[0].target_arns),
);
replayed
.delete_object
.replication_state
.as_mut()
.expect("reconstructed replication state")
.replicate_decision_str = "malformed-decision".to_string();
store
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("delete original bucket");
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create replacement bucket");
let replacement = store.bucket_incarnation_id(bucket).await.expect("replacement incarnation");
assert_ne!(original, replacement);
let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
let mut metadata = (*sys.read().await.get(bucket).await.expect("replacement metadata")).clone();
metadata.notification_config_xml = br#"<NotificationConfiguration><QueueConfiguration><Id>replacement</Id><Queue>arn:rustfs:sqs:us-east-1:replacement:webhook</Queue><Event>s3:Replication:*</Event></QueueConfiguration></NotificationConfiguration>"#.to_vec();
set_bucket_metadata_in(&store.ctx, metadata)
.await
.expect("install replacement notification configuration");
assert!(!replicate_delete_with_outcome(queued, store.clone()).await);
assert!(!replicate_delete_with_outcome(replayed, store).await);
let events = test_recorder::recorded_for_bucket(bucket);
assert_eq!(events.len(), 2, "both worker failures must emit an observable local event");
assert!(
events
.iter()
.all(|event| event.object == "original-object" && event.event_name == "s3:Replication:OperationNotTracked")
);
assert_eq!(events[0].bucket_incarnation_id, Some(original));
assert_eq!(
events[1].bucket_incarnation_id, None,
"a replay without provenance must not acquire the replacement bucket's identity"
);
assert!(events.iter().all(|event| event.bucket_incarnation_id != Some(replacement)));
}
#[test]
fn replicate_object_info_from_object_info_preserves_ssec_checksum() {
let checksum = bytes::Bytes::from_static(b"ssec-checksum");
@@ -5219,7 +5335,7 @@ mod tests {
..Default::default()
};
let ri = replicate_object_info_from_object_info(oi, ReplicateDecision::default(), ReplicationType::Object);
let ri = replicate_object_info_from_object_info(oi, ReplicateDecision::default(), ReplicationType::Object, None);
assert!(ri.ssec);
assert_eq!(ri.checksum, Some(checksum));
@@ -5235,8 +5351,12 @@ mod tests {
user_tags: Arc::new("owner=a3".to_string()),
..Default::default()
};
let live =
replicate_object_info_from_object_info(object.clone(), test_replicate_decision(&[target]), ReplicationType::Metadata);
let live = replicate_object_info_from_object_info(
object.clone(),
test_replicate_decision(&[target]),
ReplicationType::Metadata,
None,
);
let persisted = live.to_mrf_entry();
let encoded = encode_mrf_file(std::slice::from_ref(&persisted)).expect("metadata MRF entry should encode");
let decoded = decode_mrf_file(&encoded).expect("metadata MRF entry should decode");
@@ -2275,6 +2275,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
let replication_generation = oi.replication_generation_snapshot();
Ok(ReplicateObjectInfo {
notification_bucket_incarnation: None,
name: oi.name.clone(),
size: oi.size,
actual_size,
@@ -2449,6 +2450,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
"Failed to parse replicate decision"
);
send_local_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -2530,6 +2532,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
"Failed to verify source delete-marker state before replication"
);
send_local_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -2564,6 +2567,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
"Skipping replication delete"
);
send_local_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -2594,6 +2598,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
"Skipping replication delete"
);
send_local_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -2693,6 +2698,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
"Replication resync task failed"
);
send_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -2801,6 +2807,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
{
Ok(object) => {
send_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name,
bucket_name: bucket.clone(),
object,
@@ -2826,6 +2833,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
);
}
send_event(EventArgs {
bucket_incarnation_id: dobj.notification_bucket_incarnation,
event_name,
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3157,6 +3165,7 @@ async fn purge_stale_delete_marker_targets(bucket: &str, dobj: &DeletedObjectRep
}
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) -> bool {
let incarnation_id = dobj.notification_bucket_incarnation;
let bucket = &dobj.bucket;
let object_name = &dobj.delete_object.object_name;
let admitted_target_arns = dobj.admitted_target_arns();
@@ -3177,6 +3186,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Skipping replication force-delete because replication config is missing"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3200,6 +3210,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Skipping replication force-delete because replication config lookup failed"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3234,6 +3245,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Skipping replication force-delete"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3262,6 +3274,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Skipping replication force-delete"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3309,6 +3322,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Skipping replication force-delete because target client is unavailable"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3339,6 +3353,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Skipping replication force-delete"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationFailed.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -3375,6 +3390,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
"Replication target operation failed"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationFailed.to_string(),
bucket_name: bucket.clone(),
object: ObjectInfo {
@@ -4239,6 +4255,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
"Skipping replication object"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: roi.to_object_info(),
@@ -4262,6 +4279,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
"Skipping replication object"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: roi.to_object_info(),
@@ -4328,6 +4346,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
"Replication resync task failed"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: roi.to_object_info(),
@@ -4472,6 +4491,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
if publication.emit_event {
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name,
bucket_name: bucket.clone(),
object: object_info,
@@ -4614,6 +4634,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
"Skipping replication object target"
);
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: self.to_object_info(),
@@ -4629,6 +4650,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await
else {
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: self.to_object_info(),
@@ -4672,6 +4694,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
);
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: self.to_object_info(),
@@ -4703,6 +4726,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
"Skipping replication object target"
);
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: object_info,
@@ -4728,6 +4752,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
"Skipping replication object target"
);
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: object_info,
@@ -4862,6 +4887,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
"Replication target operation failed"
);
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: object_info,
@@ -4890,6 +4916,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
"Replication target operation failed"
);
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: object_info,
@@ -4987,6 +5014,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await
else {
send_local_event(EventArgs {
bucket_incarnation_id: self.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: self.to_object_info(),
@@ -5020,7 +5048,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let size = match object_info.get_actual_size() {
Ok(size) => size,
Err(e) => {
note_replicate_all_size_unavailable(&bucket, &tgt_client, object_info, &e);
note_replicate_all_size_unavailable(&bucket, &tgt_client, object_info, &e, self.notification_bucket_incarnation);
return rinfo;
}
};
@@ -5029,7 +5057,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let transfer_size = if self.ssec { object_info.size } else { size };
if tgt_client.bucket.is_empty() {
note_replicate_all_target_bucket_empty(&bucket, &tgt_client, object_info);
note_replicate_all_target_bucket_empty(&bucket, &tgt_client, object_info, self.notification_bucket_incarnation);
return rinfo;
}
@@ -5075,7 +5103,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
let (put_opts, is_multipart) = match replication_put_object_options_for_target(&tgt_client, &object_info) {
Ok((put_opts, is_mp)) => (put_opts, is_mp),
Err(e) => {
fail_replicate_all_put_options(&mut rinfo, &tgt_client, &bucket, object_info, &e, start_time);
fail_replicate_all_put_options(
&mut rinfo,
&tgt_client,
&bucket,
object_info,
&e,
start_time,
self.notification_bucket_incarnation,
);
return rinfo;
}
};
@@ -5179,6 +5215,7 @@ fn note_replicate_all_target_offline(roi: &ReplicateObjectInfo, bucket: &str, tg
"Skipped replication because target is offline"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: roi.to_object_info(),
@@ -5217,6 +5254,7 @@ fn note_replicate_all_reader_unavailable(roi: &ReplicateObjectInfo, bucket: &str
"Skipped replication because object reader is unavailable"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: roi.to_object_info(),
@@ -5228,7 +5266,13 @@ fn note_replicate_all_reader_unavailable(roi: &ReplicateObjectInfo, bucket: &str
/// Log and notify that replication was skipped because the actual object size
/// is unavailable.
fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient, object_info: ObjectInfo, e: &std::io::Error) {
fn note_replicate_all_size_unavailable(
bucket: &str,
tgt_client: &TargetClient,
object_info: ObjectInfo,
e: &std::io::Error,
incarnation_id: Option<uuid::Uuid>,
) {
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
@@ -5241,6 +5285,7 @@ fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient,
"Skipped replication because actual object size is unavailable"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: object_info,
@@ -5251,7 +5296,12 @@ fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient,
/// Log and notify that replication was skipped because the target bucket is
/// empty.
fn note_replicate_all_target_bucket_empty(bucket: &str, tgt_client: &TargetClient, object_info: ObjectInfo) {
fn note_replicate_all_target_bucket_empty(
bucket: &str,
tgt_client: &TargetClient,
object_info: ObjectInfo,
incarnation_id: Option<uuid::Uuid>,
) {
debug!(
event = EVENT_RESYNC_RUNTIME_SKIPPED,
component = LOG_COMPONENT_ECSTORE,
@@ -5263,6 +5313,7 @@ fn note_replicate_all_target_bucket_empty(bucket: &str, tgt_client: &TargetClien
"Skipped replication because target bucket is empty"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: object_info,
@@ -5376,6 +5427,7 @@ async fn resolve_replicate_all_action(
"Skipping replication because newer target version exists"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: object_info.clone(),
@@ -5499,6 +5551,7 @@ async fn resolve_replicate_all_action(
"Failed replication head-object fallback"
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: object_info,
@@ -5531,6 +5584,7 @@ async fn resolve_replicate_all_action(
);
send_local_event(EventArgs {
bucket_incarnation_id: roi.notification_bucket_incarnation,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: object_info,
@@ -5559,6 +5613,7 @@ fn fail_replicate_all_put_options(
object_info: ObjectInfo,
e: &Error,
start_time: OffsetDateTime,
incarnation_id: Option<uuid::Uuid>,
) {
rinfo.replication_status = ReplicationStatusType::Failed;
rinfo.error = Some(e.to_string());
@@ -5574,6 +5629,7 @@ fn fail_replicate_all_put_options(
"Replication target operation failed"
);
send_local_event(EventArgs {
bucket_incarnation_id: incarnation_id,
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.to_string(),
object: object_info,
@@ -24,6 +24,8 @@ pub(crate) use crate::object_api::{
};
#[cfg(test)]
pub(crate) use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence};
#[cfg(test)]
pub(crate) use crate::storage_api_contracts::bucket::{BucketOperations, DeleteBucketOptions, MakeBucketOptions};
pub(crate) use crate::storage_api_contracts::list::{
ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
};
@@ -18,6 +18,8 @@ use super::replication_bandwidth_boundary::ReplicationBucketMonitor;
use super::replication_pool::DynReplicationPool;
use super::replication_state::ReplicationStats;
use super::replication_storage_boundary::ReplicationObjectStore;
#[cfg(test)]
pub(crate) use crate::runtime::global::set_object_layer as set_test_object_layer;
use crate::runtime::sources;
pub use crate::runtime::sources::{
ScannerDirtyUsageMutationObserver, ScannerDirtyUsageMutationSource, set_scanner_dirty_usage_mutation_observer,
+5 -586
View File
@@ -73,11 +73,10 @@ use rustfs_protos::proto_gen::node_service::{
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataBatchItem, RenameDataBatchRequest,
RenameDataBatchResult, RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest,
VerifyFileRequest, WriteAllRequest, WriteAllResponse, WriteFormatControlRequest, WriteMetadataRequest,
node_service_client::NodeServiceClient,
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteAllResponse,
WriteFormatControlRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
};
use rustfs_protos::{
ChannelClass, ChannelRebuildReason, ConnectionEvictionLogLevel, evict_failed_connection_with_reason, prewarm_channel_class,
@@ -105,7 +104,7 @@ use tokio::{
time::timeout,
};
use tokio_util::sync::CancellationToken;
use tonic::{Code, Request, Streaming, service::interceptor::InterceptedService};
use tonic::{Code, Request, service::interceptor::InterceptedService};
use tracing::{debug, trace, warn};
use uuid::Uuid;
@@ -1046,7 +1045,6 @@ pub struct RemoteDisk {
/// Optional per-peer gate held through complete bulk-unary decoding and validation.
bulk_body_admission: Option<Arc<Semaphore>>,
bulk_body_wait_timeout: Duration,
rename_admission: Option<Arc<Semaphore>>,
}
/// Shortest gap between two failure warnings of one kind from one remote disk.
@@ -1062,251 +1060,6 @@ struct BulkBodyAdmissionRegistry {
static BULK_BODY_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<BulkBodyAdmissionRegistry>> =
std::sync::LazyLock::new(|| StdMutex::new(BulkBodyAdmissionRegistry::default()));
const ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH: &str = "RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH";
const RENAME_ADMISSION_REGISTRY_CAPACITY: usize = 512;
#[derive(Default)]
struct RenameAdmissionRegistry {
peers: HashMap<String, Weak<Semaphore>>,
overflow: Option<Arc<Semaphore>>,
}
static RENAME_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<RenameAdmissionRegistry>> =
std::sync::LazyLock::new(|| StdMutex::new(RenameAdmissionRegistry::default()));
fn peer_rename_admission(addr: &str, enabled: bool) -> Option<Arc<Semaphore>> {
if !enabled {
return None;
}
let mut registry = RENAME_ADMISSION_REGISTRY
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if let Some(semaphore) = registry.peers.get(addr).and_then(Weak::upgrade) {
return Some(semaphore);
}
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
registry.peers.retain(|_, semaphore| semaphore.strong_count() > 0);
}
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
return Some(Arc::clone(registry.overflow.get_or_insert_with(|| {
Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT))
})));
}
let semaphore = Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT));
registry.peers.insert(addr.to_owned(), Arc::downgrade(&semaphore));
Some(semaphore)
}
struct RenameAdmissionGrant {
permit: Option<OwnedSemaphorePermit>,
batch: bool,
}
impl RenameAdmissionGrant {
fn release_one(&mut self) -> bool {
self.permit.as_mut().is_some_and(|permit| permit.split(1).is_some())
}
fn release_all(mut self) {
drop(self.permit.take());
}
fn quarantine_until(mut self, deadline: tokio::time::Instant) {
let Some(permit) = self.permit.take() else {
return;
};
let items = permit.num_permits();
if items == 0 {
return;
}
let hold = deadline.saturating_duration_since(tokio::time::Instant::now());
rustfs_io_metrics::record_ecstore_rename_quarantine(self.batch, items, hold);
let batch = self.batch;
if let Ok(handle) = tokio::runtime::Handle::try_current() {
handle.spawn(async move {
tokio::time::sleep_until(deadline).await;
drop(permit);
rustfs_io_metrics::record_ecstore_rename_quarantine_released(batch, items);
});
} else {
// Without a runtime there is no safe deadline task. Leaking until
// process exit is safer than admitting work over a still-running mutation.
rustfs_io_metrics::record_ecstore_rename_quarantine_no_runtime(batch, items);
std::mem::forget(permit);
}
}
}
struct RenameBatchCallGuard {
admission: Option<RenameAdmissionGrant>,
deadline: tokio::time::Instant,
dispatch_state: RenameAdmissionDispatchState,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RenameAdmissionDispatchState {
NotDispatched,
MayHavePublished,
}
impl RenameBatchCallGuard {
fn mark_dispatched(&mut self) {
self.dispatch_state = RenameAdmissionDispatchState::MayHavePublished;
}
fn complete(mut self) {
debug_assert_eq!(self.dispatch_state, RenameAdmissionDispatchState::MayHavePublished);
if let Some(admission) = self.admission.take() {
admission.release_all();
}
}
fn release_without_quarantine(mut self) {
if let Some(admission) = self.admission.take() {
admission.release_all();
}
}
fn into_stream(mut self, inner: Streaming<RenameDataBatchResult>, expected_mask: u16) -> RemoteRenameBatchStream {
debug_assert_eq!(self.dispatch_state, RenameAdmissionDispatchState::MayHavePublished);
RemoteRenameBatchStream {
inner,
expected_mask,
seen_mask: 0,
admission: self.admission.take(),
deadline: self.deadline,
terminal: false,
stream_started_at: rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now),
first_item_at: None,
}
}
}
impl Drop for RenameBatchCallGuard {
fn drop(&mut self) {
if self.dispatch_state == RenameAdmissionDispatchState::MayHavePublished
&& let Some(admission) = self.admission.take()
{
admission.quarantine_until(self.deadline);
}
}
}
pub(crate) struct RenameBatchClientItem<'a> {
pub slot: usize,
pub disk: String,
pub file_info: &'a FileInfo,
}
pub(crate) enum RenameBatchStartError {
NotDispatched(Error),
UnsupportedBeforeStream,
MayHavePublished(Error),
}
pub(crate) struct RemoteRenameBatchOutcome {
pub slot: usize,
pub result: Result<RenameDataResp>,
}
pub(crate) struct RemoteRenameBatchStream {
inner: Streaming<RenameDataBatchResult>,
expected_mask: u16,
seen_mask: u16,
admission: Option<RenameAdmissionGrant>,
deadline: tokio::time::Instant,
terminal: bool,
stream_started_at: Option<Instant>,
first_item_at: Option<Instant>,
}
impl RemoteRenameBatchStream {
fn terminate_ambiguous(&mut self, error: Error) -> Error {
if !self.terminal {
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
}
self.terminal = true;
if let Some(admission) = self.admission.take() {
admission.quarantine_until(self.deadline);
}
error
}
pub(crate) async fn next_outcome(&mut self) -> Result<Option<RemoteRenameBatchOutcome>> {
if self.terminal {
return Ok(None);
}
let response = match tokio::time::timeout_at(self.deadline, self.inner.message()).await {
Err(_) => return Err(self.terminate_ambiguous(DiskError::Timeout)),
Ok(Err(status)) => return Err(self.terminate_ambiguous(status.into())),
Ok(Ok(None)) if self.seen_mask == self.expected_mask => {
self.terminal = true;
return Ok(None);
}
Ok(Ok(None)) => return Err(self.terminate_ambiguous(DiskError::Unexpected)),
Ok(Ok(Some(response))) => response,
};
let Ok(slot) = usize::try_from(response.item_index) else {
return Err(self.terminate_ambiguous(DiskError::Unexpected));
};
if slot >= 16 {
return Err(self.terminate_ambiguous(DiskError::Unexpected));
}
let bit = 1_u16 << slot;
if self.expected_mask & bit == 0 || self.seen_mask & bit != 0 {
return Err(self.terminate_ambiguous(DiskError::Unexpected));
}
self.seen_mask |= bit;
if self.admission.as_mut().is_some_and(|admission| !admission.release_one()) {
return Err(self.terminate_ambiguous(DiskError::Unexpected));
}
if let Some(stream_started_at) = self.stream_started_at {
let completed_at = Instant::now();
match self.first_item_at {
Some(first_item_at) if self.seen_mask == self.expected_mask => {
rustfs_io_metrics::record_ecstore_rename_batch_client_first_to_last(
completed_at.saturating_duration_since(first_item_at),
);
}
Some(_) => {}
None => {
rustfs_io_metrics::record_ecstore_rename_batch_client_stream_to_first(
completed_at.saturating_duration_since(stream_started_at),
);
self.first_item_at = Some(completed_at);
if self.seen_mask == self.expected_mask {
rustfs_io_metrics::record_ecstore_rename_batch_client_first_to_last(Duration::ZERO);
}
}
}
}
rustfs_io_metrics::record_ecstore_rename_batch_item_completed();
let result = if response.success {
if response.error.is_some() {
Err(DiskError::FileCorrupt)
} else {
decode_msgpack_strict(&response.rename_data_resp_msgpack)
}
} else {
Err(response.error.unwrap_or_default().into())
};
Ok(Some(RemoteRenameBatchOutcome { slot, result }))
}
}
impl Drop for RemoteRenameBatchStream {
fn drop(&mut self) {
if !self.terminal {
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
}
if let Some(admission) = self.admission.take() {
admission.quarantine_until(self.deadline);
}
}
}
struct BulkBodyPermit {
_metrics: rustfs_io_metrics::internode_metrics::InternodeBulkBodyAdmissionGuard,
_permit: OwnedSemaphorePermit,
@@ -1473,10 +1226,6 @@ fn spawn_control_channel_prewarm(addr: String) {
}
impl RemoteDisk {
pub(crate) fn rename_stream_batch_enabled(&self) -> bool {
self.rename_admission.is_some()
}
async fn guarded_delete_versions_rpc(
&self,
volume: &str,
@@ -1802,8 +1551,6 @@ impl RemoteDisk {
rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
let bulk_body_admission = peer_bulk_body_admission(&addr, bulk_body_concurrency_limit()?);
let bulk_body_wait_timeout = bulk_body_wait_timeout();
let rename_admission =
peer_rename_admission(&addr, rustfs_utils::get_env_bool(ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH, false));
let disk = Self {
handle_id: Uuid::new_v4(),
@@ -1826,7 +1573,6 @@ impl RemoteDisk {
network_error_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
bulk_body_admission,
bulk_body_wait_timeout,
rename_admission,
};
record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);
@@ -2562,54 +2308,6 @@ impl RemoteDisk {
}
}
async fn execute_peer_batch_with_timeout<T, F, Fut>(
&self,
op: &'static str,
operation: F,
timeout_duration: Duration,
) -> Result<T>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T>>,
{
if timeout_duration == Duration::ZERO {
self.record_timeout(op, timeout_duration);
return Err(DiskError::Timeout);
}
match time::timeout(timeout_duration, operation()).await {
Ok(operation_result) => {
if operation_result.is_ok() {
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
}
let evict = operation_result
.as_ref()
.err()
.is_some_and(|error| !error.is_metacache_output_stream_closed() && is_network_like_disk_error(error));
self.handle_network_like_error(op, timeout_duration, &operation_result, FailureHealthAction::IgnoreFailure)
.await;
if evict {
evict_failed_connection_with_reason(
&self.addr,
ChannelRebuildReason::RpcFailure,
ConnectionEvictionLogLevel::Debug,
)
.await;
}
operation_result
}
Err(_) => {
self.record_timeout(op, timeout_duration);
evict_failed_connection_with_reason(
&self.addr,
ChannelRebuildReason::OperationTimeout,
ConnectionEvictionLogLevel::Debug,
)
.await;
Err(DiskError::Timeout)
}
}
}
fn record_timeout(&self, op: &'static str, timeout_duration: Duration) {
counter!(
"rustfs_drive_op_timeout_total",
@@ -2796,38 +2494,6 @@ impl RemoteDisk {
}))
}
async fn acquire_rename_admission_until(
&self,
item_count: usize,
deadline: tokio::time::Instant,
) -> Result<Option<RenameAdmissionGrant>> {
let Some(semaphore) = &self.rename_admission else {
return Ok(None);
};
if item_count == 0 || item_count > rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT {
return Err(Error::other("invalid internode rename admission weight"));
}
if let Some(err) = self.offline_bypass_error() {
return Err(err);
}
let weight = u32::try_from(item_count).map_err(Error::other)?;
let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
let (result, timed_out) = match tokio::time::timeout_at(deadline, Arc::clone(semaphore).acquire_many_owned(weight)).await
{
Ok(Ok(permit)) => (Ok(permit), false),
Ok(Err(_)) => (Err(Error::RemoteClientUnavailable("internode rename admission closed".to_owned())), false),
Err(_) => (Err(DiskError::DiskOngoingReq), true),
};
if let Some(started) = started {
rustfs_io_metrics::record_ecstore_rename_admission(item_count > 1, item_count, started.elapsed(), timed_out);
}
let permit = result?;
Ok(Some(RenameAdmissionGrant {
permit: Some(permit),
batch: item_count > 1,
}))
}
async fn get_bulk_client(&self) -> Result<NodeRpcClient> {
if let Some(err) = self.offline_bypass_error() {
return Err(err);
@@ -2968,18 +2634,6 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_
}
}
fn decode_msgpack_strict<T: DeserializeOwned>(binary: &[u8]) -> Result<T> {
if binary.is_empty() {
return Err(DiskError::FileCorrupt);
}
let mut cursor = Cursor::new(binary);
let value = rmp_serde::from_read(&mut cursor).map_err(|_| DiskError::FileCorrupt)?;
if cursor.position() != u64::try_from(binary.len()).map_err(Error::other)? {
return Err(DiskError::FileCorrupt);
}
Ok(value)
}
fn decode_multipart_publication_response(binary: &[u8]) -> Result<RenameDataResp> {
let result = if binary.len() > MultipartSourceOperation::max_wire_payload_bytes() {
Err(DiskError::FileCorrupt)
@@ -3177,97 +2831,6 @@ impl RemoteDisk {
.await
}
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn start_rename_data_batch(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
items: Vec<RenameBatchClientItem<'_>>,
) -> std::result::Result<RemoteRenameBatchStream, RenameBatchStartError> {
if self.rename_admission.is_none() || items.is_empty() || items.len() > 16 {
return Err(RenameBatchStartError::NotDispatched(DiskError::UnsupportedDisk));
}
let deadline = tokio::time::Instant::now() + get_max_timeout_duration();
let mut expected_mask = 0_u16;
let mut request_items = Vec::with_capacity(items.len());
for item in items {
if item.slot >= 16 {
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
}
let bit = 1_u16 << item.slot;
if expected_mask & bit != 0 {
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
}
expected_mask |= bit;
let item_index = u32::try_from(item.slot)
.map_err(Error::other)
.map_err(RenameBatchStartError::NotDispatched)?;
let file_info_msgpack = encode_file_info_msgpack(item.file_info).map_err(RenameBatchStartError::NotDispatched)?;
request_items.push(RenameDataBatchItem {
item_index,
disk: item.disk,
file_info_msgpack: file_info_msgpack.into(),
});
}
let mut request = Request::new(RenameDataBatchRequest {
src_volume: src_volume.to_owned(),
src_path: src_path.to_owned(),
dst_volume: dst_volume.to_owned(),
dst_path: dst_path.to_owned(),
items: request_items,
});
let canonical_body = rustfs_protos::canonical_rename_data_batch_request_body(request.get_ref())
.map_err(Error::other)
.map_err(RenameBatchStartError::NotDispatched)?;
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body)
.map_err(Error::other)
.map_err(RenameBatchStartError::NotDispatched)?;
let mut client = match tokio::time::timeout_at(deadline, self.get_client()).await {
Ok(result) => result.map_err(RenameBatchStartError::NotDispatched)?,
Err(_) => return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout)),
};
let admission = self
.acquire_rename_admission_until(request.get_ref().items.len(), deadline)
.await
.map_err(RenameBatchStartError::NotDispatched)?;
let guard = RenameBatchCallGuard {
admission,
deadline,
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
};
let call_timeout = deadline.saturating_duration_since(tokio::time::Instant::now());
if call_timeout == Duration::ZERO {
guard.release_without_quarantine();
return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout));
}
let mut guard = guard;
guard.mark_dispatched();
let response = match self
.execute_peer_batch_with_timeout(
"rename_data_batch",
|| async {
match client.rename_data_batch(request).await {
Ok(response) => Ok(response),
Err(status) if status.code() == Code::Unimplemented => Err(DiskError::UnsupportedDisk),
Err(status) => Err(status.into()),
}
},
call_timeout,
)
.await
{
Err(DiskError::UnsupportedDisk) => {
guard.release_without_quarantine();
return Err(RenameBatchStartError::UnsupportedBeforeStream);
}
Err(error) => return Err(RenameBatchStartError::MayHavePublished(error)),
Ok(response) => response,
};
Ok(guard.into_stream(response.into_inner(), expected_mask))
}
#[tracing::instrument(level = "trace", skip_all)]
pub(crate) async fn rename_data_borrowed_with_fence(
&self,
@@ -3339,27 +2902,6 @@ impl RemoteDisk {
let scanner_publication_lease = scanner_publication_lease_token
.map(|token| Bytes::copy_from_slice(token.as_bytes()))
.unwrap_or_default();
let rename_deadline = tokio::time::Instant::now() + get_max_timeout_duration();
let rename_admission = self.acquire_rename_admission_until(1, rename_deadline).await?;
let mut rename_admission_guard = RenameBatchCallGuard {
admission: rename_admission,
deadline: rename_deadline,
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
};
let rpc_kind = if multipart_rename {
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_MULTIPART
} else if pool_meta_cas {
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_POOL_META_CAS
} else if quota_fenced {
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE
} else if incarnation_bound {
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_INCARNATION
} else if scanner_publication_lease_token.is_some() {
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE
} else {
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_ORDINARY
};
rustfs_io_metrics::record_ecstore_rename_remote_rpc(rpc_kind);
let rpc_started = rustfs_io_metrics::put_stage_timer();
let mut retried_dual_write = false;
let response = loop {
@@ -3416,7 +2958,6 @@ impl RemoteDisk {
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
attempt_prepare_started,
);
rename_admission_guard.mark_dispatched();
let result = rustfs_io_metrics::observe_put_stage_future(
async {
if multipart_rename {
@@ -3447,7 +2988,6 @@ impl RemoteDisk {
Err(status) => return Err(status.into()),
}
};
rename_admission_guard.complete();
let response_process_started = rustfs_io_metrics::put_stage_timer();
let response_headers_started = rustfs_io_metrics::put_stage_timer();
let _ = response.metadata().len();
@@ -13423,127 +12963,6 @@ mod tests {
.expect("shared gate remains open");
}
#[tokio::test]
async fn rename_admission_is_weighted_shared_and_released_per_item() {
use std::future::Future;
use std::task::{Context, Poll, Waker};
let first = peer_rename_admission("http://rename-peer-a", true).expect("enabled rename gate");
let same = peer_rename_admission("http://rename-peer-a", true).expect("same peer rename gate");
let other = peer_rename_admission("http://rename-peer-b", true).expect("other peer rename gate");
assert!(Arc::ptr_eq(&first, &same));
assert!(!Arc::ptr_eq(&first, &other));
assert!(peer_rename_admission("http://rename-disabled", false).is_none());
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
let permit = Arc::clone(&first)
.acquire_many_owned(4)
.await
.expect("weighted batch admission");
let mut grant = RenameAdmissionGrant {
permit: Some(permit),
batch: true,
};
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
assert!(grant.release_one());
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3);
drop(grant);
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
let held = Arc::clone(&first)
.acquire_many_owned(u32::try_from(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3).expect("small limit"))
.await
.expect("reserve all but three permits");
let mut waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
assert!(matches!(waiting.as_mut().poll(&mut Context::from_waker(Waker::noop())), Poll::Pending));
assert_eq!(first.available_permits(), 0, "the FIFO waiter reserves currently available permits");
drop(waiting);
assert_eq!(
first.available_permits(),
3,
"cancelling a queued weighted acquire must return its reservation"
);
let waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
drop(held);
let acquired = tokio::time::timeout(Duration::from_secs(1), waiting)
.await
.expect("weighted waiter wakes")
.expect("rename gate remains open");
drop(acquired);
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
}
#[tokio::test(start_paused = true)]
async fn rename_admission_quarantine_holds_until_absolute_deadline() {
let gate = peer_rename_admission("http://rename-quarantine-peer", true).expect("enabled rename gate");
let permit = Arc::clone(&gate)
.acquire_many_owned(4)
.await
.expect("weighted batch admission");
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
RenameAdmissionGrant {
permit: Some(permit),
batch: true,
}
.quarantine_until(deadline);
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
tokio::time::advance(Duration::from_millis(999)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
tokio::time::advance(Duration::from_millis(1)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
}
#[tokio::test(start_paused = true)]
async fn rename_call_guard_quarantines_only_after_dispatch() {
let gate = peer_rename_admission("http://rename-call-guard-peer", true).expect("enabled rename gate");
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("pre-dispatch permits");
drop(RenameBatchCallGuard {
admission: Some(RenameAdmissionGrant {
permit: Some(permit),
batch: true,
}),
deadline,
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
});
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("dispatched permits");
let mut guard = RenameBatchCallGuard {
admission: Some(RenameAdmissionGrant {
permit: Some(permit),
batch: true,
}),
deadline,
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
};
guard.mark_dispatched();
drop(guard);
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
tokio::time::advance(Duration::from_secs(1)).await;
tokio::task::yield_now().await;
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
let permit = Arc::clone(&gate)
.acquire_many_owned(4)
.await
.expect("proven pre-mutation permits");
let mut guard = RenameBatchCallGuard {
admission: Some(RenameAdmissionGrant {
permit: Some(permit),
batch: true,
}),
deadline,
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
};
guard.mark_dispatched();
guard.release_without_quarantine();
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
}
#[tokio::test]
async fn remote_disk_builds_optional_complete_body_gate_from_configuration() {
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, Some("3"))], async {
+2 -19
View File
@@ -12378,7 +12378,6 @@ struct DecommissionCapacityLockOrderBarrierState {
external_heal_target_lock_attempted: tokio::sync::Notify,
target_gate_retry_entered: tokio::sync::Notify,
target_gate_retry_entries: AtomicUsize,
target_gate_retry_pause: std::sync::OnceLock<tokio::sync::Semaphore>,
target_gate_inline_retry_entered: tokio::sync::Notify,
target_gate_inline_retry_entries: AtomicUsize,
target_gate_exact_reloads: AtomicUsize,
@@ -12426,7 +12425,6 @@ impl DecommissionCapacityLockOrderBarrier {
external_heal_target_lock_attempted: tokio::sync::Notify::new(),
target_gate_retry_entered: tokio::sync::Notify::new(),
target_gate_retry_entries: AtomicUsize::new(0),
target_gate_retry_pause: std::sync::OnceLock::new(),
target_gate_inline_retry_entered: tokio::sync::Notify::new(),
target_gate_inline_retry_entries: AtomicUsize::new(0),
target_gate_exact_reloads: AtomicUsize::new(0),
@@ -12522,13 +12520,6 @@ impl DecommissionCapacityLockOrderBarrier {
.expect("decommission entries should observe target gate contention");
}
#[cfg(feature = "test-util")]
pub(crate) fn pause_target_gate_retries(&self) -> &tokio::sync::Semaphore {
self.state
.target_gate_retry_pause
.get_or_init(|| tokio::sync::Semaphore::new(0))
}
#[cfg(feature = "test-util")]
pub(crate) fn target_gate_inline_retries(&self) -> usize {
self.state.target_gate_inline_retry_entries.load(Ordering::Acquire)
@@ -12639,9 +12630,6 @@ impl DecommissionCapacityLockOrderBarrier {
#[cfg(test)]
impl Drop for DecommissionCapacityLockOrderBarrier {
fn drop(&mut self) {
if let Some(pause) = self.state.target_gate_retry_pause.get() {
pause.close();
}
self.state.owner_release.notify_one();
self.state.external_object_capacity_probe_release.notify_one();
self.state.external_object_commit_phase_release.notify_one();
@@ -12770,7 +12758,7 @@ async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) {
}
#[cfg(test)]
async fn observe_decommission_target_gate_retry(store_id: uuid::Uuid) {
fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
@@ -12781,11 +12769,6 @@ async fn observe_decommission_target_gate_retry(store_id: uuid::Uuid) {
if let Some(barrier) = barrier {
barrier.target_gate_retry_entries.fetch_add(1, Ordering::AcqRel);
barrier.target_gate_retry_entered.notify_one();
if let Some(pause) = barrier.target_gate_retry_pause.get()
&& let Ok(permit) = pause.acquire().await
{
permit.forget();
}
}
}
@@ -17655,7 +17638,7 @@ impl ECStore {
target_busy_attempt: usize,
) -> Result<Option<DecommissionCapacityTargetPermit>> {
#[cfg(test)]
observe_decommission_target_gate_retry(self.id).await;
notify_decommission_target_gate_retry(self.id);
let mut wait_attempt = target_busy_attempt;
let target_guard = loop {
let retry_delay =
+10 -20
View File
@@ -8360,16 +8360,13 @@ mod decommission_lock_order_tests {
.await
.expect("hold the target gate before starting both waiters");
let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id);
// Both workers must observe the original gate as busy, but slow disk I/O
// in the holder or first waiter must not consume the second waiter's retry budget.
let retry_pause = retry_observer.pause_target_gate_retries();
let mut workers = tokio::task::JoinSet::new();
let mut workers = Vec::new();
for object in objects {
let worker_store = Arc::clone(&other_store);
let worker_bucket = bucket.clone();
let source_set = other_store.pools[0].get_disks_by_key(object);
workers.spawn(async move {
workers.push(tokio::spawn(async move {
worker_store
.decommission_entry_for_test(
0,
@@ -8381,7 +8378,7 @@ mod decommission_lock_order_tests {
source_set,
)
.await
});
}));
}
retry_observer.wait_until_target_gate_retries(2).await;
assert_eq!(
@@ -8428,7 +8425,6 @@ mod decommission_lock_order_tests {
);
drop(target_guard);
retry_pause.add_permits(1);
retry_observer.wait_until_owner_paused().await;
assert_eq!(
retry_observer.target_gate_exact_reloads(),
@@ -8447,13 +8443,6 @@ mod decommission_lock_order_tests {
.expect("a pinned waiter must not hold the alternate target gate");
drop(alternate_probe);
retry_observer.release_owner();
tokio::time::timeout(Duration::from_secs(30), workers.join_next())
.await
.expect("the pinned waiter should finish before the other retries")
.expect("the pinned waiter must exist")
.expect("the pinned waiter task should not panic")
.expect("the pinned waiter migration should succeed");
retry_pause.add_permits(1);
retry_observer.wait_until_owner_paused().await;
assert_eq!(
retry_observer.target_gate_exact_reloads(),
@@ -8473,12 +8462,13 @@ mod decommission_lock_order_tests {
drop(original_probe);
retry_observer.release_owner();
tokio::time::timeout(Duration::from_secs(30), workers.join_next())
.await
.expect("the reselected waiter should finish after permit transfer")
.expect("the reselected waiter must exist")
.expect("the reselected waiter task should not panic")
.expect("the reselected waiter migration should succeed");
for worker in workers {
tokio::time::timeout(Duration::from_secs(30), worker)
.await
.expect("same-target waiter should finish after permit transfer")
.expect("same-target waiter task should not panic")
.expect("same-target waiter migration should succeed");
}
assert_eq!(
retry_observer.target_gate_exact_reloads(),
2,
+4 -93
View File
@@ -604,10 +604,10 @@ pub(crate) async fn initialize_local_disk_maps(
let disk = match new_disk(ep, opt).await {
Ok(disk) => disk,
Err(crate::disk::error::DiskError::FileAccessDenied | crate::disk::error::DiskError::VolumeNotFound) => {
// Preserve the configured slot when a local volume is unavailable.
// LocalDisk logs the cause; format loading retains authority
// over fatal errors and quorum. Never create a missing root.
Err(crate::disk::error::DiskError::FileAccessDenied) => {
// LocalDisk already logged the filesystem cause and repair
// hint. Keep this slot unavailable so ECStore initialization
// can apply its existing fatal-error and quorum rules.
global_local_disk_map.insert(ep.to_string(), None);
continue;
}
@@ -888,95 +888,6 @@ mod tests {
disk_map.keys().collect::<Vec<_>>()
);
}
#[tokio::test]
async fn missing_local_volume_preserves_offline_slot_without_creating_root() {
use crate::disk::{DiskOption, error::DiskError};
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
for drive_count in [2, 3, 4] {
let directory = tempfile::tempdir().expect("create volume parent");
let mut endpoints = Vec::new();
for index in 0..drive_count {
let path = directory.path().join(format!("disk{index}"));
if index != 0 {
std::fs::create_dir(&path).expect("create surviving volume");
}
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(index);
endpoints.push(endpoint);
}
let missing = endpoints[0].to_string();
let pools = EndpointServerPools(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: drive_count,
endpoints: endpoints.into(),
cmd_line: "missing-volume".into(),
platform: "test".into(),
}]);
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let options = DiskOption {
cleanup: true,
health_check: false,
};
super::initialize_local_disk_maps(&ctx, pools.clone(), &options)
.await
.expect("missing volume should remain offline for quorum evaluation");
let map = ctx.local_disk_map();
let map = map.read().await;
assert_eq!(map.len(), drive_count);
assert!(matches!(map.get(&missing), Some(None)));
assert_eq!(map.values().flatten().count(), drive_count - 1);
drop(map);
let drives = ctx.local_disk_set_drives();
let drives = drives.read().await;
assert_eq!(drives[0][0].len(), drive_count);
assert!(drives[0][0][0].is_none());
assert!(drives[0][0][1..].iter().all(Option::is_some));
drop(drives);
let (disks, errors) = crate::store::init_format::init_disks(&ctx, &pools.0[0].endpoints, &options).await;
assert!(disks[0].is_none());
assert_eq!(errors[0], Some(DiskError::VolumeNotFound));
assert!(disks[1..].iter().all(Option::is_some));
assert!(errors[1..].iter().all(Option::is_none));
assert!(!directory.path().join("disk0").exists(), "startup must not create a missing mount root");
}
}
#[tokio::test]
async fn invalid_local_volume_still_fails_registry_initialization() {
use crate::disk::DiskOption;
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
let directory = tempfile::tempdir().expect("create volume parent");
let path = directory.path().join("regular-file");
std::fs::write(&path, b"preserve existing file").expect("create invalid volume");
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let pools = EndpointServerPools(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 1,
endpoints: vec![endpoint].into(),
cmd_line: "invalid-volume".into(),
platform: "test".into(),
}]);
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let options = DiskOption {
cleanup: true,
health_check: false,
};
super::initialize_local_disk_maps(&ctx, pools, &options)
.await
.expect_err("a non-directory volume must retain its initialization error");
assert_eq!(std::fs::read(&path).expect("read existing file"), b"preserve existing file");
}
#[tokio::test]
async fn multipart_invalid_registry_pool_index_has_no_authority_or_registry_effect() {
use crate::disk::DiskOption;
@@ -82,6 +82,8 @@ impl EventNotifier {
#[derive(Debug, Default)]
pub struct EventArgs {
/// Bucket instance verified by storage when producing this event.
pub bucket_incarnation_id: Option<uuid::Uuid>,
pub event_name: String,
pub bucket_name: String,
pub object: ObjectInfo,
@@ -135,6 +137,7 @@ pub(crate) mod test_recorder {
#[derive(Clone, Debug)]
pub(crate) struct RecordedEvent {
pub(crate) event_name: String,
pub(crate) bucket_incarnation_id: Option<Uuid>,
pub(crate) bucket: String,
pub(crate) object: String,
pub(crate) version_id: Option<Uuid>,
@@ -154,6 +157,7 @@ pub(crate) mod test_recorder {
register_event_dispatch_hook(|args| {
recorded().lock().unwrap_or_else(|err| err.into_inner()).push(RecordedEvent {
event_name: args.event_name,
bucket_incarnation_id: args.bucket_incarnation_id,
bucket: args.bucket_name,
object: args.object.name,
version_id: args.object.version_id,
@@ -15,6 +15,8 @@
use crate::diagnostics::admin_server_info::get_local_server_property;
use crate::runtime::sources as runtime_sources;
use crate::storage_api_contracts::admin::StorageAdminApi;
#[cfg(test)]
use chrono::Utc;
use jiff::Timestamp;
use rustfs_heal_contracts::heal_channel::DriveState;
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
@@ -598,6 +600,10 @@ mod test {
use serial_test::serial;
use std::time::Duration;
fn chrono_to_jiff_timestamp(timestamp: chrono::DateTime<Utc>) -> jiff::Timestamp {
jiff::Timestamp::try_from(std::time::SystemTime::from(timestamp)).expect("test timestamp should fit in jiff")
}
#[test]
fn tes_types() {
let t = MetricType::ALL;
@@ -679,10 +685,10 @@ mod test {
#[test]
fn scanner_metrics_mapping_preserves_partial_source_status() {
let current_started = Timestamp::now() - jiff::SignedDuration::from_secs(5);
let current_started = Utc::now() - chrono::Duration::seconds(5);
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
current_cycle_active: true,
current_started,
current_started: chrono_to_jiff_timestamp(current_started),
last_cycle_partial_source: "usage".to_string(),
last_cycle_partial_source_code: 1,
cycle_timeout_total: 3,
@@ -697,7 +703,7 @@ mod test {
});
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_started, current_started);
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
assert_eq!(scanner.last_cycle_partial_source, "usage");
assert_eq!(scanner.last_cycle_partial_source_code, 1);
assert_eq!(scanner.cycle_timeout_total, 3);
@@ -717,8 +723,8 @@ mod test {
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
let previous_init_time = *rustfs_scanner_metrics::GLOBAL_INIT_TIME.read().await;
let previous_cycle = global_metrics().get_cycle().await;
let init_time = Timestamp::now() - jiff::SignedDuration::from_hours(1);
let cycle_started = Timestamp::now() - jiff::SignedDuration::from_secs(5);
let init_time = Utc::now() - chrono::Duration::hours(1);
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
*rustfs_scanner_metrics::GLOBAL_INIT_TIME.write().await = Some(init_time);
let cycle = CurrentCycle {
current: 0,
@@ -742,7 +748,7 @@ mod test {
aggregated.merge(decoded);
let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_started, cycle_started);
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(cycle_started));
}
#[test]
File diff suppressed because it is too large Load Diff
+1 -5
View File
@@ -17333,12 +17333,8 @@ mod tests {
source.metadata_only = true;
source.etag = Some("metadata-copy-etag".to_string());
let dst_opts = ObjectOptions {
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
..opts.clone()
};
let copied = set_disks
.copy_object(bucket, object, bucket, object, &mut source, &opts, &dst_opts)
.copy_object(bucket, object, bucket, object, &mut source, &opts, &opts)
.await
.expect("metadata self-copy should succeed");
assert_eq!(copied.user_defined.get("x-amz-meta-copy-check").map(String::as_str), Some("present"));
+20 -23
View File
@@ -9294,7 +9294,12 @@ impl SetDisks {
)
.await?;
if let Some((_, deleted_object)) = replication_delete {
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await;
ReplicationLifecycleBridge::schedule_delete(
bucket.to_string(),
deleted_object,
opts.expected_bucket_incarnation_id,
)
.await;
}
deleted += 1;
}
@@ -11563,7 +11568,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
)
.await?;
if let Some((_, deleted_object)) = replication_delete {
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await;
ReplicationLifecycleBridge::schedule_delete(
bucket.to_string(),
deleted_object,
opts.expected_bucket_incarnation_id,
)
.await;
}
}
} else {
@@ -12732,6 +12742,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
send_event(EventArgs {
event_name: event_name.to_string(),
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
bucket_name: bucket.to_string(),
object: obj_info,
user_agent: "Internal: [ILM-Transition]".to_string(),
@@ -12878,6 +12889,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, &opts).await?;
send_event(EventArgs {
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
bucket_name: bucket.to_string(),
object: restored_info,
user_agent: "Internal: [Restore-Completed]".to_string(),
@@ -13014,6 +13026,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, opts).await?;
send_event(EventArgs {
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
bucket_name: bucket.to_string(),
object: restored_info,
user_agent: "Internal: [Restore-Completed]".to_string(),
@@ -17182,12 +17195,7 @@ mod metadata_mutation_generation_tests {
copy_object,
&mut copy_info,
&ObjectOptions::default(),
&ObjectOptions {
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
&ObjectOptions::default(),
)
.await
.expect("metadata COPY should succeed");
@@ -17365,11 +17373,7 @@ mod metadata_mutation_generation_tests {
let metadata = Arc::make_mut(&mut source.user_defined);
metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "new=value".to_string());
metadata.insert("expires".to_string(), expires.format(&Rfc3339).expect("expiry"));
let dst_opts = ObjectOptions {
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
..Default::default()
};
set.copy_object(bucket, "object", bucket, "object", &mut source, &opts, &dst_opts)
set.copy_object(bucket, "object", bucket, "object", &mut source, &opts, &opts)
.await
.expect("explicit metadata wins over externalized fields");
let mut copied = set.get_object_info(bucket, "object", &opts).await.expect("copied metadata");
@@ -17380,7 +17384,7 @@ mod metadata_mutation_generation_tests {
copied.metadata_only = true;
copied.user_tags = Arc::new(String::new());
copied.expires = None;
set.copy_object(bucket, "object", bucket, "object", &mut copied, &opts, &dst_opts)
set.copy_object(bucket, "object", bucket, "object", &mut copied, &opts, &opts)
.await
.expect("explicit empty replacement");
let cleared = set.get_object_info(bucket, "object", &opts).await.expect("cleared metadata");
@@ -17397,10 +17401,6 @@ mod metadata_mutation_generation_tests {
}
let valid_object = "valid-sidecar";
let dst_opts = ObjectOptions {
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
..Default::default()
};
let (mut valid_source, _) = put_and_prime(&set_disks, bucket, valid_object, b"valid sidecar body").await;
persist_part_checksum_sidecar(&set_disks, bucket, valid_object, r#"[[1,[["CRC32C","AAAAAA=="]]]]"#).await;
valid_source.metadata_only = true;
@@ -17413,7 +17413,7 @@ mod metadata_mutation_generation_tests {
valid_object,
&mut valid_source,
&ObjectOptions::default(),
&dst_opts,
&ObjectOptions::default(),
)
.await
.expect("metadata-only copy should preserve a valid checksum sidecar");
@@ -17486,7 +17486,7 @@ mod metadata_mutation_generation_tests {
conflicting_object,
&mut conflicting_source,
&ObjectOptions::default(),
&dst_opts,
&ObjectOptions::default(),
)
.await
.expect_err("metadata-only copy must reject conflicting checksum aliases");
@@ -25532,9 +25532,6 @@ mod put_object_tmp_cleanup_tests {
&ObjectOptions {
no_lock: true,
namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()),
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
)
+1 -6
View File
@@ -11043,12 +11043,7 @@ mod dangling_read_tests {
object,
&mut source,
&ObjectOptions::default(),
&ObjectOptions {
object_lock_config_snapshot: Some(Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
&ObjectOptions::default(),
)
.await
.expect("a CopyObject source without a version id must resolve like a latest GET");
-6
View File
@@ -205,12 +205,6 @@ fn heal_format_fence_lost_error() -> Error {
}
impl ECStore {
/// Whether this exact storage instance requires an incarnation-bound write
/// for the object currently owned by a bucket-heal scope.
pub fn object_requires_incarnation(&self, bucket: &str, object: &str) -> bool {
bucket_heal_scope_for_object(bucket, object).is_some_and(|scope| std::ptr::eq(Arc::as_ptr(&scope.store), self))
}
pub async fn purge_delete_marker_with_proof(
self: &Arc<Self>,
bucket: &str,
+16 -155
View File
@@ -752,60 +752,18 @@ impl ECStore {
Ok(ec)
}
async fn init_pool_metadata_for_startup(self: &Arc<Self>) -> Result<(PoolMeta, PoolMetaReplicaState)> {
#[instrument(level = "debug", skip(self, rx))]
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
runtime_sources::ensure_boot_time().await;
let should_persist_pool_meta = self
.pools
.first()
.is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
// Lock order is save gate -> distributed metadata fence, matching
// Lock order is save gate -> distributed metadata writer, matching
// runtime mutations. RPC locks are available before ECStore publication
// and do not depend on the later fleet proof.
let mut startup_write_state = self.pool_meta_save_gate.write().await;
#[cfg(feature = "e2e-test-hooks")]
let startup_attempt = uuid::Uuid::new_v4();
// A committed snapshot needs only a shared namespace fence. A writer
// transition releases this reader and re-reads every observation below
// under the existing exclusive fence before applying side effects.
if !startup_write_state.bootstrap_identity_proven() && startup_write_state.attested_pool_indices().is_empty() {
let pool = self
.pools
.first()
.ok_or_else(|| Error::other("startup has no storage pools"))?;
let lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, "pool.bin").await?;
let reader = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
let load = load_pool_meta_for_startup(self.pools.clone(), &mut startup_write_state);
#[cfg(feature = "e2e-test-hooks")]
let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load);
match load.await {
Ok((meta, replicas)) => {
let topology_update = meta.validate(self.pools.clone())?;
let requires_writer = should_persist_pool_meta
&& (topology_update
|| (replicas.needs_repair && replicas.repair_write_safe)
|| startup_write_state.identity_requires_repair());
if !topology_update && !requires_writer {
startup_write_state.ensure_startup_identity_initialized()?;
#[cfg(feature = "e2e-test-hooks")]
crate::core::pools::startup_cas_test_observe(serde_json::json!({
"kind": "startup-classifier", "attempt": startup_attempt,
"elected_writer": should_persist_pool_meta,
"needs_repair": replicas.needs_repair,
"repair_write_safe": replicas.repair_write_safe,
"topology_update": topology_update,
}));
self.install_startup_pool_metadata(&meta, Some(&reader)).await?;
return Ok((meta, replicas));
}
}
Err(err)
if should_persist_pool_meta
&& err
.pool_metadata_failure()
.is_some_and(|failure| failure.kind == crate::error::PoolMetadataFailure::ReadUnavailable) => {}
Err(err) => return Err(err),
}
drop(reader);
}
let startup_pool_meta_guard = if should_persist_pool_meta {
let pool = self
.pools
@@ -866,6 +824,7 @@ impl ECStore {
"repair_write_safe": pool_meta_replica_state.repair_write_safe,
"topology_update": update,
}));
let endpoints = runtime_sources::endpoint_pools_or_default();
let mut installed_pool_meta = if update {
PoolMeta::new(&self.pools, &meta)?
@@ -922,34 +881,19 @@ impl ECStore {
installed_pool_meta = persist.await?;
}
self.install_startup_pool_metadata(&installed_pool_meta, startup_pool_meta_guard.as_deref())
.await?;
if startup_pool_meta_guard
.as_ref()
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
{
return Err(Error::other("startup pool metadata writer fence was lost before publication"));
}
{
let mut pool_meta = self.pool_meta.write().await;
*pool_meta = installed_pool_meta.clone();
}
drop(startup_pool_meta_guard);
drop(startup_write_state);
Ok((installed_pool_meta, pool_meta_replica_state))
}
async fn install_startup_pool_metadata(
&self,
metadata: &PoolMeta,
fence: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<()> {
let mut pool_meta = self.pool_meta.write().await;
if fence.is_some_and(|guard| guard.is_lock_lost() || guard.is_released()) {
return Err(Error::other("startup pool metadata fence was lost before publication"));
}
*pool_meta = metadata.clone();
Ok(())
}
#[instrument(level = "debug", skip(self, rx))]
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
runtime_sources::ensure_boot_time().await;
let (installed_pool_meta, pool_meta_replica_state) = self.init_pool_metadata_for_startup().await?;
let endpoints = runtime_sources::endpoint_pools_or_default();
resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?;
let rebalance_resume_required = {
let rebalance_meta = self.rebalance_meta.read().await;
@@ -3787,89 +3731,6 @@ mod tests {
(instance_ctx, store, shutdown)
}
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn committed_startup_pool_metadata_uses_shared_fence_without_disk_writes() {
use crate::disk::{FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET};
use crate::storage_api_contracts::namespace::NamespaceLocking;
for reopen in [false, true] {
let directory = tempfile::tempdir().expect("create committed metadata deployment");
let (_, seeded, seed_shutdown) =
without_storage_class_env(build_isolated_test_store(directory.path(), "startup-read-fence", &[4])).await;
let (store, shutdown) = if reopen {
seed_shutdown.cancel();
drop(seeded);
let (_, reopened, shutdown) =
without_storage_class_env(build_isolated_test_store(directory.path(), "startup-read-fence", &[4])).await;
(reopened, shutdown)
} else {
(seeded, seed_shutdown)
};
let disks = store.pools[0].disk_set[0].disk_inventory().await;
let snapshot = || {
let mut files = std::collections::BTreeMap::new();
for (index, slot) in disks.iter().enumerate() {
let disk = slot.as_ref().expect("committed drive handle");
for object in [POOL_META_NAME, POOL_META_IDENTITY_NAME, FORMAT_CONFIG_FILE] {
let root = disk.path().join(RUSTFS_META_BUCKET).join(object);
assert!(root.exists(), "committed authority must be present: {root:?}");
let mut pending = vec![root];
while let Some(path) = pending.pop() {
let metadata = std::fs::symlink_metadata(&path).expect("inspect authority file type");
assert!(!metadata.file_type().is_symlink(), "authority fixture must not follow symlinks");
if metadata.is_dir() {
for entry in std::fs::read_dir(&path).expect("inspect authority object directory") {
pending.push(entry.expect("authority directory entry").path());
}
} else {
assert!(metadata.is_file(), "authority fixture must contain regular files");
let contents = std::fs::read(&path).expect("read physical authority bytes");
let relative = path
.strip_prefix(disk.path())
.expect("authority stays under fixture drive")
.to_path_buf();
files.insert((index, relative), contents);
}
}
}
}
files
};
let before = snapshot();
let lock = store.pools[0]
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
.await
.expect("metadata namespace lock");
let mut reader = lock
.get_read_lock(Duration::from_secs(1))
.await
.expect("hold a competing metadata reader");
let (metadata, replicas) = tokio::time::timeout(Duration::from_secs(2), store.init_pool_metadata_for_startup())
.await
.expect("committed startup must not wait for an exclusive lock")
.expect("committed startup metadata is readable under a shared fence");
assert_eq!(metadata.pools[0].cmd_line, "startup-read-fence-pool-0");
assert!(!replicas.needs_repair);
assert!(!reader.is_lock_lost());
let after = snapshot();
assert_eq!(
after, before,
"shared startup observation cannot mutate metadata, identity, or format authority"
);
assert!(reader.release(), "release the snapshot's shared fence");
let mut invalid = metadata.clone();
invalid.pools[0].cmd_line = "must-not-be-published".to_string();
store
.install_startup_pool_metadata(&invalid, Some(&reader))
.await
.expect_err("a released fence cannot publish startup metadata");
assert_eq!(store.pool_meta.read().await.pools[0].cmd_line, metadata.pools[0].cmd_line);
drop(reader);
shutdown.cancel();
}
}
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn interrupted_format_bootstrap_recovers_two_pools_through_store_init() {
+50 -311
View File
@@ -169,34 +169,36 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
{
return Err(Error::CorruptedFormat);
}
let native_read_candidate = if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum))) {
select_existing_format_read_quorum(&formats, &errs, set_drive_count)
} else {
None
};
let partial_native_with_blanks = formats_present > 0
if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum)))
&& let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count)
{
retain_format_quorum_members(
instance_ctx,
disks,
&format,
&members,
Some(&unformatted_bootstrap_routes),
set_drive_count,
)
.await?;
// Loading an existing format grants no authority to initialize pool
// metadata or migrate formats. The persisted storage classes and each
// object's EC metadata still determine service and operation quorums.
return Ok(LoadedFormat {
format,
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
});
}
let resumable_partial_migration = formats_present > 0
&& errs.iter().any(|error| matches!(error, Some(DiskError::UnformattedDisk)))
&& format_quorum.as_ref().is_some_and(Result::is_err);
let resumable_partial_migration = partial_native_with_blanks
&& format_quorum.as_ref().is_some_and(Result::is_err)
&& formats.iter().zip(&errs).all(|(format, error)| {
format.is_some() || matches!(error, Some(DiskError::UnformattedDisk | DiskError::DiskNotFound))
});
let allow_migration = first_disk && (all_unformatted || resumable_partial_migration);
if partial_native_with_blanks && !allow_migration && native_read_candidate.is_none() {
return Err(Error::ErasureReadQuorum);
}
if allow_migration || (partial_native_with_blanks && native_read_candidate.is_some()) {
if first_disk && (all_unformatted || resumable_partial_migration) {
info!(all_unformatted, resumable_partial_migration, "checking for a legacy storage format");
let mode = match native_read_candidate.as_ref() {
Some((reference, _)) => LegacyFormatMode::NativeRead {
reference,
native_errors: &errs,
allow_migration,
},
None => LegacyFormatMode::Migration,
};
match try_migrate_format(disks, &formats, set_count, set_drive_count, mode).await {
match try_migrate_format(disks, &formats, set_count, set_drive_count).await {
Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => {
info!("Migrated format from MinIO config");
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, None, set_drive_count).await?;
@@ -215,10 +217,10 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
);
return Err(Error::CorruptedFormat);
}
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::ReadCompatible) => {}
Ok(LegacyFormatOutcome::None) => {}
Err(e) => return Err(e),
}
if first_disk && fresh_bootstrap_proven {
if fresh_bootstrap_proven {
let fm = init_format_erasure(instance_ctx, disks, set_count, set_drive_count, deployment_id).await?;
return Ok(LoadedFormat {
format: fm,
@@ -227,26 +229,6 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
}
}
// Resolve legacy migration before admitting native reads. Blank replacement
// slots contribute no votes and receive no formatting/bootstrap authority.
if let Some((format, members)) = native_read_candidate {
retain_format_quorum_members(
instance_ctx,
disks,
&format,
&members,
Some(&unformatted_bootstrap_routes),
set_drive_count,
)
.await?;
// Loading an existing format grants no authority to initialize pool
// metadata or migrate formats. The persisted storage classes and each
// object's EC metadata still determine service and operation quorums.
return Ok(LoadedFormat {
format,
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
});
}
info!(
"first_disk: {}, should_init_erasure_disks: {}",
first_disk,
@@ -289,11 +271,9 @@ async fn verified_legacy_adoption_source(
set_count: usize,
set_drive_count: usize,
) -> Result<PoolMetaBootstrapAuthority> {
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count, LegacyFormatMode::Migration).await {
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count).await {
Ok(LegacyFormatOutcome::Migrated { .. }) => Ok(PoolMetaBootstrapAuthority::LegacyAdoption),
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::ReadCompatible | LegacyFormatOutcome::Incompatible) => {
Ok(PoolMetaBootstrapAuthority::None)
}
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::Incompatible) => Ok(PoolMetaBootstrapAuthority::None),
Err(err) => {
debug!(
error = %err,
@@ -463,49 +443,25 @@ enum LegacyFormatOutcome {
/// version mismatch, or a parse failure). The caller must decide how to
/// proceed; creating a fresh format would leave the legacy objects unreadable.
Incompatible,
/// Observed legacy formats agree with the native read quorum, without granting writer authority.
ReadCompatible,
/// No MinIO `format.json` was present on any disk (a normal fresh install).
None,
}
/// Native reads validate observed legacy identities without electing a migration writer.
enum LegacyFormatMode<'a> {
Migration,
NativeRead {
reference: &'a FormatV3,
native_errors: &'a [Option<DiskError>],
allow_migration: bool,
},
}
/// Tries to migrate an on-disk MinIO `format.json` into RustFS format files.
///
/// Returns [`LegacyFormatOutcome`] describing whether a legacy format was present
/// and, if so, whether it was compatible. `Err` is returned for read or persist
/// failures that prevent a conclusive migration decision. Native read admission
/// checks every observed legacy format against its existing layout and tolerates
/// unavailable nonvoters. Migration requires complete observations and a strict majority.
/// failures that prevent a conclusive migration decision.
async fn try_migrate_format(
disks: &[Option<DiskStore>],
rustfs_formats: &[Option<FormatV3>],
set_count: usize,
set_drive_count: usize,
mode: LegacyFormatMode<'_>,
) -> Result<LegacyFormatOutcome> {
let (native_reference, native_errors, allow_migration) = match mode {
LegacyFormatMode::Migration => (None, None, true),
LegacyFormatMode::NativeRead {
reference,
native_errors,
allow_migration,
} => (Some(reference), Some(native_errors), allow_migration),
};
let legacy_format_max_bytes = legacy_format_max_bytes(disks.len())?;
let mut legacy_seen = false;
let mut invalid_legacy_format = false;
let mut read_error: Option<Error> = None;
let mut unavailable_error = None;
let mut legacy_formats = vec![None; disks.len()];
let reads = stream::iter(
@@ -517,13 +473,6 @@ async fn try_migrate_format(
let Some(disk) = disk else {
return (index, None);
};
// Native format loading already proved this peer unavailable.
// Reuse that observation and keep it ineligible for migration.
if let Some(error) = native_errors.and_then(|errors| errors.get(index)).and_then(Option::as_ref)
&& is_network_like_disk_error(error)
{
return (index, Some(Err(error.clone())));
}
let reader = match disk.read_file(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE).await {
Ok(reader) => reader,
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return (index, None),
@@ -563,9 +512,6 @@ async fn try_migrate_format(
match read {
Ok(Some(format)) => legacy_formats[index] = Some(format),
Ok(None) => invalid_legacy_format = true,
Err(err) if native_reference.is_some() && is_network_like_disk_error(&err) => {
unavailable_error.get_or_insert(err);
}
Err(err) => {
read_error.get_or_insert_with(|| err.into());
}
@@ -582,15 +528,6 @@ async fn try_migrate_format(
return Ok(LegacyFormatOutcome::Incompatible);
}
if let Some(reference) = native_reference {
if !formats_match_reference_slots(&legacy_formats, reference, 0) {
return Ok(LegacyFormatOutcome::Incompatible);
}
if !allow_migration || unavailable_error.is_some() || legacy_formats.iter().flatten().count() <= disks.len() / 2 {
return Ok(LegacyFormatOutcome::ReadCompatible);
}
}
let format = match get_format_erasure_in_quorum(&legacy_formats, 0) {
Ok(format) => format,
Err(_) => return Ok(LegacyFormatOutcome::Incompatible),
@@ -740,7 +677,7 @@ fn select_existing_format_read_quorum(
|| check_format_erasure_value_for_topology(reference, formats.len(), set_drive_count).is_err()
|| formats.iter().zip(errors).any(|(format, error)| match (format, error) {
(Some(_), None) => false,
(None, Some(DiskError::DiskNotFound | DiskError::UnformattedDisk)) => false,
(None, Some(DiskError::DiskNotFound)) => false,
(None, Some(error)) => !is_network_like_disk_error(error),
_ => true,
})
@@ -1508,23 +1445,13 @@ mod tests {
}
#[test]
fn existing_format_read_quorum_excludes_blank_and_network_unavailable_slots() {
let (formats, mut errors) = half_pool_formats();
errors[2] = Some(DiskError::UnformattedDisk);
errors[3] = Some(DiskError::Timeout);
let (_, members) = select_existing_format_read_quorum(&formats, &errors, 4)
.expect("blank and unreachable slots cannot block an existing native read quorum");
assert!(!members[2] && !members[3]);
assert_eq!(members.iter().filter(|member| **member).count(), 12);
}
#[test]
fn existing_format_read_quorum_rejects_conflicts_and_invalid_slots() {
fn existing_format_read_quorum_rejects_conflicts_invalid_slots_and_blank_disks() {
for failure in [
"conflict",
"wrong_slot",
"nil",
"duplicate",
"blank",
"corrupt",
"access_denied",
"tie",
@@ -1546,6 +1473,7 @@ mod tests {
format.erasure.sets[5][3] = format.erasure.sets[0][0];
}
}
"blank" => errors[2] = Some(DiskError::UnformattedDisk),
"corrupt" => errors[2] = Some(DiskError::CorruptedFormat),
"access_denied" => errors[2] = Some(DiskError::FileAccessDenied),
"tie" => {
@@ -1604,186 +1532,6 @@ mod tests {
}
}
#[tokio::test]
async fn existing_format_read_quorum_quarantines_blank_replacement_without_writes() {
for first_disk in [false, true] {
for surviving_slot in 0..2 {
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
let blank = 1 - surviving_slot;
let mut format = FormatV3::new(1, 2);
format.erasure.this = format.erasure.sets[0][surviving_slot];
save_format_file(&disks[surviving_slot], &Some(format.clone()))
.await
.expect("persist surviving format");
let original = disks.clone();
let before = format_bytes(&original).await;
let loaded = connect_load_init_formats_with_instance_ctx(
&Arc::new(InstanceContext::new()),
first_disk,
&mut disks,
1,
2,
None,
)
.await
.expect("existing native format must load at read quorum with a blank replacement");
assert_eq!(loaded.format.shared_identity(), format.shared_identity());
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
assert!(disks[surviving_slot].is_some());
assert!(disks[blank].is_none(), "blank replacement cannot vote or serve data");
assert_eq!(format_bytes(&original).await, before, "read admission cannot write either format");
assert!(matches!(
load_format_erasure(original[blank].as_ref().expect("blank handle"), false).await,
Err(DiskError::UnformattedDisk)
));
}
}
}
#[tokio::test]
async fn native_read_quorum_with_blank_and_unavailable_slots_validates_legacy_without_writes() {
for first_disk in [false, true] {
for scenario in [
"native-remote",
"legacy-missing",
"foreign-remote",
"corrupt-remote",
"wrong-slot-missing",
] {
let (_directory, mut disks) = local_disks_with_set_width(4, 4).await;
let reference = FormatV3::new(1, 4);
for slot in 0..2 {
let mut format = reference.clone();
format.erasure.this = format.erasure.sets[0][slot];
save_format_file(&disks[slot], &Some(format.clone()))
.await
.expect("persist native read quorum");
if scenario != "native-remote" {
write_legacy_format(&disks[slot], &format).await;
}
}
match scenario {
"foreign-remote" => {
let mut foreign = FormatV3::new(1, 4);
foreign.erasure.this = foreign.erasure.sets[0][2];
write_legacy_format(&disks[2], &foreign).await;
}
"corrupt-remote" => write_legacy_bytes(&disks[2], bytes::Bytes::from_static(b"corrupt legacy")).await,
"wrong-slot-missing" => {
let mut wrong_slot = reference.clone();
wrong_slot.erasure.this = reference.erasure.sets[0][0];
write_legacy_format(&disks[2], &wrong_slot).await;
}
_ => {}
}
let original = disks.clone();
let before = format_bytes(&original).await;
if scenario.ends_with("remote") {
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve unavailable peer address");
let address = listener.local_addr().expect("peer address");
drop(listener);
let mut endpoint = Endpoint::try_from(format!("http://{address}/disk").as_str()).expect("remote endpoint");
endpoint.is_local = false;
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(3);
let remote = crate::cluster::rpc::RemoteDisk::new(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
Arc::new(crate::cluster::rpc::TcpHttpInternodeDataTransport),
)
.await
.expect("construct unavailable remote handle");
let remote = Arc::new(crate::disk::Disk::Remote(Box::new(remote)));
let error = remote
.read_all(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
.await
.expect_err("remote peer must be unavailable");
assert!(is_network_like_disk_error(&error), "expected unavailable peer, got {error:?}");
disks[3] = Some(remote);
} else {
disks[3] = None;
}
let result = connect_load_init_formats_with_instance_ctx(
&Arc::new(InstanceContext::new()),
first_disk,
&mut disks,
1,
4,
None,
)
.await;
if matches!(scenario, "native-remote" | "legacy-missing") {
let loaded = result.unwrap_or_else(|error| panic!("{scenario} must admit native reads: {error:?}"));
assert_eq!(loaded.format.shared_identity(), reference.shared_identity());
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
assert!(disks[..2].iter().all(Option::is_some));
assert!(disks[2..].iter().all(Option::is_none), "unavailable and blank slots cannot vote");
} else {
assert!(
matches!(result, Err(Error::CorruptedFormat)),
"{scenario} observed legacy evidence must reject reads"
);
}
assert_eq!(format_bytes(&original).await, before, "{scenario} cannot write native formats");
}
}
}
#[tokio::test]
async fn partial_native_format_with_legacy_quorum_reads_without_migration_writes() {
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
let legacy = FormatV3::new(1, 2);
write_legacy_majority(&disks, &legacy).await;
let mut native = legacy.clone();
native.erasure.this = native.erasure.sets[0][0];
save_format_file(&disks[0], &Some(native))
.await
.expect("persist partial native format");
let original = disks.clone();
let before = format_bytes(&disks).await;
let loaded =
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), false, &mut disks, 1, 2, None)
.await
.expect("follower can read the existing native layout without migrating legacy formats");
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
assert!(disks[0].is_some() && disks[1].is_none(), "legacy-only slots cannot vote for native reads");
assert_eq!(format_bytes(&original).await, before, "follower legacy inspection cannot persist formats");
}
#[tokio::test]
async fn follower_below_native_read_quorum_skips_legacy_reads_without_writes() {
let (_directory, mut disks) = local_disks_with_set_width(3, 3).await;
let mut format = FormatV3::new(1, 3);
format.erasure.this = format.erasure.sets[0][0];
save_format_file(&disks[0], &Some(format))
.await
.expect("persist one native format below the two-drive read quorum");
for disk in &disks {
write_legacy_bytes(disk, bytes::Bytes::from_static(b"invalid legacy format")).await;
}
let before = format_bytes(&disks).await;
let result = connect_load_init_formats(false, &mut disks, 1, 3, None).await;
assert!(
matches!(result, Err(Error::ErasureReadQuorum)),
"a non-reader must reject quorum before probing corrupt legacy files"
);
assert_eq!(format_bytes(&disks).await, before);
for disk in disks.iter().flatten() {
assert_eq!(
disk.read_all(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE)
.await
.expect("legacy bytes must remain untouched"),
bytes::Bytes::from_static(b"invalid legacy format")
);
}
}
#[tokio::test]
async fn existing_format_load_succeeds_with_a_strict_majority() {
let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await;
@@ -2627,7 +2375,7 @@ mod tests {
#[tokio::test]
async fn legacy_format_migration_resumes_partial_rustfs_writes() {
for drive_count in [2, 3, 4] {
for drive_count in [3, 4] {
let (_temp_dir, mut disks) = local_disks(drive_count).await;
let legacy = FormatV3::new(1, drive_count);
write_legacy_majority(&disks, &legacy).await;
@@ -2686,32 +2434,23 @@ mod tests {
#[tokio::test]
async fn legacy_format_migration_rejects_a_foreign_partial_rustfs_format() {
for first_disk in [false, true] {
for drive_count in [2, 3] {
let (_temp_dir, mut disks) = local_disks(drive_count).await;
let legacy = FormatV3::new(1, drive_count);
write_legacy_majority(&disks, &legacy).await;
let mut foreign = FormatV3::new(1, drive_count);
foreign.erasure.this = foreign.erasure.sets[0][0];
let foreign_id = foreign.id;
save_format_file(&disks[0], &Some(foreign))
.await
.expect("foreign partial RustFS format should be written");
let (_temp_dir, mut disks) = local_disks(3).await;
let legacy = FormatV3::new(1, 3);
write_legacy_majority(&disks, &legacy).await;
let mut foreign = FormatV3::new(1, 3);
foreign.erasure.this = foreign.erasure.sets[0][0];
let foreign_id = foreign.id;
save_format_file(&disks[0], &Some(foreign))
.await
.expect("foreign partial RustFS format should be written");
let result = connect_load_init_formats(first_disk, &mut disks, 1, drive_count, None).await;
if !first_disk && drive_count == 3 {
assert!(matches!(result, Err(Error::ErasureReadQuorum)), "a follower below read quorum must wait");
} else {
assert!(
matches!(result, Err(Error::CorruptedFormat)),
"an eligible reader or migration writer must reject the foreign legacy layout"
);
}
let (formats, _) = load_format_erasure_all(&disks, false).await;
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
}
}
assert!(matches!(
connect_load_init_formats(true, &mut disks, 1, 3, None).await,
Err(Error::CorruptedFormat)
));
let (formats, _) = load_format_erasure_all(&disks, false).await;
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
}
#[tokio::test]
+1 -2
View File
@@ -83,7 +83,6 @@ rustfs-common = { workspace = true }
rustfs-heal-contracts = { workspace = true }
rustfs-madmin = { workspace = true }
rustfs-utils = { workspace = true }
jiff = { workspace = true }
tokio = { workspace = true, features = ["sync", "io-util", "time", "macros", "fs", "rt-multi-thread"] }
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
tonic = { workspace = true }
@@ -111,7 +110,7 @@ walkdir = { workspace = true }
http = { workspace = true }
temp-env = { workspace = true, features = ["async_closure"] }
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
jiff = { workspace = true }
chrono = { workspace = true }
[lib]
doctest = false
+1 -1
View File
@@ -2871,7 +2871,7 @@ mod tests {
}
fn w13_timestamp() -> String {
jiff::Timestamp::now().strftime("%Y-%m-%dT%H:%M:%SZ").to_string()
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
}
fn w13_selection_contains(selection: &str, lane: &str) -> bool {
+126 -92
View File
@@ -18,7 +18,7 @@ use crate::sys::{get_claims_from_token_with_secret, get_claims_from_token_with_s
use crate::{
cache::{Cache, CacheEntity, LockedCache},
error::{Error as IamError, is_err_no_such_group, is_err_no_such_policy, is_err_no_such_user},
store::{GroupEdit, GroupInfo, MappedPolicy, Store, UserType, object::IAM_CONFIG_PREFIX},
store::{GroupInfo, MappedPolicy, Store, UserType, object::IAM_CONFIG_PREFIX},
sys::{
MAX_SVCSESSION_POLICY_SIZE, SESSION_POLICY_NAME, SESSION_POLICY_NAME_EXTRACTED, SITE_REPLICATOR_CLAIM,
SITE_REPLICATOR_SERVICE_ACCOUNT, STATUS_DISABLED, STATUS_ENABLED, UpdateServiceAccountOpts,
@@ -1122,9 +1122,7 @@ where
let direct_policies = mapped_policy.to_slice();
let has_identity_mapping = !direct_policies.is_empty();
// An uncached group mapping can contain a Deny even when a direct policy exists.
let mut policies = direct_policies;
policies.extend(self.policy_db_get_with_groups(Vec::new(), groups).await?);
let policies = self.policy_db_get_with_groups(direct_policies, groups).await?;
Ok((policies, has_identity_mapping))
}
@@ -1580,7 +1578,7 @@ where
args: &AddOrUpdateUserReq,
updated_at: OffsetDateTime,
) -> Result<OffsetDateTime> {
self.add_user_at_with_secret_change(access_key, args, updated_at, true)
self.add_user_at_with_secret_change(access_key, args, updated_at)
.await
.map(|(updated_at, _, _guard)| updated_at)
}
@@ -1591,7 +1589,6 @@ where
access_key: &str,
args: &AddOrUpdateUserReq,
updated_at: OffsetDateTime,
allow_status_change: bool,
) -> Result<(OffsetDateTime, bool, tokio::sync::OwnedMutexGuard<crate::cache::MutationLockState>)> {
// A virtual OIDC parent is a canonical base64url-encoded SHA-256 digest.
// It has no stored user record to collide with in the checks below.
@@ -1649,14 +1646,9 @@ where
});
user_entry.update_at = Some(updated_at);
let secret_changed = if allow_status_change {
self.api
.save_user_identity(access_key, UserType::Reg, user_entry.clone(), None)
.await?;
secret_changed
} else {
self.api.save_regular_user_preserving_status(access_key, &user_entry).await?
};
self.api
.save_user_identity(access_key, UserType::Reg, user_entry.clone(), None)
.await?;
self.update_user_with_claims(access_key, user_entry)?;
@@ -2001,25 +1993,38 @@ where
// whose time predates the entity's load time — a replicated edit
// whose source time is older than this node's startup would
// otherwise never reach the cache.
drop(cache);
let (edit, current) = self.api.begin_group_edit(group).await?;
let gi = match current {
Some(mut gi) => {
let gi = match cache.groups.get(group) {
Some(res) => {
let mut gi = res.clone();
let mut uniq_set: HashSet<String, std::collections::hash_map::RandomState> =
HashSet::from_iter(gi.members.iter().cloned());
uniq_set.extend(members);
uniq_set.extend(members.iter().cloned());
gi.members = uniq_set.into_iter().collect();
gi.update_at = Some(updated_at);
gi
}
None => {
let mut gi = GroupInfo::new(members);
let mut gi = GroupInfo::new(members.clone());
gi.update_at = Some(updated_at);
gi
}
};
self.save_group_and_publish(group, &*edit, gi).await?;
drop(cache);
self.api.save_group_info(group, gi.clone()).await?;
self.cache.with_write_lock(|cache| {
let now = OffsetDateTime::now_utc();
cache.add_or_update_group(group, &gi, now);
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
members.iter().for_each(|member| {
let mut m = user_group_memberships.get(member).cloned().unwrap_or_default();
m.insert(group.to_string());
cache.add_or_update_user_group_membership(member, &m, now);
});
});
Ok(updated_at)
}
@@ -2035,8 +2040,13 @@ where
return Err(Error::InvalidArgument);
}
let (edit, current) = self.api.begin_group_edit(name).await?;
let mut gi = current.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
let cache = self.cache.snapshot();
let groups = Arc::clone(&cache.groups);
let mut gi = match groups.get(name) {
Some(gi) => gi.clone(),
None => return Err(Error::NoSuchGroup(name.to_string())),
};
drop(cache);
if enable {
gi.status = STATUS_ENABLED.to_owned();
@@ -2045,7 +2055,11 @@ where
}
gi.update_at = Some(updated_at);
self.save_group_and_publish(name, &*edit, gi).await?;
self.api.save_group_info(name, gi.clone()).await?;
// Cache publication time is the local clock, not the record stamp
// (see `add_users_to_group_at`).
self.cache.add_or_update_group(name, &gi, OffsetDateTime::now_utc());
Ok(updated_at)
}
@@ -2132,27 +2146,37 @@ where
update_cache_only: bool,
updated_at: OffsetDateTime,
) -> Result<OffsetDateTime> {
let removed: HashSet<_> = members.into_iter().collect();
let remove = |mut gi: GroupInfo| {
gi.members.retain(|m| !removed.contains(m));
gi.update_at = Some(updated_at);
gi
};
if update_cache_only {
return self.cache.with_write_lock(|cache| {
let gi = cache
.state()
.groups
.get(name)
.cloned()
.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
self.publish_group_unlocked(cache, name, &remove(gi));
Ok(updated_at)
});
let cache = self.cache.snapshot();
let mut gi = cache
.groups
.get(name)
.cloned()
.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
drop(cache);
let s: HashSet<&String> = HashSet::from_iter(gi.members.iter());
let d: HashSet<&String> = HashSet::from_iter(members.iter());
gi.members = s.difference(&d).map(|v| v.to_string()).collect::<Vec<String>>();
gi.update_at = Some(updated_at);
if !update_cache_only {
self.api.save_group_info(name, gi.clone()).await?;
}
let (edit, current) = self.api.begin_group_edit(name).await?;
let gi = current.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
self.save_group_and_publish(name, &*edit, remove(gi)).await?;
self.cache.with_write_lock(|cache| {
// Sample after storage completes so a concurrent reload cannot
// make this publication older than the cache it must update.
let now = OffsetDateTime::now_utc();
cache.add_or_update_group(name, &gi, now);
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
members.iter().for_each(|member| {
if let Some(m) = user_group_memberships.get(member) {
let mut m = m.clone();
m.remove(name);
cache.add_or_update_user_group_membership(member, &m, now);
}
});
});
Ok(updated_at)
}
@@ -2177,6 +2201,7 @@ where
let cache = self.cache.snapshot();
let users_cache = Arc::clone(&cache.users);
let groups_cache = Arc::clone(&cache.groups);
drop(cache);
for member in members.iter() {
@@ -2189,60 +2214,65 @@ where
}
}
if !members.is_empty() {
return self.remove_members_from_group_at(group, members, false, updated_at).await;
}
let (edit, current) = self.api.begin_group_edit(group).await?;
let gi = current.ok_or_else(|| Error::NoSuchGroup(group.to_string()))?;
if !gi.members.is_empty() {
let gi = if members.is_empty() {
// Reload from backend so we see latest members (e.g. after user was deleted elsewhere)
let mut m = HashMap::new();
self.api.load_group(group, &mut m).await?;
m.get(group).cloned().ok_or_else(|| Error::NoSuchGroup(group.to_string()))?
} else {
groups_cache
.get(group)
.cloned()
.ok_or_else(|| Error::NoSuchGroup(group.to_string()))?
};
if members.is_empty() && !gi.members.is_empty() {
return Err(Error::GroupNotEmpty);
}
edit.delete_empty().await?;
if members.is_empty() {
if let Err(err) = self.api.delete_mapped_policy(group, UserType::Reg, true).await
&& !is_err_no_such_policy(&err)
{
return Err(err);
}
self.cache.with_write_lock(|cache| {
let now = OffsetDateTime::now_utc();
Self::replace_cached_group_members(cache, group, &[], now);
cache.delete_group(group, now);
cache.delete_group_policy(group, now);
});
if let Err(err) = self.api.delete_group_info(group).await
&& !is_err_no_such_group(&err)
{
return Err(err);
}
Ok(updated_at)
self.cache.with_write_lock(|cache| {
let now = OffsetDateTime::now_utc();
self.remove_group_from_memberships_map_unlocked(cache, group, now);
cache.delete_group(group, now);
cache.delete_group_policy(group, now);
});
return Ok(updated_at);
}
self.remove_members_from_group_at(group, members, false, updated_at).await
}
async fn save_group_and_publish(&self, group: &str, edit: &dyn GroupEdit, gi: GroupInfo) -> Result<()> {
edit.save(&gi).await?;
self.cache
.with_write_lock(|cache| self.publish_group_unlocked(cache, group, &gi));
Ok(())
}
fn publish_group_unlocked(&self, cache: &mut LockedCache, group: &str, gi: &GroupInfo) {
let now = OffsetDateTime::now_utc();
cache.add_or_update_group(group, gi, now);
Self::replace_cached_group_members(cache, group, &gi.members, now);
}
fn replace_cached_group_members(cache: &mut LockedCache, group: &str, members: &[String], now: OffsetDateTime) {
// This node may have missed peer edits, so reconcile against the complete record,
// including memberships that were already stale before this edit began.
let mut missing: HashSet<_> = members.iter().collect();
let reverse = Arc::clone(&cache.state().user_group_memberships);
for (user, groups) in reverse.iter() {
if groups.contains(group) && !missing.remove(user) {
let mut groups = groups.clone();
groups.remove(group);
if groups.is_empty() {
cache.delete_user_group_membership(user, now);
} else {
cache.add_or_update_user_group_membership(user, &groups, now);
}
fn remove_group_from_memberships_map_unlocked(&self, cache: &mut LockedCache, group: &str, now: OffsetDateTime) {
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
for (k, v) in user_group_memberships.iter() {
if v.contains(group) {
let mut m = v.clone();
m.remove(group);
cache.add_or_update_user_group_membership(k, &m, now);
}
}
for user in missing {
let mut groups = reverse.get(user).cloned().unwrap_or_default();
groups.insert(group.to_owned());
cache.add_or_update_user_group_membership(user, &groups, now);
}
fn update_group_memberships_map_unlocked(&self, cache: &mut LockedCache, group: &str, gi: &GroupInfo, now: OffsetDateTime) {
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
for member in gi.members.iter() {
let mut m = user_group_memberships.get(member).cloned().unwrap_or_default();
m.insert(group.to_string());
cache.add_or_update_user_group_membership(member, &m, now);
}
}
@@ -2267,7 +2297,7 @@ where
self.cache.with_write_lock(|cache| {
let now = OffsetDateTime::now_utc();
Self::replace_cached_group_members(cache, group, &[], now);
self.remove_group_from_memberships_map_unlocked(cache, group, now);
cache.delete_group(group, now);
cache.delete_group_policy(group, now);
});
@@ -2277,8 +2307,12 @@ where
let gi = m[group].clone();
self.cache
.with_write_lock(|cache| self.publish_group_unlocked(cache, group, &gi));
self.cache.with_write_lock(|cache| {
let now = OffsetDateTime::now_utc();
cache.add_or_update_group(group, &gi, now);
self.remove_group_from_memberships_map_unlocked(cache, group, now);
self.update_group_memberships_map_unlocked(cache, group, &gi, now);
});
Ok(())
}
+1 -5
View File
@@ -104,12 +104,8 @@ pub(crate) mod crate_boundary {
}
pub(crate) mod object_store {
pub(crate) use super::storage_contracts::{
HTTPPreconditions, ListOperations, NamespaceLocking, ObjectInfoOrErr, ObjectOperations,
};
pub(crate) use super::storage_contracts::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
pub(crate) use rustfs_ecstore::api::config::com::read_config_preserve_empty_with_metadata_opts;
pub(crate) use rustfs_ecstore::api::object::WriteCommitGuard;
pub(crate) use rustfs_ecstore::api::set_disk::get_lock_acquire_timeout;
}
pub(crate) mod runtime {
-19
View File
@@ -21,14 +21,6 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
use std::collections::{HashMap, HashSet};
use time::OffsetDateTime;
/// A backend-owned transaction for one named group. Persistence must fail if its lock is lost.
#[async_trait::async_trait]
pub trait GroupEdit: Send + Sync {
async fn save(&self, group: &GroupInfo) -> Result<()>;
/// Remove the policy mapping, then the group record, after the caller verified emptiness.
async fn delete_empty(&self) -> Result<()>;
}
#[async_trait::async_trait]
pub trait Store: Clone + Send + Sync + 'static {
fn has_watcher(&self) -> bool;
@@ -42,11 +34,6 @@ pub trait Store: Clone + Send + Sync + 'static {
async fn delete_iam_config(&self, path: impl AsRef<str> + Send) -> Result<()>;
async fn save_user_identity(&self, name: &str, user_type: UserType, item: UserIdentity, ttl: Option<usize>) -> Result<()>;
/// Create a regular identity or replace one without changing its persisted status.
/// Return whether a previous secret was replaced. Backends without conditional writes fail closed.
async fn save_regular_user_preserving_status(&self, _name: &str, _item: &UserIdentity) -> Result<bool> {
Err(crate::error::Error::IAMActionNotAllowed)
}
async fn delete_user_identity(&self, name: &str, user_type: UserType) -> Result<()>;
async fn load_user_identity(&self, name: &str, user_type: UserType) -> Result<UserIdentity>;
@@ -54,12 +41,6 @@ pub trait Store: Clone + Send + Sync + 'static {
async fn load_users(&self, user_type: UserType, m: &mut HashMap<String, UserIdentity>) -> Result<()>;
async fn load_secret_key(&self, name: &str, user_type: UserType) -> Result<String>;
/// Serialize edits to this group across all writers sharing the backend, then read its
/// durable state. Only confirmed absence yields `None`. Keep the returned guard through
/// save/delete and local cache publication; dropping it releases the transaction lock.
async fn begin_group_edit(&self, _name: &str) -> Result<(Box<dyn GroupEdit>, Option<GroupInfo>)> {
Err(crate::error::Error::other("backend does not support serialized group edits"))
}
async fn save_group_info(&self, name: &str, item: GroupInfo) -> Result<()>;
async fn delete_group_info(&self, name: &str) -> Result<()>;
async fn load_group(&self, name: &str, m: &mut HashMap<String, GroupInfo>) -> Result<()>;
+3 -196
View File
@@ -12,9 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::{GroupEdit, GroupInfo, MappedPolicy, Store, UserType};
use super::{GroupInfo, MappedPolicy, Store, UserType};
use crate::error::{Error, Result, is_err_config_not_found, is_err_no_such_group};
use crate::storage_api::object_store::WriteCommitGuard;
use crate::{
IAM_CONFIG_ROOT_PREFIX, IamStorageError, IamStore, classify_iam_system_path_failure_reason, delete_iam_config,
read_iam_config_no_lock, read_iam_config_with_metadata, save_iam_config, save_iam_config_with_opts,
@@ -42,8 +41,8 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, error, warn};
use crate::storage_api::object_store::{
HTTPPreconditions, ListOperations, NamespaceLocking, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
get_lock_acquire_timeout, read_config_preserve_empty_with_metadata_opts,
HTTPPreconditions, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
read_config_preserve_empty_with_metadata_opts,
};
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{IAM_CONFIG_ROOT_PREFIX}/iam"));
@@ -86,29 +85,6 @@ fn get_group_info_path(group: &str) -> String {
path_join_buf(&[&IAM_CONFIG_GROUPS_PREFIX, group, IAM_GROUP_MEMBERS_FILE])
}
fn unchanged_status_precondition(
previous: &UserIdentity,
item: &UserIdentity,
etag: Option<String>,
) -> Result<(HTTPPreconditions, bool)> {
if previous.credentials.status != item.credentials.status
|| previous.credentials.is_temp()
|| previous.credentials.is_service_account()
{
return Err(Error::IAMActionNotAllowed);
}
let tag = etag
.filter(|tag| !tag.is_empty() && tag != "*")
.ok_or_else(|| Error::other("missing IAM identity revision"))?;
Ok((
HTTPPreconditions {
if_match: Some(tag),
..Default::default()
},
previous.credentials.secret_key != item.credentials.secret_key,
))
}
fn get_policy_doc_path(name: &str) -> String {
path_join_buf(&[&IAM_CONFIG_POLICIES_PREFIX, name, IAM_POLICY_FILE])
}
@@ -232,53 +208,6 @@ pub struct ObjectStore {
object_api: Arc<IamStore>,
}
struct ObjectGroupEdit {
api: Arc<IamStore>,
path: String,
policy_path: String,
guard: WriteCommitGuard,
}
impl ObjectGroupEdit {
fn fenced(&self, mut opts: IamObjectOptions) -> Result<IamObjectOptions> {
if self.guard.is_lock_lost() {
return Err(Error::other("group transaction lock lost"));
}
if let Some(signal) = self.guard.lock_lost_signal() {
opts.add_namespace_lock_lost_signal(signal);
}
Ok(opts)
}
}
#[async_trait::async_trait]
impl GroupEdit for ObjectGroupEdit {
async fn save(&self, group: &GroupInfo) -> Result<()> {
let data = ObjectStore::prepare_data_for_storage(&serde_json::to_vec(group)?)?;
let opts = self.fenced(IamObjectOptions {
max_parity: true,
..Default::default()
})?;
save_iam_config_with_opts(self.api.clone(), &self.path, data, &opts).await?;
Ok(())
}
async fn delete_empty(&self) -> Result<()> {
for path in [&self.policy_path, &self.path] {
let opts = self.fenced(IamObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
..Default::default()
})?;
match self.api.delete_object(ObjectStore::BUCKET_NAME, path, opts).await {
Ok(_) | Err(IamStorageError::FileNotFound | IamStorageError::ObjectNotFound(_, _)) => (),
Err(err) => return Err(err.into()),
}
}
Ok(())
}
}
impl ObjectStore {
const BUCKET_NAME: &'static str = ".rustfs.sys";
@@ -968,34 +897,6 @@ impl Store for ObjectStore {
e
})
}
async fn save_regular_user_preserving_status(&self, name: &str, item: &UserIdentity) -> Result<bool> {
let path = get_user_identity_path(name, UserType::Reg);
let existing =
read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), &path, &LoadMode::Locked.read_opts()).await;
let (conditions, rotated) = match existing.map_err(Error::from) {
Ok((data, metadata)) => {
let previous: UserIdentity = Self::decode_iam_config_strict(&data)?;
unchanged_status_precondition(&previous, item, metadata.etag)?
}
Err(err) if is_err_config_not_found(&err) => (
HTTPPreconditions {
if_none_match: Some("*".into()),
..Default::default()
},
false,
),
Err(err) => return Err(err),
};
let opts = IamObjectOptions {
max_parity: true,
http_preconditions: Some(conditions),
..Default::default()
};
let data = Self::prepare_data_for_storage(&serde_json::to_vec(item)?)?;
save_iam_config_with_opts(self.object_api.clone(), &path, data, &opts).await?;
Ok(rotated)
}
async fn delete_user_identity(&self, name: &str, user_type: UserType) -> Result<()> {
self.delete_iam_config(get_user_identity_path(name, user_type))
.await
@@ -1052,28 +953,6 @@ impl Store for ObjectStore {
Ok(u.credentials.secret_key)
}
async fn begin_group_edit(&self, name: &str) -> Result<(Box<dyn GroupEdit>, Option<GroupInfo>)> {
let path = get_group_info_path(name);
// Take the transaction lock before any object locks. A distinct key lets ordinary
// config reads, writes and deletes keep taking their own namespace locks.
let lock = self.object_api.new_ns_lock(Self::BUCKET_NAME, &format!("{path}.txn")).await?;
let guard = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout())
.await
.map_err(Error::other)?;
let group = match self.load_iam_config_strict(&path).await {
Ok(gi) => Some(gi),
Err(Error::ConfigNotFound) => None,
Err(err) => return Err(err),
};
let edit = ObjectGroupEdit {
api: self.object_api.clone(),
path,
policy_path: get_mapped_policy_path(name, UserType::Reg, true),
guard,
};
Ok((Box::new(edit), group))
}
async fn save_group_info(&self, name: &str, item: GroupInfo) -> Result<()> {
self.save_iam_config(item, get_group_info_path(name)).await
}
@@ -1624,78 +1503,6 @@ mod tests {
assert!(matches!(err, Error::NoSuchUser(name) if name == "missing-sts"));
}
#[test]
fn restricted_user_upsert_requires_unchanged_status_and_exact_revision() {
let old = super::UserIdentity::from(Credentials {
status: "off".into(),
secret_key: "old-fixture".into(),
..Default::default()
});
let mut new = old.clone();
new.credentials.secret_key = "new-fixture".into();
let (predicate, rotated) =
super::unchanged_status_precondition(&old, &new, Some("rev1".into())).expect("status remains disabled");
assert!(rotated);
assert_eq!(predicate.if_match.as_deref(), Some("rev1"));
assert!(super::unchanged_status_precondition(&old, &new, None).is_err());
new.credentials.status = "on".into();
assert!(matches!(
super::unchanged_status_precondition(&old, &new, Some("rev1".into())),
Err(crate::error::Error::IAMActionNotAllowed)
));
}
#[tokio::test]
#[serial]
async fn restricted_user_upsert_checks_durable_status_before_replacing_credentials() {
use crate::store::{Store, UserType};
let _credentials = test_cred();
let temp = tempfile::tempdir().expect("IAM fixture directory");
let env = Box::pin(
rustfs_test_utils::TestECStoreEnv::builder()
.base_dir(temp.path())
.init_bucket_metadata(false)
.build(),
)
.await;
let store = ObjectStore::new(env.ecstore);
let name = "status-fixture";
let mut entry = super::UserIdentity::from(Credentials {
access_key: name.into(),
status: "off".into(),
secret_key: "first-fixture".into(),
..Default::default()
});
assert!(
!store
.save_regular_user_preserving_status(name, &entry)
.await
.expect("create absent account")
);
entry.credentials.status = "on".into();
entry.credentials.secret_key = "second-fixture".into();
assert!(matches!(
store.save_regular_user_preserving_status(name, &entry).await,
Err(Error::IAMActionNotAllowed)
));
let persisted = store
.load_user_identity(name, UserType::Reg)
.await
.expect("reload denied write");
assert_eq!(persisted.credentials.status, "off");
assert_eq!(persisted.credentials.secret_key, "first-fixture");
entry.credentials.status = "off".into();
assert!(
store
.save_regular_user_preserving_status(name, &entry)
.await
.expect("rotate while disabled")
);
let persisted = store.load_user_identity(name, UserType::Reg).await.expect("reload rotation");
assert_eq!(persisted.credentials.status, "off");
assert_eq!(persisted.credentials.secret_key, "second-fixture");
}
fn test_cred() -> Credentials {
if let Some(cred) = crate::root_credentials::credentials() {
return cred;
+65 -425
View File
@@ -76,8 +76,6 @@ pub const STATUS_ENABLED: &str = "enabled";
pub const STATUS_DISABLED: &str = "disabled";
pub const POLICYNAME: &str = "policy";
/// Signed subset of the OIDC policy selection granted independently of IAM mappings.
pub const OIDC_EXPLICIT_POLICIES_CLAIM: &str = "x-rustfs-internal-oidc-explicit-policies";
pub const PLUGIN_ATTEMPT_CREDENTIAL_CLAIM: &str = "rustfs:plugin-attempt-v1";
pub const SESSION_POLICY_NAME: &str = "sessionPolicy";
pub const SESSION_POLICY_NAME_EXTRACTED: &str = "sessionPolicy-extracted";
@@ -1297,22 +1295,6 @@ impl<T: Store> IamSys<T> {
access_key: &str,
args: &AddOrUpdateUserReq,
updated_at: OffsetDateTime,
) -> Result<OffsetDateTime> {
self.upsert_regular_user(access_key, args, updated_at, true).await
}
/// Create a regular user, or update its credentials without changing the stored account status.
pub async fn create_user_preserving_status(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result<OffsetDateTime> {
self.upsert_regular_user(access_key, args, OffsetDateTime::now_utc(), false)
.await
}
async fn upsert_regular_user(
&self,
access_key: &str,
args: &AddOrUpdateUserReq,
updated_at: OffsetDateTime,
allow_status_change: bool,
) -> Result<OffsetDateTime> {
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
let iam = self.clone();
@@ -1320,10 +1302,7 @@ impl<T: Store> IamSys<T> {
let args = args.clone();
#[cfg(test)]
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
let operation = async move {
iam.create_user_at_inner(&access_key, &args, updated_at, allow_status_change)
.await
};
let operation = async move { iam.create_user_at_inner(&access_key, &args, updated_at).await };
#[cfg(test)]
let task = runtime.spawn(async move {
if let Some(probe) = notification_probe {
@@ -1341,7 +1320,6 @@ impl<T: Store> IamSys<T> {
access_key: &str,
args: &AddOrUpdateUserReq,
updated_at: OffsetDateTime,
allow_status_change: bool,
) -> Result<OffsetDateTime> {
if !is_access_key_valid(access_key) {
return Err(IamError::InvalidAccessKeyLength);
@@ -1357,7 +1335,7 @@ impl<T: Store> IamSys<T> {
let (updated_at, secret_changed, parent_guard) = self
.store
.add_user_at_with_secret_change(access_key, args, updated_at, allow_status_change)
.add_user_at_with_secret_change(access_key, args, updated_at)
.await?;
if secret_changed {
self.revoke_sts_sessions_for_parent(access_key).await?;
@@ -1692,10 +1670,16 @@ impl<T: Store> IamSys<T> {
parent_user: &str,
groups: &Option<Vec<String>>,
) -> Result<Option<HashMap<String, Value>>> {
let Some((resolved, _)) = self.verified_federated_policy(claims, parent_user, groups).await? else {
let Some(policy_names) = self.verified_federated_policy_names(claims, parent_user, groups).await? else {
return Ok(None);
};
let (resolved, _) = self.store.merge_policies(&policy_names.join(",")).await;
let resolved_names = MappedPolicy::new(&resolved).to_slice();
if policy_names.iter().any(|policy_name| !resolved_names.contains(policy_name)) {
return Err(IamError::InvalidArgument);
}
let boundary = match claims.get(SESSION_POLICY_NAME) {
Some(Value::String(encoded)) => {
decode_session_policy(encoded)?;
@@ -1712,9 +1696,6 @@ impl<T: Store> IamSys<T> {
if claims.get(OIDC_VIRTUAL_PARENT_CLAIM).and_then(Value::as_str) == Some(parent_user) {
verified_claims.insert(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), Value::String(parent_user.to_string()));
}
if let Some(explicit) = claims.get(OIDC_EXPLICIT_POLICIES_CLAIM) {
verified_claims.insert(OIDC_EXPLICIT_POLICIES_CLAIM.to_owned(), explicit.clone());
}
if let Some(boundary) = boundary {
verified_claims.insert(FEDERATED_POLICY_BOUNDARY_CLAIM.to_string(), Value::String(boundary));
}
@@ -1896,24 +1877,20 @@ impl<T: Store> IamSys<T> {
Ok(policy_names)
}
async fn resolve_oidc_policy(&self, names: &[String]) -> Result<(String, Policy)> {
if names.is_empty() || names.iter().any(|name| !is_safe_claim_policy_name(name)) {
fn signed_oidc_policy_names(claims: &HashMap<String, Value>, parent_user: &str) -> Result<Vec<String>> {
let policy_names = Self::safe_claim_policy_names(claims, parent_user);
if policy_names.is_empty() {
return Err(IamError::InvalidArgument);
}
let (resolved, policy) = self.store.merge_policies(&names.join(",")).await;
let resolved_names = MappedPolicy::new(&resolved).to_slice();
if names.iter().any(|name| !resolved_names.contains(name)) {
return Err(IamError::InvalidArgument);
}
Ok((resolved, policy))
Ok(policy_names)
}
async fn verified_federated_policy(
async fn verified_federated_policy_names(
&self,
claims: &HashMap<String, Value>,
parent_user: &str,
groups: &Option<Vec<String>>,
) -> Result<Option<(String, Policy)>> {
) -> Result<Option<Vec<String>>> {
if !is_rustfs_oidc_claims(claims) {
return Ok(None);
}
@@ -1921,72 +1898,12 @@ impl<T: Store> IamSys<T> {
return Err(IamError::InvalidArgument);
}
let explicit = claims.get(OIDC_EXPLICIT_POLICIES_CLAIM);
if explicit.is_none() {
// Earlier grouped credentials cannot distinguish an explicit Deny from a detached grant.
if claims.contains_key("groups") || groups.as_ref().is_some_and(|groups| !groups.is_empty()) {
return Err(IamError::InvalidArgument);
}
if !is_verified_federated_session(claims, parent_user) && !has_verified_federated_policy(claims) {
let names = self.oidc_policy_names(claims, parent_user, groups).await?;
return self.resolve_oidc_policy(&names).await.map(Some);
}
}
let issued: Vec<String> = claims
.get(POLICYNAME)
.and_then(Value::as_str)
.ok_or(IamError::InvalidArgument)?
.split(',')
.map(str::trim)
.map(str::to_owned)
.collect();
if issued.iter().any(|name| !is_safe_claim_policy_name(name)) {
return Err(IamError::InvalidArgument);
}
let Some(explicit) = explicit else {
return self.resolve_oidc_policy(&issued).await.map(Some);
let policy_names = if is_verified_federated_session(claims, parent_user) {
Self::signed_oidc_policy_names(claims, parent_user)?
} else {
self.oidc_policy_names(claims, parent_user, groups).await?
};
if !is_verified_federated_session(claims, parent_user) {
return Err(IamError::InvalidArgument);
}
let explicit: HashSet<&str> = explicit
.as_array()
.ok_or(IamError::InvalidArgument)?
.iter()
.map(|v| {
v.as_str()
.filter(|name| issued.iter().any(|n| n == name))
.ok_or(IamError::InvalidArgument)
})
.collect::<Result<_>>()?;
// Use the issuer's signed group list, including on replicated and derived credentials.
let signed_groups: Option<Vec<String>> = claims
.get("groups")
.map(serde::Deserialize::deserialize)
.transpose()
.map_err(|_| IamError::InvalidArgument)?;
if signed_groups.is_none() && groups.as_ref().is_some_and(|groups| !groups.is_empty()) {
return Err(IamError::InvalidArgument);
}
let current = self.store.sts_policy_db_get(parent_user, &signed_groups).await?;
let active: Vec<_> = issued
.iter()
.filter(|name| explicit.contains(name.as_str()) || current.contains(name))
.cloned()
.collect();
let (resolved, mut policy) = self.resolve_oidc_policy(&active).await?;
// Later attachments can restrict the credential, but cannot widen its issuance ceiling.
let new: Vec<_> = current.into_iter().filter(|name| !issued.contains(name)).collect();
if !new.is_empty() {
let (_, restrictions) = self.resolve_oidc_policy(&new).await?;
policy.statements.extend(
restrictions
.statements
.into_iter()
.filter(|s| s.effect == rustfs_policy::policy::Effect::Deny),
);
}
Ok(Some((resolved, policy)))
Ok(Some(policy_names))
}
/// Compatibility wrapper for service-account authorization entry points.
@@ -2016,9 +1933,6 @@ impl<T: Store> IamSys<T> {
let state = Self::policy_plugin_state().await;
if marked {
if args.is_owner
// Plugin-attempt sessions are confined to their allowlisted work;
// they cannot use a default permission to manage the parent identity.
|| self_service_deny_check(args)
|| args.claims.get(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM) != Some(&Value::Bool(true))
|| !matches!(state, PolicyPluginState::Disabled)
{
@@ -2167,10 +2081,10 @@ impl<T: Store> IamSys<T> {
session_policy,
} => {
let session_ok = evaluate_prepared_session_policy(session_policy, args).await;
if session_ok == Some(false) && is_rustfs_oidc_claims(args.claims) {
return Err(Error::IAMActionNotAllowed);
if let Some(ok) = session_ok {
return Ok(ok && (*is_owner || combined_policy.is_allowed(args).await));
}
session_ok != Some(false) && (*is_owner || derived_policy_allowed(combined_policy, args).await?)
*is_owner || combined_policy.is_allowed(args).await
}
PreparedIamMode::ServiceAccount {
is_owner,
@@ -2191,18 +2105,12 @@ impl<T: Store> IamSys<T> {
true
};
if !boundary_allowed {
return Err(Error::IAMActionNotAllowed);
}
let parent_allowed =
*bypass_parent_policy || *is_owner || derived_policy_allowed(combined_policy, &parent_args).await?;
(*bypass_parent_policy || *is_owner || combined_policy.is_allowed(&parent_args).await) && boundary_allowed;
match mode {
PreparedServicePolicyMode::Inherited => parent_allowed,
PreparedServicePolicyMode::SessionBound => {
let session_ok = evaluate_prepared_session_policy(session_policy, args).await;
if session_ok == Some(false) && is_rustfs_oidc_claims(args.claims) {
return Err(Error::IAMActionNotAllowed);
}
if let Some(ok) = session_ok {
return Ok(ok && parent_allowed);
}
@@ -2222,24 +2130,9 @@ impl<T: Store> IamSys<T> {
};
if policies.is_empty() {
// The policy lookup also returns an empty set for an inactive user.
// Only a live identity gets the default for its own credential changes.
let mode = if self_service_deny_check(args)
&& self
.store
.get_user(args.account)
.await
.is_some_and(|u| u.credentials.is_valid())
{
PreparedIamMode::Regular {
combined_policy: Policy::default(),
}
} else {
PreparedIamMode::Deny
};
return PreparedIamAuth {
needs_existing_object_tag: false,
mode,
mode: PreparedIamMode::Deny,
};
}
@@ -2266,19 +2159,24 @@ impl<T: Store> IamSys<T> {
}
pub(crate) async fn prepare_sts_auth(&self, args: &Args<'_>, parent_user: &str) -> PreparedIamAuth {
let federated_policy = match self.verified_federated_policy(args.claims, parent_user, args.groups).await {
Ok(policy) => policy,
let federated_policy_names = match self
.verified_federated_policy_names(args.claims, parent_user, args.groups)
.await
{
Ok(policy_names) => policy_names,
Err(_) => {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::TerminalDeny,
mode: PreparedIamMode::Deny,
};
}
};
let federated_session = federated_policy_names.is_some();
let verified_federated_session = is_verified_federated_session(args.claims, parent_user);
let is_owner = !is_rustfs_oidc_claims(args.claims) && crate::is_root_access_key(parent_user);
let role_arn = args.get_role_arn();
let (effective_groups, groups_source, mut policy_names) = if federated_policy.is_some() {
let (effective_groups, groups_source, policies) = if federated_session {
(args.groups.clone(), "oidc_claims", Vec::new())
} else if is_owner {
(None, "owner", Vec::new())
@@ -2310,27 +2208,21 @@ impl<T: Store> IamSys<T> {
}
},
};
let p = match self.policy_db_get(parent_user, &effective_groups).await {
Ok(policies) => policies,
Err(_) if self_service_deny_check(args) => {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::Deny,
};
}
Err(_) => Vec::new(),
};
let p = self.policy_db_get(parent_user, &effective_groups).await.unwrap_or_default();
(effective_groups, groups_source, p)
};
let mut policy_names = if let Some(policy_names) = federated_policy_names {
policy_names
} else {
policies
};
if !is_owner && policy_names.is_empty() && !is_rustfs_oidc_claims(args.claims) {
policy_names = Self::safe_claim_policy_names(args.claims, parent_user);
}
let mut combined_policy = Policy::default();
if let Some((_, policy)) = federated_policy {
combined_policy = policy;
} else if !is_owner {
if !is_owner {
if policy_names.is_empty() {
if args.deny_only {
combined_policy = Policy::default();
@@ -2349,7 +2241,7 @@ impl<T: Store> IamSys<T> {
requested_policies = %requested_policies,
"prepare_sts_auth: no STS policy names resolved"
);
if args.deny_only && !self_service_deny_check(args) {
if args.deny_only && !verified_federated_session {
combined_policy = Policy::default();
} else {
return PreparedIamAuth {
@@ -2363,7 +2255,7 @@ impl<T: Store> IamSys<T> {
.iter()
.any(|policy_name| !resolved_policy_names.iter().any(|resolved| resolved == policy_name));
if has_unresolved_policy_names {
if self_service_deny_check(args) {
if verified_federated_session {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::Deny,
@@ -2440,14 +2332,14 @@ impl<T: Store> IamSys<T> {
if is_rustfs_oidc_claims(args.claims) && !has_verified_federated_policy {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::TerminalDeny,
mode: PreparedIamMode::Deny,
};
}
let federated_boundary = if has_verified_federated_policy {
if !is_valid_verified_federated_policy(args.claims, parent_user) {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::TerminalDeny,
mode: PreparedIamMode::Deny,
};
}
match federated_policy_boundary(args.claims) {
@@ -2455,7 +2347,7 @@ impl<T: Store> IamSys<T> {
Err(_) => {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::TerminalDeny,
mode: PreparedIamMode::Deny,
};
}
}
@@ -2474,8 +2366,10 @@ impl<T: Store> IamSys<T> {
let is_owner = !is_rustfs_oidc_claims(args.claims) && crate::is_root_access_key(parent_user);
let role_arn = args.get_role_arn();
let svc_policies = if is_owner || bypass_parent_policy || has_verified_federated_policy {
let svc_policies = if is_owner || bypass_parent_policy {
Vec::new()
} else if has_verified_federated_policy {
Self::safe_claim_policy_names(args.claims, parent_user)
} else if role_arn.is_some() {
let Ok(arn) = ARN::parse(role_arn.unwrap_or_default()) else {
tracing::warn!(
@@ -2499,7 +2393,7 @@ impl<T: Store> IamSys<T> {
policies
};
if !is_owner && !bypass_parent_policy && !has_verified_federated_policy && svc_policies.is_empty() {
if !is_owner && !bypass_parent_policy && svc_policies.is_empty() {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::Deny,
@@ -2508,17 +2402,12 @@ impl<T: Store> IamSys<T> {
let combined_policy = if is_owner || bypass_parent_policy {
Policy::default()
} else if has_verified_federated_policy {
let Ok(Some((_, policy))) = self.verified_federated_policy(args.claims, parent_user, args.groups).await else {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::TerminalDeny,
};
};
policy
} else {
let (a, c) = self.store.merge_policies(&svc_policies.join(",")).await;
if a.is_empty() {
let resolved = MappedPolicy::new(&a).to_slice();
if a.is_empty()
|| has_verified_federated_policy && svc_policies.iter().any(|policy_name| !resolved.contains(policy_name))
{
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::Deny,
@@ -2589,26 +2478,8 @@ async fn prepared_session_policy_needs_existing_object_tag_for_args(policy: &Pre
}
}
fn self_service_deny_check(args: &Args<'_>) -> bool {
use rustfs_policy::policy::action::{Action, AdminAction};
args.deny_only
&& matches!(
args.action,
Action::AdminAction(
AdminAction::ChangeOwnPasswordAdminAction
| AdminAction::CreateUserAdminAction
| AdminAction::ManageOwnMFAAdminAction
)
)
}
fn prepare_session_policy(args: &Args<'_>, empty_is_none: bool) -> PreparedSessionPolicy {
let Some(policy_str) = extract_session_policy_text(args.claims) else {
if self_service_deny_check(args)
&& (args.claims.contains_key(SESSION_POLICY_NAME) || args.claims.contains_key(SESSION_POLICY_NAME_EXTRACTED))
{
return PreparedSessionPolicy::DenyAll;
}
return PreparedSessionPolicy::None;
};
@@ -2681,7 +2552,6 @@ pub fn remove_verified_federated_policy(claims: &mut HashMap<String, Value>) ->
let removed = claims.remove(VERIFIED_FEDERATED_POLICY_CLAIM).is_some();
claims.remove(FEDERATED_POLICY_BOUNDARY_CLAIM);
claims.remove(OIDC_VIRTUAL_PARENT_CLAIM);
claims.remove(OIDC_EXPLICIT_POLICIES_CLAIM);
removed
}
@@ -2690,10 +2560,6 @@ fn merge_derived_service_account_claims(
source_claims: &HashMap<String, Value>,
verified_federated_policy: Option<HashMap<String, Value>>,
) -> Result<()> {
if verified_federated_policy.is_some() {
// Only the parent's signed groups may participate in federated policy resolution.
target_claims.remove("groups");
}
for (key, value) in source_claims {
if key != "exp" {
target_claims.insert(key.clone(), value.clone());
@@ -2760,24 +2626,6 @@ fn extract_session_policy_text(claims: &HashMap<String, Value>) -> Option<String
String::from_utf8(bytes).ok()
}
async fn derived_policy_allowed(policy: &Policy, args: &Args<'_>) -> Result<bool> {
let allowed = policy.is_allowed(args).await;
// A bucket policy may supplement a missing Allow, but not override a live OIDC Deny.
if !allowed
&& is_rustfs_oidc_claims(args.claims)
&& (args.deny_only
|| !policy
.is_allowed(&Args {
deny_only: true,
..args.clone()
})
.await)
{
return Err(Error::IAMActionNotAllowed);
}
Ok(allowed)
}
async fn evaluate_prepared_session_policy(policy: &PreparedSessionPolicy, args: &Args<'_>) -> Option<bool> {
match policy {
PreparedSessionPolicy::None => None,
@@ -2864,7 +2712,7 @@ mod tests {
use crate::cache::{Cache, CacheEntity};
use crate::error::Error;
use crate::manager::get_default_policies;
use crate::store::{GroupEdit, GroupInfo, MappedPolicy, Store, UserType};
use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
use rustfs_credentials::{Credentials, init_global_action_credentials};
use rustfs_policy::auth::{UserIdentity, get_new_credentials_with_metadata};
use rustfs_policy::policy::Args;
@@ -3600,30 +3448,10 @@ mod tests {
derived_save_started: Arc<tokio::sync::Notify>,
release_derived_save: Arc<tokio::sync::Notify>,
block_group_save: Arc<std::sync::atomic::AtomicBool>,
group_locks: Arc<Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>>,
saved_groups: Arc<Mutex<HashMap<String, GroupInfo>>>,
group_save_started: Arc<tokio::sync::Notify>,
group_save_release: Arc<tokio::sync::Notify>,
}
struct MockGroupEdit {
store: StsTestMockStore,
name: String,
_guard: tokio::sync::OwnedMutexGuard<()>,
}
#[async_trait::async_trait]
impl GroupEdit for MockGroupEdit {
async fn save(&self, group: &GroupInfo) -> Result<()> {
self.store.save_group_info(&self.name, group.clone()).await
}
async fn delete_empty(&self) -> Result<()> {
self.store.delete_mapped_policy(&self.name, UserType::Reg, true).await?;
self.store.delete_group_info(&self.name).await
}
}
impl StsTestMockStore {
fn new(empty_policies: bool) -> Self {
Self {
@@ -3644,12 +3472,6 @@ mod tests {
derived_save_started: Arc::new(tokio::sync::Notify::new()),
release_derived_save: Arc::new(tokio::sync::Notify::new()),
block_group_save: Arc::new(std::sync::atomic::AtomicBool::new(false)),
group_locks: Default::default(),
saved_groups: Arc::new(Mutex::new(if empty_policies {
HashMap::new()
} else {
HashMap::from([("testgroup".to_owned(), GroupInfo::new(vec!["sts-fallback-test-parent".to_owned()]))])
})),
group_save_started: Arc::new(tokio::sync::Notify::new()),
group_save_release: Arc::new(tokio::sync::Notify::new()),
}
@@ -3792,36 +3614,15 @@ mod tests {
Err(Error::InvalidArgument)
}
async fn begin_group_edit(&self, name: &str) -> Result<(Box<dyn GroupEdit>, Option<GroupInfo>)> {
let lock = self
.group_locks
.lock()
.expect("group locks")
.entry(name.to_owned())
.or_default()
.clone();
let guard = lock.lock_owned().await;
Ok((
Box::new(MockGroupEdit {
store: self.clone(),
name: name.to_owned(),
_guard: guard,
}),
self.saved_groups.lock().expect("saved groups").get(name).cloned(),
))
}
async fn save_group_info(&self, name: &str, item: GroupInfo) -> Result<()> {
async fn save_group_info(&self, _name: &str, _item: GroupInfo) -> Result<()> {
if self.block_group_save.load(std::sync::atomic::Ordering::SeqCst) {
self.group_save_started.notify_one();
self.group_save_release.notified().await;
}
self.saved_groups.lock().expect("saved groups").insert(name.to_owned(), item);
Ok(())
}
async fn delete_group_info(&self, name: &str) -> Result<()> {
self.saved_groups.lock().expect("saved groups").remove(name);
async fn delete_group_info(&self, _name: &str) -> Result<()> {
Ok(())
}
@@ -3874,14 +3675,11 @@ mod tests {
Err(Error::InvalidArgument)
}
async fn delete_mapped_policy(&self, name: &str, user_type: UserType, is_group: bool) -> Result<()> {
async fn delete_mapped_policy(&self, name: &str, user_type: UserType, _is_group: bool) -> Result<()> {
self.deleted_mapped_policies
.lock()
.expect("deleted_mapped_policies mutex poisoned")
.push((name.to_string(), user_type));
if is_group {
return Ok(());
}
Err(Error::InvalidArgument)
}
@@ -3990,7 +3788,9 @@ mod tests {
users.insert(PARENT_USER.to_string(), parent_identity);
users.extend(saved_users);
let groups = self.saved_groups.lock().expect("saved groups").clone();
let group = GroupInfo::new(vec![PARENT_USER.to_string()]);
let mut groups = HashMap::new();
groups.insert(GROUP_NAME.to_string(), group);
let group_policy = MappedPolicy::new("readwrite");
let mut group_policies = HashMap::new();
@@ -4023,82 +3823,6 @@ mod tests {
IamSys::new(cache)
}
async fn shared_group_caches() -> (StsTestMockStore, Arc<IamCache<StsTestMockStore>>, Arc<IamCache<StsTestMockStore>>) {
let store = StsTestMockStore::new(false);
let (first, second) = temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], async {
(
IamCache::new(store.clone()).await.expect("first cache"),
IamCache::new(store.clone()).await.expect("second cache"),
)
})
.await;
(store, first, second)
}
#[tokio::test]
#[serial]
async fn group_edit_status_and_removal_survive_shared_backend_reload() {
let (store, first, second) = shared_group_caches().await;
let group = "testgroup";
let member = "sts-fallback-test-parent";
second
.add_users_to_group("unrelated", vec![member.to_owned()])
.await
.expect("seed unrelated membership");
store.block_group_save.store(true, std::sync::atomic::Ordering::SeqCst);
let mut removal = Box::pin(first.remove_users_from_group(group, vec![member.to_owned()]));
assert!(futures::poll!(&mut removal).is_pending());
store.block_group_save.store(false, std::sync::atomic::Ordering::SeqCst);
let mut status = Box::pin(second.set_group_status(group, false));
assert!(futures::poll!(&mut status).is_pending(), "status must wait for the other writer");
store.group_save_release.notify_one();
removal.await.expect("commit removal");
status.await.expect("commit status");
store.load_all(&first.cache).await.expect("reload durable groups");
for node in [&first, &second] {
let state = node.cache.snapshot();
assert!(state.groups[group].members.is_empty());
assert_eq!(state.groups[group].status, "disabled");
assert_eq!(state.user_group_memberships[member], HashSet::from(["unrelated".to_owned()]));
}
}
#[tokio::test]
#[serial]
async fn group_edit_delete_orders_emptiness_check_with_add() {
let (store, first, second) = shared_group_caches().await;
let group = "testgroup";
let member = "sts-fallback-test-parent";
second
.remove_users_from_group(group, vec![member.to_owned()])
.await
.expect("empty group");
store.block_group_save.store(true, std::sync::atomic::Ordering::SeqCst);
let mut add = Box::pin(first.add_users_to_group(group, vec![member.to_owned()]));
assert!(futures::poll!(&mut add).is_pending());
let mut delete = Box::pin(second.remove_users_from_group(group, vec![]));
assert!(futures::poll!(&mut delete).is_pending(), "delete must wait before checking emptiness");
store.group_save_release.notify_one();
add.await.expect("restore membership");
assert!(matches!(delete.await, Err(Error::GroupNotEmpty)));
assert!(store.deleted_mapped_policies.lock().expect("policy deletions").is_empty());
store.block_group_save.store(false, std::sync::atomic::Ordering::SeqCst);
second.set_group_status(group, false).await.expect("refresh from peer add");
assert!(second.cache.snapshot().user_group_memberships[member].contains(group));
first
.remove_users_from_group(group, vec![member.to_owned()])
.await
.expect("empty group");
second
.remove_users_from_group(group, vec![])
.await
.expect("delete empty group");
assert!(matches!(first.set_group_status(group, true).await, Err(Error::NoSuchGroup(_))));
assert!(!second.cache.snapshot().user_group_memberships.contains_key(member));
assert!(!store.saved_groups.lock().expect("durable groups").contains_key(group));
}
async fn assert_group_write_during_reload_is_published(remove: bool) {
let iam_sys = Arc::new(temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], test_iam_sys()).await);
let member = "sts-fallback-test-parent";
@@ -6058,15 +5782,13 @@ mod tests {
("sub".to_string(), Value::String("subject-123".to_string())),
("parent".to_string(), Value::String(parent_user.to_string())),
(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), Value::String(parent_user.to_string())),
(POLICYNAME.to_string(), Value::String("readwrite,readonly".to_string())),
(OIDC_EXPLICIT_POLICIES_CLAIM.to_string(), serde_json::json!(["readonly"])),
("groups".to_string(), serde_json::json!(["testgroup"])),
(POLICYNAME.to_string(), Value::String("readwrite".to_string())),
]);
let conditions = HashMap::new();
let sts_args = Args {
account: "OIDCGROUPCALLER01",
groups: &groups,
action: Action::S3Action(S3Action::PutObjectAction),
action: Action::S3Action(S3Action::GetObjectAction),
bucket: "bucket",
conditions: &conditions,
is_owner: false,
@@ -6101,10 +5823,7 @@ mod tests {
.await
.expect("group-authorized OIDC caller should create a service account");
assert_eq!(replication_claims.get(VERIFIED_FEDERATED_POLICY_CLAIM), Some(&Value::Bool(true)));
assert_eq!(
replication_claims.get(OIDC_EXPLICIT_POLICIES_CLAIM),
claims.get(OIDC_EXPLICIT_POLICIES_CLAIM)
);
assert_eq!(replication_claims.get(POLICYNAME), Some(&Value::String("readwrite".to_string())));
let service_claims = iam_sys
.get_claims_for_svc_acc(&service_account.access_key)
@@ -6120,83 +5839,6 @@ mod tests {
iam_sys.is_allowed(&service_args).await,
"OIDC service account must resolve the policy names selected at STS issuance"
);
let now = OffsetDateTime::now_utc();
let deny_read = Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::bucket/*"}]}"#).expect("parse Deny");
iam_sys
.store
.cache
.add_or_update_policy_doc("deny-read", &PolicyDoc::new(deny_read), now);
for (mapping, enabled, read, write) in [
("", true, true, false),
("readwrite", false, true, false),
("readwrite,deny-read", true, false, true),
("readwrite,missing-deny", true, false, false),
] {
if mapping.is_empty() {
iam_sys.store.cache.delete_group_policy("testgroup", now);
} else {
iam_sys
.store
.cache
.add_or_update_group_policy("testgroup", &MappedPolicy::new(mapping), now);
}
let mut group = GroupInfo::new(Vec::new());
if !enabled {
group.status = STATUS_DISABLED.to_owned();
}
iam_sys.store.cache.add_or_update_group("testgroup", &group, now);
for (action, expected) in [(S3Action::GetObjectAction, read), (S3Action::PutObjectAction, write)] {
let sts = Args {
action: Action::S3Action(action),
..sts_args.clone()
};
let service = Args {
action: sts.action,
..service_args.clone()
};
let prepared = iam_sys.prepare_sts_auth(&sts, parent_user).await;
assert_eq!(
iam_sys.eval_prepared(&prepared, &sts).await,
expected,
"STS: {mapping}, enabled={enabled}"
);
assert_eq!(
iam_sys.is_allowed(&service).await,
expected,
"service account: {mapping}, enabled={enabled}"
);
if mapping == "readwrite,deny-read" && action == S3Action::GetObjectAction {
let child = iam_sys.prepare_service_account_auth(&service, parent_user).await;
for (prepared, args) in [(&prepared, &sts), (&child, &service)] {
assert!(
matches!(iam_sys.try_eval_prepared(prepared, args).await, Err(Error::IAMActionNotAllowed)),
"a newly mapped Deny must stop bucket policy fallback"
);
}
}
}
}
iam_sys.store.cache.delete_group_policy("testgroup", now);
for (source, account) in [(&claims, sts_args.account), (&service_claims, service_args.account)] {
let mut legacy = source.clone();
legacy.remove(OIDC_EXPLICIT_POLICIES_CLAIM);
let args = Args {
claims: &legacy,
account,
deny_only: true,
..sts_args.clone()
};
let prepared = if account == sts_args.account {
iam_sys.prepare_sts_auth(&args, parent_user).await
} else {
iam_sys.prepare_service_account_auth(&args, parent_user).await
};
assert!(
!iam_sys.eval_prepared(&prepared, &args).await,
"ambiguous legacy grouped credentials must be reissued"
);
}
}
#[tokio::test]
@@ -6224,8 +5866,6 @@ mod tests {
("parent".to_string(), Value::String(parent_user.to_string())),
(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), Value::String(parent_user.to_string())),
(POLICYNAME.to_string(), Value::String("readonly,deny-delete".to_string())),
(OIDC_EXPLICIT_POLICIES_CLAIM.to_string(), serde_json::json!(["readonly", "deny-delete"])),
("groups".to_string(), serde_json::json!(["testgroup"])),
]);
let groups = Some(vec!["testgroup".to_string()]);
let conditions = HashMap::new();
@@ -6430,7 +6070,7 @@ mod tests {
let prepared = iam_sys.prepare_sts_auth(&args, parent_user).await;
assert!(matches!(prepared.mode, PreparedIamMode::TerminalDeny));
assert!(matches!(prepared.mode, PreparedIamMode::Deny));
assert!(!iam_sys.eval_prepared(&prepared, &args).await);
}
+5
View File
@@ -33,18 +33,23 @@ default = []
diagnostic-internode-http2-ping = []
diagnostic-read-version-caller = []
hotpath = [
"hotpath/hotpath",
"hotpath/tokio",
"rustfs-s3-ops/hotpath",
]
hotpath-alloc = [
"hotpath",
"hotpath/hotpath-alloc",
"rustfs-s3-ops/hotpath-alloc",
]
hotpath-cpu = [
"hotpath",
"hotpath/hotpath-cpu",
"rustfs-s3-ops/hotpath-cpu",
]
[dependencies]
hotpath.workspace = true
metrics = { workspace = true }
rustfs-s3-ops = { workspace = true }
num_cpus = { workspace = true }
-362
View File
@@ -59,7 +59,6 @@ use std::sync::{
LazyLock, Mutex,
atomic::{AtomicBool, AtomicU64, Ordering},
};
use std::time::Duration;
/// Global switch for detailed per-stage PUT metrics (path label, stage durations).
/// When `false`, `record_put_object_path` and `record_put_object_stage_duration`
@@ -450,125 +449,6 @@ pub const PUT_RENAME_QUORUM_FANOUT_STATE_PANIC: &str = "panic";
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST: &str = "quorum_first";
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL: &str = "quorum_tail";
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR: &str = "error";
pub const ECSTORE_RENAME_RPC_KIND_ORDINARY: &str = "ordinary";
pub const ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE: &str = "quota_fence";
pub const ECSTORE_RENAME_RPC_KIND_INCARNATION: &str = "incarnation";
pub const ECSTORE_RENAME_RPC_KIND_MULTIPART: &str = "multipart";
pub const ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE: &str = "scanner_lease";
pub const ECSTORE_RENAME_RPC_KIND_POOL_META_CAS: &str = "pool_meta_cas";
pub const ECSTORE_RENAME_RPC_KINDS: [&str; 6] = [
ECSTORE_RENAME_RPC_KIND_ORDINARY,
ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE,
ECSTORE_RENAME_RPC_KIND_INCARNATION,
ECSTORE_RENAME_RPC_KIND_MULTIPART,
ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE,
ECSTORE_RENAME_RPC_KIND_POOL_META_CAS,
];
pub const ECSTORE_RENAME_PEER_WIDTHS: [&str; 18] = [
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "gt16",
];
pub const ECSTORE_RENAME_COVERAGE: [&str; 3] = ["none", "partial", "full"];
pub const ECSTORE_RENAME_BATCH_EXECUTION: [&str; 10] = [
"planned_groups",
"planned_items",
"stream_groups",
"stream_items",
"completed_items",
"fallback_groups",
"fallback_items",
"not_dispatched_items",
"may_have_published_items",
"terminal_missing_items",
];
pub const ECSTORE_RENAME_ADMISSION: [&str; 10] = [
"unary_attempts",
"unary_items",
"unary_wait_ns_total",
"unary_wait_ns_max",
"unary_timeouts",
"batch_attempts",
"batch_items",
"batch_wait_ns_total",
"batch_wait_ns_max",
"batch_timeouts",
];
pub const ECSTORE_RENAME_QUARANTINE: [&str; 12] = [
"unary_events",
"unary_items",
"unary_hold_ns_total",
"unary_hold_ns_max",
"unary_released_items",
"unary_no_runtime_items",
"batch_events",
"batch_items",
"batch_hold_ns_total",
"batch_hold_ns_max",
"batch_released_items",
"batch_no_runtime_items",
];
pub const ECSTORE_RENAME_BATCH_TIMING: [&str; 4] = [
"client_stream_to_first",
"client_first_to_last",
"server_dispatch_to_first",
"server_first_to_last",
];
static ECSTORE_RENAME_RPC_COUNTS: [AtomicU64; 6] = [const { AtomicU64::new(0) }; 6];
static ECSTORE_RENAME_REMOTE_WIDTH_COUNTS: [AtomicU64; 18] = [const { AtomicU64::new(0) }; 18];
static ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS: [AtomicU64; 18] = [const { AtomicU64::new(0) }; 18];
static ECSTORE_RENAME_COVERAGE_COUNTS: [AtomicU64; 3] = [const { AtomicU64::new(0) }; 3];
static ECSTORE_RENAME_BATCH_EXECUTION_COUNTS: [AtomicU64; 10] = [const { AtomicU64::new(0) }; 10];
static ECSTORE_RENAME_ADMISSION_COUNTS: [AtomicU64; 10] = [const { AtomicU64::new(0) }; 10];
static ECSTORE_RENAME_QUARANTINE_COUNTS: [AtomicU64; 12] = [const { AtomicU64::new(0) }; 12];
static ECSTORE_RENAME_BATCH_TIMING_COUNTS: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4];
static ECSTORE_RENAME_BATCH_TIMING_TOTAL_NS: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4];
static ECSTORE_RENAME_BATCH_TIMING_MAX_NS: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4];
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct EcstoreRenameDiagnosticsSnapshot {
pub rpc_counts: [u64; 6],
pub remote_width_counts: [u64; 18],
pub batchable_width_counts: [u64; 18],
pub coverage_counts: [u64; 3],
pub batch_execution_counts: [u64; 10],
pub admission_counts: [u64; 10],
pub quarantine_counts: [u64; 12],
pub batch_timing_counts: [u64; 4],
pub batch_timing_total_ns: [u64; 4],
pub batch_timing_max_ns: [u64; 4],
}
impl EcstoreRenameDiagnosticsSnapshot {
pub fn is_empty(&self) -> bool {
self.rpc_counts
.iter()
.chain(self.remote_width_counts.iter())
.chain(self.batchable_width_counts.iter())
.chain(self.coverage_counts.iter())
.chain(self.batch_execution_counts.iter())
.chain(self.admission_counts.iter())
.chain(self.quarantine_counts.iter())
.chain(self.batch_timing_counts.iter())
.chain(self.batch_timing_total_ns.iter())
.chain(self.batch_timing_max_ns.iter())
.all(|count| *count == 0)
}
}
pub fn snapshot_ecstore_rename_diagnostics() -> EcstoreRenameDiagnosticsSnapshot {
EcstoreRenameDiagnosticsSnapshot {
rpc_counts: std::array::from_fn(|index| ECSTORE_RENAME_RPC_COUNTS[index].load(Ordering::Relaxed)),
remote_width_counts: std::array::from_fn(|index| ECSTORE_RENAME_REMOTE_WIDTH_COUNTS[index].load(Ordering::Relaxed)),
batchable_width_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS[index].load(Ordering::Relaxed)),
coverage_counts: std::array::from_fn(|index| ECSTORE_RENAME_COVERAGE_COUNTS[index].load(Ordering::Relaxed)),
batch_execution_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_EXECUTION_COUNTS[index].load(Ordering::Relaxed)),
admission_counts: std::array::from_fn(|index| ECSTORE_RENAME_ADMISSION_COUNTS[index].load(Ordering::Relaxed)),
quarantine_counts: std::array::from_fn(|index| ECSTORE_RENAME_QUARANTINE_COUNTS[index].load(Ordering::Relaxed)),
batch_timing_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_TIMING_COUNTS[index].load(Ordering::Relaxed)),
batch_timing_total_ns: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_TIMING_TOTAL_NS[index].load(Ordering::Relaxed)),
batch_timing_max_ns: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_TIMING_MAX_NS[index].load(Ordering::Relaxed)),
}
}
#[inline(always)]
pub fn get_stage_metrics_enabled() -> bool {
@@ -2783,169 +2663,6 @@ pub fn record_put_rename_quorum_wait_fanout(
}
}
/// Records a rename RPC after its canonical request digest has been attached.
///
/// This uses the PUT-stage diagnostics switch because it is temporary hot-path
/// evidence, but the metric is ECStore-scoped: scanner and pool metadata
/// renames are not S3 PutObject operations.
#[inline(always)]
pub fn record_ecstore_rename_remote_rpc(kind: &'static str) {
if !put_stage_metrics_enabled() {
return;
}
let Some(index) = ECSTORE_RENAME_RPC_KINDS.iter().position(|candidate| *candidate == kind) else {
return;
};
ECSTORE_RENAME_RPC_COUNTS[index].fetch_add(1, Ordering::Relaxed);
}
#[inline(always)]
fn rename_peer_width_index(width: usize) -> usize {
width.min(17)
}
/// Records the per-peer opportunity for a future body-bound rename batch.
#[inline(always)]
pub fn record_ecstore_rename_batch_candidate_peer(remote_disks: usize, batchable_disks: usize) {
if !put_stage_metrics_enabled() || remote_disks == 0 {
return;
}
ECSTORE_RENAME_REMOTE_WIDTH_COUNTS[rename_peer_width_index(remote_disks)].fetch_add(1, Ordering::Relaxed);
ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS[rename_peer_width_index(batchable_disks)].fetch_add(1, Ordering::Relaxed);
let coverage_index = if batchable_disks == 0 {
0
} else if batchable_disks == remote_disks {
2
} else {
1
};
ECSTORE_RENAME_COVERAGE_COUNTS[coverage_index].fetch_add(1, Ordering::Relaxed);
}
#[inline(always)]
fn record_ecstore_rename_batch_execution(index: usize, count: usize) {
if put_stage_metrics_enabled() {
let count = u64::try_from(count).unwrap_or(u64::MAX);
ECSTORE_RENAME_BATCH_EXECUTION_COUNTS[index].fetch_add(count, Ordering::Relaxed);
}
}
#[inline(always)]
pub fn record_ecstore_rename_batch_planned(items: usize) {
record_ecstore_rename_batch_execution(0, 1);
record_ecstore_rename_batch_execution(1, items);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_stream_started(items: usize) {
record_ecstore_rename_batch_execution(2, 1);
record_ecstore_rename_batch_execution(3, items);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_item_completed() {
record_ecstore_rename_batch_execution(4, 1);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_fallback(items: usize) {
record_ecstore_rename_batch_execution(5, 1);
record_ecstore_rename_batch_execution(6, items);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_not_dispatched(items: usize) {
record_ecstore_rename_batch_execution(7, items);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_may_have_published(items: usize) {
record_ecstore_rename_batch_execution(8, items);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_terminal_missing(items: usize) {
record_ecstore_rename_batch_execution(9, items);
}
#[inline(always)]
pub fn record_ecstore_rename_admission(batch: bool, items: usize, wait: Duration, timed_out: bool) {
if !put_stage_metrics_enabled() {
return;
}
let base = usize::from(batch) * 5;
let items = u64::try_from(items).unwrap_or(u64::MAX);
let wait_ns = u64::try_from(wait.as_nanos()).unwrap_or(u64::MAX);
ECSTORE_RENAME_ADMISSION_COUNTS[base].fetch_add(1, Ordering::Relaxed);
ECSTORE_RENAME_ADMISSION_COUNTS[base + 1].fetch_add(items, Ordering::Relaxed);
ECSTORE_RENAME_ADMISSION_COUNTS[base + 2].fetch_add(wait_ns, Ordering::Relaxed);
ECSTORE_RENAME_ADMISSION_COUNTS[base + 3].fetch_max(wait_ns, Ordering::Relaxed);
if timed_out {
ECSTORE_RENAME_ADMISSION_COUNTS[base + 4].fetch_add(1, Ordering::Relaxed);
}
}
#[inline(always)]
pub fn record_ecstore_rename_quarantine(batch: bool, items: usize, hold: Duration) {
if !put_stage_metrics_enabled() {
return;
}
let base = usize::from(batch) * 6;
let items = u64::try_from(items).unwrap_or(u64::MAX);
let hold_ns = u64::try_from(hold.as_nanos()).unwrap_or(u64::MAX);
ECSTORE_RENAME_QUARANTINE_COUNTS[base].fetch_add(1, Ordering::Relaxed);
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 1].fetch_add(items, Ordering::Relaxed);
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 2].fetch_add(hold_ns, Ordering::Relaxed);
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 3].fetch_max(hold_ns, Ordering::Relaxed);
}
#[inline(always)]
pub fn record_ecstore_rename_quarantine_released(batch: bool, items: usize) {
if put_stage_metrics_enabled() {
let base = usize::from(batch) * 6;
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 4].fetch_add(u64::try_from(items).unwrap_or(u64::MAX), Ordering::Relaxed);
}
}
#[inline(always)]
pub fn record_ecstore_rename_quarantine_no_runtime(batch: bool, items: usize) {
if put_stage_metrics_enabled() {
let base = usize::from(batch) * 6;
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 5].fetch_add(u64::try_from(items).unwrap_or(u64::MAX), Ordering::Relaxed);
}
}
#[inline(always)]
fn record_ecstore_rename_batch_timing(index: usize, duration: Duration) {
if !put_stage_metrics_enabled() {
return;
}
let duration_ns = u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX);
ECSTORE_RENAME_BATCH_TIMING_COUNTS[index].fetch_add(1, Ordering::Relaxed);
ECSTORE_RENAME_BATCH_TIMING_TOTAL_NS[index].fetch_add(duration_ns, Ordering::Relaxed);
ECSTORE_RENAME_BATCH_TIMING_MAX_NS[index].fetch_max(duration_ns, Ordering::Relaxed);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_client_stream_to_first(duration: Duration) {
record_ecstore_rename_batch_timing(0, duration);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_client_first_to_last(duration: Duration) {
record_ecstore_rename_batch_timing(1, duration);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_server_dispatch_to_first(duration: Duration) {
record_ecstore_rename_batch_timing(2, duration);
}
#[inline(always)]
pub fn record_ecstore_rename_batch_server_first_to_last(duration: Duration) {
record_ecstore_rename_batch_timing(3, duration);
}
/// Record generic internal operation stage duration (non-PUT paths).
/// Use this for metacache walks, listing, lifecycle, and other background
/// operations that are NOT part of the PUT object hot path.
@@ -4868,85 +4585,6 @@ mod tests {
);
}
#[test]
fn ecstore_rename_batch_candidate_metrics_are_gated_and_bounded() {
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
set_put_stage_metrics_enabled(false);
let disabled_before = snapshot_ecstore_rename_diagnostics();
record_ecstore_rename_remote_rpc(ECSTORE_RENAME_RPC_KIND_ORDINARY);
record_ecstore_rename_batch_candidate_peer(4, 4);
record_ecstore_rename_admission(false, 1, Duration::from_nanos(10), false);
record_ecstore_rename_quarantine(true, 4, Duration::from_nanos(30));
record_ecstore_rename_batch_client_stream_to_first(Duration::from_nanos(10));
assert_eq!(snapshot_ecstore_rename_diagnostics(), disabled_before);
set_put_stage_metrics_enabled(true);
let before = snapshot_ecstore_rename_diagnostics();
for kind in ECSTORE_RENAME_RPC_KINDS {
record_ecstore_rename_remote_rpc(kind);
}
record_ecstore_rename_batch_candidate_peer(4, 4);
record_ecstore_rename_batch_candidate_peer(4, 3);
record_ecstore_rename_batch_candidate_peer(4, 0);
record_ecstore_rename_batch_candidate_peer(20, 20);
record_ecstore_rename_batch_planned(4);
record_ecstore_rename_batch_stream_started(4);
record_ecstore_rename_batch_item_completed();
record_ecstore_rename_batch_fallback(3);
record_ecstore_rename_batch_not_dispatched(2);
record_ecstore_rename_batch_may_have_published(4);
record_ecstore_rename_batch_terminal_missing(1);
record_ecstore_rename_admission(false, 1, Duration::from_nanos(10), false);
record_ecstore_rename_admission(true, 4, Duration::from_nanos(20), true);
record_ecstore_rename_quarantine(true, 4, Duration::from_nanos(30));
record_ecstore_rename_quarantine_released(true, 3);
record_ecstore_rename_quarantine_no_runtime(true, 1);
record_ecstore_rename_batch_client_stream_to_first(Duration::from_nanos(10));
record_ecstore_rename_batch_client_first_to_last(Duration::from_nanos(20));
record_ecstore_rename_batch_server_dispatch_to_first(Duration::from_nanos(30));
record_ecstore_rename_batch_server_first_to_last(Duration::from_nanos(40));
set_put_stage_metrics_enabled(false);
let after = snapshot_ecstore_rename_diagnostics();
for index in 0..ECSTORE_RENAME_RPC_KINDS.len() {
assert_eq!(after.rpc_counts[index] - before.rpc_counts[index], 1);
}
assert_eq!(after.remote_width_counts[4] - before.remote_width_counts[4], 3);
assert_eq!(after.remote_width_counts[17] - before.remote_width_counts[17], 1);
assert_eq!(after.batchable_width_counts[0] - before.batchable_width_counts[0], 1);
assert_eq!(after.batchable_width_counts[3] - before.batchable_width_counts[3], 1);
assert_eq!(after.batchable_width_counts[4] - before.batchable_width_counts[4], 1);
assert_eq!(after.batchable_width_counts[17] - before.batchable_width_counts[17], 1);
assert_eq!(after.coverage_counts[0] - before.coverage_counts[0], 1);
assert_eq!(after.coverage_counts[1] - before.coverage_counts[1], 1);
assert_eq!(after.coverage_counts[2] - before.coverage_counts[2], 2);
assert_eq!(
std::array::from_fn::<_, 10, _>(|index| {
after.batch_execution_counts[index] - before.batch_execution_counts[index]
}),
[1, 4, 1, 4, 1, 1, 3, 2, 4, 1]
);
assert_eq!(
std::array::from_fn::<_, 10, _>(|index| after.admission_counts[index] - before.admission_counts[index]),
[1, 1, 10, 10, 0, 1, 4, 20, 20, 1]
);
assert_eq!(
std::array::from_fn::<_, 12, _>(|index| after.quarantine_counts[index] - before.quarantine_counts[index]),
[0, 0, 0, 0, 0, 0, 1, 4, 30, 30, 3, 1]
);
assert_eq!(
std::array::from_fn::<_, 4, _>(|index| after.batch_timing_counts[index] - before.batch_timing_counts[index]),
[1, 1, 1, 1]
);
assert_eq!(
std::array::from_fn::<_, 4, _>(|index| { after.batch_timing_total_ns[index] - before.batch_timing_total_ns[index] }),
[10, 20, 30, 40]
);
for (actual, expected) in after.batch_timing_max_ns.into_iter().zip([10, 20, 30, 40]) {
assert!(actual >= expected);
}
}
#[test]
fn test_put_object_diagnostic_buckets() {
assert_eq!(put_object_size_bucket(0), "unknown");
+2
View File
@@ -58,6 +58,8 @@ zeroize = { workspace = true, features = ["derive"] }
# Configuration and storage
url = { workspace = true }
tempfile = { workspace = true }
# Caching
moka = { workspace = true, features = ["future"] }
+8 -8
View File
@@ -65,7 +65,7 @@ use aws_sdk_kms::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_kms::types::{DataKeySpec, KeySpec as AwsKeySpec, KeyState as AwsKeyState, KeyUsageType, Tag};
use aws_smithy_runtime_api::client::orchestrator::HttpResponse;
use aws_smithy_types::Blob;
use jiff::ZonedDateTime;
use jiff::Zoned;
use tokio_util::sync::CancellationToken;
use super::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, empty_key_page, list_keys_page_size};
@@ -234,12 +234,12 @@ fn map_key_usage(usage: Option<&KeyUsageType>) -> KeyUsage {
}
}
fn to_zoned(value: Option<&aws_smithy_types::DateTime>) -> Option<ZonedDateTime> {
fn to_zoned(value: Option<&aws_smithy_types::DateTime>) -> Option<Zoned> {
let value = value?;
let nanos = i32::try_from(value.subsec_nanos()).ok()?;
jiff::Timestamp::new(value.secs(), nanos)
.ok()
.map(|timestamp| timestamp.to_zoned_datetime(jiff::tz::TimeZone::UTC))
.map(|timestamp| timestamp.to_zoned(jiff::tz::TimeZone::UTC))
}
/// `None` for an empty context so the request omits the field entirely rather
@@ -593,7 +593,7 @@ impl KmsBackend for AwsKmsBackend {
encrypted_key: ciphertext.into_inner(),
nonce: Vec::new(),
encryption_context: request.encryption_context,
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
// AWS addresses backing keys internally and never exposes a version.
master_key_version: None,
// AWS binds the context into the blob itself; this field describes
@@ -876,7 +876,7 @@ impl KmsBackend for AwsKmsBackend {
/// report what AWS has already done: a key that is gone counts as removed,
/// a key that left `PendingDeletion` counts as a state change, and a key
/// still pending is never expired by RustFS regardless of the deadline.
async fn remove_expired_key(&self, key_id: &str, _now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
async fn remove_expired_key(&self, key_id: &str, _now: &Zoned) -> Result<ExpiredKeyRemoval> {
match self.describe(key_id).await {
Ok(metadata) => Ok(match metadata.key_state {
KeyState::PendingDeletion => ExpiredKeyRemoval::NotExpired,
@@ -1492,7 +1492,7 @@ mod tests {
let (http_client, backend) = scripted_backend(vec![ok_event(key_metadata_json("test-key", "PendingDeletion"))]);
assert_eq!(
backend
.remove_expired_key("test-key", &ZonedDateTime::now())
.remove_expired_key("test-key", &Zoned::now())
.await
.expect("observation should succeed"),
ExpiredKeyRemoval::NotExpired
@@ -1502,7 +1502,7 @@ mod tests {
let (_http, backend) = scripted_backend(vec![error_event(400, "NotFoundException", "key does not exist")]);
assert_eq!(
backend
.remove_expired_key("test-key", &ZonedDateTime::now())
.remove_expired_key("test-key", &Zoned::now())
.await
.expect("a key AWS already removed is a completed removal"),
ExpiredKeyRemoval::Removed
@@ -1511,7 +1511,7 @@ mod tests {
let (_http, backend) = scripted_backend(vec![ok_event(key_metadata_json("test-key", "Enabled"))]);
assert_eq!(
backend
.remove_expired_key("test-key", &ZonedDateTime::now())
.remove_expired_key("test-key", &Zoned::now())
.await
.expect("observation should succeed"),
ExpiredKeyRemoval::StateChanged
+20 -20
View File
@@ -35,7 +35,7 @@ use aes_gcm::{
use argon2::{Algorithm, Argon2, Params, Version};
use async_trait::async_trait;
use base64_simd::STANDARD as BASE64;
use jiff::ZonedDateTime;
use jiff::Zoned;
use rand::RngExt;
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
use serde::{Deserialize, Serialize};
@@ -582,14 +582,14 @@ struct StoredMasterKey {
description: Option<String>,
metadata: HashMap<String, String>,
#[serde(with = "crate::time_serde::zoned")]
created_at: ZonedDateTime,
created_at: Zoned,
#[serde(with = "crate::time_serde::option_zoned")]
rotated_at: Option<ZonedDateTime>,
rotated_at: Option<Zoned>,
created_by: Option<String>,
/// Scheduled deletion deadline; absent on records written before deadline
/// persistence landed, so it must stay optional for backward compatibility.
#[serde(default, with = "crate::time_serde::option_zoned")]
deletion_date: Option<ZonedDateTime>,
deletion_date: Option<Zoned>,
/// Encrypted key material (32 bytes encoded in base64 for AES-256)
encrypted_key_material: String,
/// Nonce used for encryption
@@ -685,10 +685,10 @@ impl<'de> Deserialize<'de> for StoredMasterKey {
}
#[derive(Deserialize)]
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] ZonedDateTime);
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] Zoned);
#[derive(Deserialize)]
struct OptionalZonedValue(#[serde(with = "crate::time_serde::option_zoned")] Option<ZonedDateTime>);
struct OptionalZonedValue(#[serde(with = "crate::time_serde::option_zoned")] Option<Zoned>);
struct StoredMasterKeyVisitor;
@@ -1556,7 +1556,7 @@ impl LocalKmsClient {
encrypted_key,
nonce,
encryption_context: request.encryption_context.clone(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: None,
context_binding,
};
@@ -1596,7 +1596,7 @@ impl LocalKmsClient {
encrypted_key,
nonce,
encryption_context: request.encryption_context.clone(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
// Local rotation is rejected, so the key has a single material version.
master_key_version: None,
context_binding,
@@ -1860,7 +1860,7 @@ impl LocalKmsClient {
let mut master_key = self.load_master_key(key_id).await?;
ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::ScheduleDeletion)?;
master_key.status = KeyStatus::PendingDeletion;
master_key.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(pending_window_days as u64 * 86400));
master_key.deletion_date = Some(Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400));
// Preserve the existing key material (see enable_key): scheduling deletion must not
// regenerate the master key, or cancelling the deletion later would recover a key that
@@ -1985,7 +1985,7 @@ impl KmsBackend for LocalKmsBackend {
key_state: KeyState::Enabled,
key_usage: request.key_usage,
description: request.description,
creation_date: ZonedDateTime::now(),
creation_date: Zoned::now(),
deletion_date: None,
origin: "KMS".to_string(),
key_manager: "CUSTOMER".to_string(),
@@ -2113,7 +2113,7 @@ impl KmsBackend for LocalKmsBackend {
Ok((_stored, key_material)) => {
let mut tombstone = master_key.clone();
tombstone.status = KeyStatus::Deleted;
tombstone.deletion_date = Some(ZonedDateTime::now());
tombstone.deletion_date = Some(Zoned::now());
self.client.save_master_key(&tombstone, &key_material).await?;
}
Err(error) => {
@@ -2136,7 +2136,7 @@ impl KmsBackend for LocalKmsBackend {
key_usage: master_key.usage,
key_state: KeyState::PendingDeletion, // AWS KMS compatibility
creation_date: master_key.created_at,
deletion_date: Some(ZonedDateTime::now()),
deletion_date: Some(Zoned::now()),
key_manager: "CUSTOMER".to_string(),
origin: "AWS_KMS".to_string(),
tags: master_key.metadata,
@@ -2162,7 +2162,7 @@ impl KmsBackend for LocalKmsBackend {
)));
}
let deletion_date = ZonedDateTime::now() + Duration::from_secs(days as u64 * 86400);
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
master_key.status = KeyStatus::PendingDeletion;
master_key.deletion_date = Some(deletion_date.clone());
@@ -2317,7 +2317,7 @@ impl KmsBackend for LocalKmsBackend {
.with_update_key_metadata(true)
}
async fn remove_expired_key(&self, key_id: &str, now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
// The per-key write lock serializes this against a concurrent
// cancellation, closing the check-then-remove race.
let _write_guard = self.client.lock_key_for_write(key_id).await;
@@ -3088,12 +3088,12 @@ mod tests {
description: Option<String>,
metadata: HashMap<String, String>,
#[serde(with = "crate::time_serde::zoned")]
created_at: ZonedDateTime,
created_at: Zoned,
#[serde(with = "crate::time_serde::option_zoned")]
rotated_at: Option<ZonedDateTime>,
rotated_at: Option<Zoned>,
created_by: Option<String>,
#[serde(default, with = "crate::time_serde::option_zoned")]
deletion_date: Option<ZonedDateTime>,
deletion_date: Option<Zoned>,
encrypted_key_material: String,
nonce: Vec<u8>,
#[serde(default)]
@@ -4284,7 +4284,7 @@ mod tests {
let (_stored, key_material) = client.decode_stored_key(&key_id).await.expect("decode stored key");
let mut tombstone = client.load_master_key(&key_id).await.expect("load key");
tombstone.status = KeyStatus::Deleted;
tombstone.deletion_date = Some(ZonedDateTime::now());
tombstone.deletion_date = Some(Zoned::now());
client
.save_master_key(&tombstone, &key_material)
.await
@@ -4292,7 +4292,7 @@ mod tests {
// The sweep primitive completes the crashed removal...
let outcome = backend
.remove_expired_key(&key_id, &ZonedDateTime::now())
.remove_expired_key(&key_id, &Zoned::now())
.await
.expect("tombstone completion");
assert_eq!(outcome, crate::backends::ExpiredKeyRemoval::Removed);
@@ -4300,7 +4300,7 @@ mod tests {
// ...and stays idempotent once the key is gone.
let outcome = backend
.remove_expired_key(&key_id, &ZonedDateTime::now())
.remove_expired_key(&key_id, &Zoned::now())
.await
.expect("repeat removal");
assert_eq!(outcome, crate::backends::ExpiredKeyRemoval::Removed);
+3 -3
View File
@@ -17,7 +17,7 @@
use crate::error::{KmsError, Result};
use crate::types::*;
use async_trait::async_trait;
use jiff::ZonedDateTime;
use jiff::Zoned;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -608,7 +608,7 @@ pub trait KmsBackend: Send + Sync {
///
/// The default rejects the operation for backends without deletion
/// support.
async fn remove_expired_key(&self, _key_id: &str, _now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
async fn remove_expired_key(&self, _key_id: &str, _now: &Zoned) -> Result<ExpiredKeyRemoval> {
Err(KmsError::unsupported_capability("backend without deletion support", "remove_expired_key"))
}
@@ -934,7 +934,7 @@ mod tests {
#[tokio::test]
async fn default_remove_expired_key_is_unsupported() {
let error = MinimalBackend
.remove_expired_key("any-key", &jiff::ZonedDateTime::now())
.remove_expired_key("any-key", &jiff::Zoned::now())
.await
.expect_err("backends without deletion support must reject expired-key removal");
assert!(
+5 -5
View File
@@ -34,7 +34,7 @@ use aes_gcm::{
aead::{Aead, KeyInit, Payload},
};
use async_trait::async_trait;
use jiff::ZonedDateTime;
use jiff::Zoned;
use rand::RngExt;
use std::collections::HashMap;
use tracing::debug;
@@ -65,7 +65,7 @@ pub struct StaticKmsBackend {
/// across a restart and differs between nodes of one cluster, because there
/// is no creation event to anchor it to; callers must treat it as "when this
/// node loaded the key", not as the key's birth date.
created_at: ZonedDateTime,
created_at: Zoned,
}
impl StaticKmsBackend {
@@ -89,7 +89,7 @@ impl StaticKmsBackend {
Ok(Self {
key_id: static_config.key_id.clone(),
key: key.into(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
})
}
@@ -149,7 +149,7 @@ impl StaticKmsBackend {
encrypted_key: encrypted,
nonce: nonce_bytes.to_vec(),
encryption_context: request.encryption_context.clone(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
// The static backend has a single fixed key with no rotation.
master_key_version: None,
// The context is already bound as AAD by the Static cipher path
@@ -200,7 +200,7 @@ impl StaticKmsBackend {
encrypted_key: encrypted,
nonce: nonce_bytes.to_vec(),
encryption_context: request.encryption_context.clone(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
// The static backend has a single fixed key with no rotation.
master_key_version: None,
// The context is already bound as AAD by the Static cipher path
+40 -40
View File
@@ -35,7 +35,7 @@ use crate::policy::{self, AttemptError, OpClass, OwnWrite, OwnWrites, RetryPolic
use crate::types::*;
use async_trait::async_trait;
use base64_simd::STANDARD as BASE64;
use jiff::ZonedDateTime;
use jiff::Zoned;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::future::Future;
@@ -90,7 +90,7 @@ struct VaultKeyData {
/// Key usage type
usage: KeyUsage,
/// Key creation timestamp
created_at: ZonedDateTime,
created_at: Zoned,
/// Key status
status: KeyStatus,
/// Key version
@@ -104,7 +104,7 @@ struct VaultKeyData {
/// Scheduled deletion deadline; absent on records written before deadline
/// persistence landed, so it must stay optional for backward compatibility.
#[serde(default)]
deletion_date: Option<ZonedDateTime>,
deletion_date: Option<Zoned>,
/// When the key's material last became current through a rotation.
///
/// Written by [`VaultKmsClient::rotate_key`] as part of the same
@@ -117,7 +117,7 @@ struct VaultKeyData {
/// See [`crate::deletion_worker`] for why collapsing the two is safe for the
/// rotation-age gauge.
#[serde(default)]
rotated_at: Option<ZonedDateTime>,
rotated_at: Option<Zoned>,
/// Encrypted key material (base64 encoded)
encrypted_key_material: String,
/// Version that pre-versioning envelopes (no `master_key_version`) resolve to.
@@ -381,7 +381,7 @@ struct VaultKeyVersionRecord {
/// Encrypted key material (base64 encoded)
encrypted_key_material: String,
/// When this version's material was created
created_at: ZonedDateTime,
created_at: Zoned,
}
/// Sub-path (under each key path) reserved for immutable version records.
@@ -1305,7 +1305,7 @@ impl VaultKmsClient {
encrypted_key,
nonce,
encryption_context: request.encryption_context.clone(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: Some(key_data.version),
context_binding,
};
@@ -1347,7 +1347,7 @@ impl VaultKmsClient {
encrypted_key,
nonce,
encryption_context: request.encryption_context.clone(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: Some(key_data.version),
context_binding,
};
@@ -1718,7 +1718,7 @@ impl VaultKmsClient {
let key_data = VaultKeyData {
algorithm: algorithm.to_string(),
usage: attributes.map_or(KeyUsage::EncryptDecrypt, |request| request.key_usage.clone()),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
status: KeyStatus::Active,
version: 1,
description: attributes.and_then(|request| request.description.clone()),
@@ -2003,7 +2003,7 @@ impl VaultKmsClient {
let record = VaultKeyVersionRecord {
version: new_version,
encrypted_key_material: new_material.clone(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
};
if !self.try_create_key_version_record(key_id, &record).await? {
// A record for the next version already exists: an interrupted rotation
@@ -2027,7 +2027,7 @@ impl VaultKmsClient {
// the recorded time never runs ahead of the current version.
key_data.version = new_version;
key_data.encrypted_key_material = new_material;
key_data.rotated_at = Some(ZonedDateTime::now());
key_data.rotated_at = Some(Zoned::now());
// Fresh material, fresh AES-GCM nonce budget: the wrap ceiling is per
// key material version, so the counter restarts with the same commit
// that makes the new material current.
@@ -2114,7 +2114,7 @@ impl VaultKmsBackend {
/// record whose key material is missing: persisting it would cement the
/// empty-material state under a fresh document version — a damaged key
/// must go through an explicit repair operation, not a lifecycle update.
async fn mark_key_pending_deletion(&self, key_id: &str, deletion_date: &ZonedDateTime) -> Result<()> {
async fn mark_key_pending_deletion(&self, key_id: &str, deletion_date: &Zoned) -> Result<()> {
self.client
.update_key_data_with_cas(key_id, |key_data| {
ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::ScheduleDeletion)?;
@@ -2144,7 +2144,7 @@ impl KmsBackend for VaultKmsBackend {
key_state: KeyState::Enabled,
key_usage: request.key_usage,
description: request.description,
creation_date: ZonedDateTime::now(),
creation_date: Zoned::now(),
deletion_date: None,
origin: "VAULT".to_string(),
key_manager: "VAULT".to_string(),
@@ -2287,7 +2287,7 @@ impl KmsBackend for VaultKmsBackend {
.await?;
self.client.delete_key(key_id).await?;
key_metadata.key_state = KeyState::PendingDeletion;
key_metadata.deletion_date = Some(ZonedDateTime::now());
key_metadata.deletion_date = Some(Zoned::now());
// No deletion date: the key is permanently gone, not scheduled.
None
@@ -2306,7 +2306,7 @@ impl KmsBackend for VaultKmsBackend {
)));
}
let deletion_date = ZonedDateTime::now() + Duration::from_secs(days as u64 * 86400);
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
self.mark_key_pending_deletion(key_id, &deletion_date).await?;
key_metadata.key_state = KeyState::PendingDeletion;
key_metadata.deletion_date = Some(deletion_date.clone());
@@ -2415,7 +2415,7 @@ impl KmsBackend for VaultKmsBackend {
.with_production_supported(true)
}
async fn remove_expired_key(&self, key_id: &str, now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
// Tombstone under check-and-set with per-attempt re-validation: a
// cancellation landing between the read and the write makes the write
// conflict, and the re-read then observes the cancelled state and
@@ -2513,7 +2513,7 @@ mod tests {
VaultKeyData {
algorithm: "AES_256".to_string(),
usage: KeyUsage::EncryptDecrypt,
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
status: KeyStatus::Active,
version: 1,
description: None,
@@ -3197,7 +3197,7 @@ mod tests {
let key_data = VaultKeyData {
algorithm: "AES_256".to_string(),
usage: KeyUsage::EncryptDecrypt,
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
status: KeyStatus::Active,
version: 1,
description: None,
@@ -3235,8 +3235,8 @@ mod tests {
let mut key_data = healthy_key_data();
key_data.wrap_budget_reserved = 7_000_000;
key_data.baseline_version = Some(2);
key_data.rotated_at = Some(ZonedDateTime::now());
key_data.deletion_date = Some(ZonedDateTime::now());
key_data.rotated_at = Some(Zoned::now());
key_data.deletion_date = Some(Zoned::now());
key_data.description = Some("described".to_string());
let value = serde_json::to_value(&key_data).expect("serialize key data");
@@ -3683,11 +3683,11 @@ mod tests {
/// of deadline persistence turns this test red.
#[test]
fn vault_key_data_deletion_date_round_trips_and_stays_backward_compatible() {
let deadline = ZonedDateTime::now() + Duration::from_secs(7 * 86400);
let deadline = Zoned::now() + Duration::from_secs(7 * 86400);
let key_data = VaultKeyData {
algorithm: "AES_256".to_string(),
usage: KeyUsage::EncryptDecrypt,
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
status: KeyStatus::PendingDeletion,
version: 1,
description: None,
@@ -3703,7 +3703,7 @@ mod tests {
let mut value = serde_json::to_value(&key_data).expect("serialize");
let restored: VaultKeyData = serde_json::from_value(value.clone()).expect("round trip");
assert_eq!(
restored.deletion_date.as_ref().map(ZonedDateTime::timestamp),
restored.deletion_date.as_ref().map(Zoned::timestamp),
Some(deadline.timestamp()),
"deletion deadline must survive the KV2 round trip"
);
@@ -4055,10 +4055,10 @@ mod tests {
/// cancellation and destroyed the key.
#[tokio::test]
async fn wired_expired_key_sweep_yields_to_concurrent_cancellation() {
let now = ZonedDateTime::now() + Duration::from_secs(3600);
let now = Zoned::now() + Duration::from_secs(3600);
let mut pending = healthy_key_data();
pending.status = KeyStatus::PendingDeletion;
pending.deletion_date = Some(ZonedDateTime::now());
pending.deletion_date = Some(Zoned::now());
let vault = ScriptedVault::serve(vec![
ScriptedResponse::ok(kv2_metadata_read_data(1)),
@@ -4103,7 +4103,7 @@ mod tests {
async fn wired_cancel_deletion_after_sweep_tombstone_fails_closed() {
let mut pending = healthy_key_data();
pending.status = KeyStatus::PendingDeletion;
pending.deletion_date = Some(ZonedDateTime::now());
pending.deletion_date = Some(Zoned::now());
let mut tombstoned = healthy_key_data();
tombstoned.status = KeyStatus::Deleted;
@@ -4192,7 +4192,7 @@ mod tests {
async fn wired_immediate_deletion_of_a_pending_key_destroys_it() {
let mut pending = healthy_key_data();
pending.status = KeyStatus::PendingDeletion;
pending.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(7 * 86400));
pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
let vault = ScriptedVault::serve(vec![
// describe_key: key info plus stored metadata.
@@ -4328,7 +4328,7 @@ mod tests {
async fn wired_schedule_deletion_regates_after_conflict() {
let mut already_pending = healthy_key_data();
already_pending.status = KeyStatus::PendingDeletion;
already_pending.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(7 * 86400));
already_pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
let vault = ScriptedVault::serve(vec![
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
@@ -4747,7 +4747,7 @@ mod tests {
let record_v2 = VaultKeyVersionRecord {
version: 2,
encrypted_key_material: base64_simd::STANDARD.encode_to_string(material_v2),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
};
// A well-formed envelope wrapped under version 2 — under a reverted
// guard this decrypt would *succeed*, which is exactly the masked
@@ -4763,7 +4763,7 @@ mod tests {
encrypted_key,
nonce,
encryption_context: HashMap::new(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: Some(2),
context_binding: None,
};
@@ -4843,7 +4843,7 @@ mod tests {
let record_v2 = VaultKeyVersionRecord {
version: 2,
encrypted_key_material: adopted_material.clone(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
};
let (vault, client) = scripted_client(vec![
@@ -5042,7 +5042,7 @@ mod tests {
/// Reverting either half turns this test red.
#[tokio::test]
async fn wired_rotate_persists_rotation_time_and_describe_reports_it() {
let created_at = ZonedDateTime::now() - Duration::from_secs(365 * 86400);
let created_at = Zoned::now() - Duration::from_secs(365 * 86400);
let mut key_data = healthy_key_data();
key_data.created_at = created_at.clone();
key_data.version = 2;
@@ -5069,7 +5069,7 @@ mod tests {
let persisted: VaultKeyData =
serde_json::from_value(committed["data"].clone()).expect("the committed record must deserialize");
assert_eq!(
persisted.rotated_at.as_ref().map(ZonedDateTime::timestamp),
persisted.rotated_at.as_ref().map(Zoned::timestamp),
Some(reported.timestamp()),
"the rotation must persist the time it reports: {committed}"
);
@@ -5091,12 +5091,12 @@ mod tests {
.await
.expect("describe the rotated key");
assert_eq!(
described.rotated_at.as_ref().map(ZonedDateTime::timestamp),
described.rotated_at.as_ref().map(Zoned::timestamp),
Some(reported.timestamp()),
"describe must report the persisted rotation time"
);
assert_ne!(
described.rotated_at.as_ref().map(ZonedDateTime::timestamp),
described.rotated_at.as_ref().map(Zoned::timestamp),
Some(created_at.timestamp()),
"a rotated key must not be aged from its creation"
);
@@ -5109,7 +5109,7 @@ mod tests {
#[tokio::test]
async fn wired_describe_key_invents_no_rotation_time_for_legacy_records() {
let mut key_data = healthy_key_data();
key_data.created_at = ZonedDateTime::now() - Duration::from_secs(365 * 86400);
key_data.created_at = Zoned::now() - Duration::from_secs(365 * 86400);
key_data.version = 4;
key_data.baseline_version = Some(1);
@@ -5143,7 +5143,7 @@ mod tests {
/// rest of their contents intact.
#[test]
fn vault_key_data_rotated_at_round_trips_and_stays_backward_compatible() {
let rotated_at = ZonedDateTime::now();
let rotated_at = Zoned::now();
let mut key_data = healthy_key_data();
key_data.rotated_at = Some(rotated_at.clone());
key_data.baseline_version = Some(1);
@@ -5153,7 +5153,7 @@ mod tests {
let mut value = serde_json::to_value(&key_data).expect("serialize");
let restored: VaultKeyData = serde_json::from_value(value.clone()).expect("round trip");
assert_eq!(
restored.rotated_at.as_ref().map(ZonedDateTime::timestamp),
restored.rotated_at.as_ref().map(Zoned::timestamp),
Some(rotated_at.timestamp()),
"the rotation time must survive the KV2 round trip"
);
@@ -5190,7 +5190,7 @@ mod tests {
encrypted_key,
nonce,
encryption_context: HashMap::new(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: None,
context_binding: None,
};
@@ -5250,7 +5250,7 @@ mod tests {
encrypted_key,
nonce,
encryption_context: HashMap::new(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: None,
context_binding: None,
};
@@ -5302,7 +5302,7 @@ mod tests {
encrypted_key,
nonce,
encryption_context: HashMap::new(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: None,
context_binding: None,
};
+18 -18
View File
@@ -33,7 +33,7 @@ use crate::policy::{self, AttemptError, OpClass, OwnWrite, OwnWrites, RetryPolic
use crate::types::*;
use async_trait::async_trait;
use base64_simd::STANDARD as BASE64;
use jiff::ZonedDateTime;
use jiff::Zoned;
use moka::future::Cache;
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap};
@@ -141,8 +141,8 @@ struct TransitKeyMetadata {
description: Option<String>,
tags: HashMap<String, String>,
key_state: KeyState,
created_at: ZonedDateTime,
deletion_date: Option<ZonedDateTime>,
created_at: Zoned,
deletion_date: Option<Zoned>,
origin: String,
created_by: Option<String>,
current_version: u32,
@@ -160,8 +160,8 @@ struct TransitKeyMetadataPersisted {
description: Option<String>,
tags: HashMap<String, String>,
key_state: KeyState,
created_at: ZonedDateTime,
deletion_date: Option<ZonedDateTime>,
created_at: Zoned,
deletion_date: Option<Zoned>,
origin: String,
created_by: Option<String>,
current_version: u32,
@@ -336,7 +336,7 @@ impl TransitKeyMetadata {
description: request.description.clone(),
tags: request.tags.clone(),
key_state: KeyState::Enabled,
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
deletion_date: None,
origin: request.origin.clone().unwrap_or_else(|| "VAULT_TRANSIT".to_string()),
created_by: None,
@@ -360,7 +360,7 @@ impl TransitKeyMetadata {
description: None,
tags: HashMap::new(),
key_state: KeyState::Enabled,
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
deletion_date: None,
origin: "VAULT_TRANSIT".to_string(),
created_by: None,
@@ -1055,7 +1055,7 @@ impl VaultTransitKmsClient {
encrypted_key: encrypted_key.into_bytes(),
nonce: Vec::new(),
encryption_context: request.encryption_context.clone(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
// Transit ciphertext already self-describes its key version
// ("vault:vN:..."), so the envelope never carries one.
master_key_version: None,
@@ -1100,7 +1100,7 @@ impl VaultTransitKmsClient {
encrypted_key: encrypted.into_bytes(),
nonce: Vec::new(),
encryption_context: request.encryption_context.clone(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
// Transit ciphertext already self-describes its key version
// ("vault:vN:..."), so the envelope never carries one.
master_key_version: None,
@@ -1543,7 +1543,7 @@ impl VaultTransitKmsClient {
pending_window_days: u32,
_context: Option<&OperationContext>,
) -> Result<()> {
let deletion_date = ZonedDateTime::now() + Duration::from_secs(pending_window_days as u64 * 86400);
let deletion_date = Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400);
self.mutate_key_metadata(key_id, |metadata| {
ensure_key_state_permits(key_id, &metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
metadata.key_state = KeyState::PendingDeletion;
@@ -1592,7 +1592,7 @@ impl VaultTransitKmsClient {
description: metadata.description,
metadata: metadata.tags,
created_at: metadata.created_at,
rotated_at: Some(ZonedDateTime::now()),
rotated_at: Some(Zoned::now()),
created_by: metadata.created_by,
deletion_date: None,
})
@@ -1810,7 +1810,7 @@ impl KmsBackend for VaultTransitKmsBackend {
// Transit key is destroyed leaves a key that is already unusable.
// Re-checked against every fresh snapshot: a concurrent
// cancellation that commits first fails this removal.
let now = ZonedDateTime::now();
let now = Zoned::now();
self.client
.mutate_key_metadata(&key_id, |metadata| match metadata.key_state {
KeyState::PendingDeletion | KeyState::Unavailable => {
@@ -1844,7 +1844,7 @@ impl KmsBackend for VaultTransitKmsBackend {
)));
}
let scheduled = ZonedDateTime::now() + Duration::from_secs(days as u64 * 86400);
let scheduled = Zoned::now() + Duration::from_secs(days as u64 * 86400);
self.client
.mutate_key_metadata(&key_id, |metadata| {
// Re-run the gate against every fresh snapshot: the check
@@ -1934,7 +1934,7 @@ impl KmsBackend for VaultTransitKmsBackend {
.with_production_supported(true)
}
async fn remove_expired_key(&self, key_id: &str, now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
// The transit key's existence anchors "already removed": once it is
// gone only stale scheduling metadata can remain, so clean that up.
match self.client.read_transit_key(key_id).await {
@@ -3220,7 +3220,7 @@ mod tests {
encrypted_key: b"vault:v1:AAAA".to_vec(),
nonce: Vec::new(),
encryption_context: HashMap::new(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: Some(1),
context_binding: None,
};
@@ -3297,7 +3297,7 @@ mod tests {
async fn wired_immediate_deletion_of_a_pending_key_destroys_it() {
let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
pending.key_state = KeyState::PendingDeletion;
pending.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(7 * 86400));
pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
let vault = ScriptedVault::serve(vec![
// Current key metadata for the response.
ScriptedResponse::ok(transit_key_read_data("wired-key")),
@@ -3354,7 +3354,7 @@ mod tests {
async fn wired_expired_sweep_backs_off_when_cancel_wins_the_cas_race() {
let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
pending.key_state = KeyState::PendingDeletion;
pending.deletion_date = Some(ZonedDateTime::now());
pending.deletion_date = Some(Zoned::now());
let cancelled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
let vault = ScriptedVault::serve(vec![
// The transit key still exists.
@@ -3378,7 +3378,7 @@ mod tests {
.await
.expect("vault transit backend should build");
let now = ZonedDateTime::now() + Duration::from_secs(3600);
let now = Zoned::now() + Duration::from_secs(3600);
let outcome = backend
.remove_expired_key("wired-key", &now)
.await
+8 -8
View File
@@ -75,7 +75,7 @@ use crate::error::{KmsError, Result};
use crate::manager::KmsManager;
use crate::service::ObjectEncryptionService;
use crate::types::{CreateKeyRequest, EncryptionAlgorithm, EncryptionMetadata, KeyUsage};
use jiff::ZonedDateTime;
use jiff::Zoned;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
@@ -272,10 +272,10 @@ pub struct DrillRecoveryEvidence {
pub struct DrillRpoEvidence {
/// When the export sealed the bundle.
#[serde(with = "crate::time_serde::zoned")]
pub snapshot_completed_at: ZonedDateTime,
pub snapshot_completed_at: Zoned,
/// When the disaster struck.
#[serde(with = "crate::time_serde::zoned")]
pub disaster_at: ZonedDateTime,
pub disaster_at: Zoned,
/// The data-loss window: work committed inside it is not in the bundle.
pub rpo_window_millis: u64,
/// Keys created after the fence closed, which the bundle cannot contain.
@@ -322,7 +322,7 @@ pub struct DrillEvidence {
pub dataset: DrillDataset,
/// When the run started.
#[serde(with = "crate::time_serde::zoned")]
pub started_at: ZonedDateTime,
pub started_at: Zoned,
/// Bundle evidence.
pub bundle: DrillBundleEvidence,
/// Recovery evidence.
@@ -508,7 +508,7 @@ impl PhaseClock {
/// still produces the record of *how* it failed.
pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result<DrillEvidence> {
request.validate()?;
let started_at = ZonedDateTime::now();
let started_at = Zoned::now();
let mut clock = PhaseClock::new();
let mut findings = Vec::new();
@@ -557,7 +557,7 @@ pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result<
},
)
.await?;
let snapshot_completed_at = ZonedDateTime::now();
let snapshot_completed_at = Zoned::now();
clock.finish(DrillPhase::Export);
// Past the fence: this key belongs to the window the bundle cannot
@@ -576,7 +576,7 @@ pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result<
let bundle_digest_before = tree_digest(&bundle_dir).await?;
let destroyed_files = apply_disaster(&live_dir, request.disaster).await?;
let disaster_at = ZonedDateTime::now();
let disaster_at = Zoned::now();
clock.finish(DrillPhase::Disaster);
// Recovery step one is always the same: preserve what survived, then
@@ -980,7 +980,7 @@ fn walk_files(root: &Path) -> std::io::Result<Vec<PathBuf>> {
Ok(files)
}
fn elapsed_millis(from: &ZonedDateTime, to: &ZonedDateTime) -> u64 {
fn elapsed_millis(from: &Zoned, to: &Zoned) -> u64 {
let nanos = to.timestamp().as_nanosecond() - from.timestamp().as_nanosecond();
u64::try_from(nanos / 1_000_000).unwrap_or(0)
}
+2 -2
View File
@@ -63,7 +63,7 @@ use aes_gcm::{
Aes256Gcm, Key, Nonce,
aead::{Aead, KeyInit, Payload},
};
use jiff::ZonedDateTime;
use jiff::Zoned;
use rand::RngExt;
use serde::Deserialize;
use sha2::{Digest, Sha256};
@@ -454,7 +454,7 @@ async fn build_and_write_bundle(
// Normalized to UTC so the stored spelling is host-independent: the
// local zone's name (or a POSIX TZ string in minimal containers) has
// no business inside a portable bundle.
created_at: ZonedDateTime::now().with_time_zone(jiff::tz::TimeZone::UTC),
created_at: Zoned::now().with_time_zone(jiff::tz::TimeZone::UTC),
rustfs_version: request.rustfs_version.clone(),
deployment_identity: request.deployment_identity.clone(),
backend: BackupBackendKind::Local,
+2 -2
View File
@@ -22,7 +22,7 @@
use crate::backup::capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
use crate::backup::error::BackupError;
use jiff::ZonedDateTime;
use jiff::Zoned;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::BTreeSet;
@@ -503,7 +503,7 @@ pub struct BackupManifest {
pub backup_id: String,
/// Creation time of the bundle.
#[serde(with = "crate::time_serde::zoned")]
pub created_at: ZonedDateTime,
pub created_at: Zoned,
/// RustFS version that produced the bundle.
pub rustfs_version: String,
/// Opaque identity of the producing deployment, used by restore preflight
+5 -5
View File
@@ -239,7 +239,7 @@ impl KmsCache {
mod tests {
use super::*;
use crate::types::{KeyState, KeyUsage};
use jiff::ZonedDateTime;
use jiff::Zoned;
use metrics_util::MetricKind;
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
use std::time::Duration;
@@ -264,7 +264,7 @@ mod tests {
key_state: KeyState::Enabled,
key_usage: KeyUsage::EncryptDecrypt,
description: None,
creation_date: ZonedDateTime::now(),
creation_date: Zoned::now(),
deletion_date: None,
origin: "KMS".to_string(),
key_manager: "CUSTOMER".to_string(),
@@ -342,7 +342,7 @@ mod tests {
key_state: KeyState::Enabled,
key_usage: KeyUsage::EncryptDecrypt,
description: Some("Test key".to_string()),
creation_date: ZonedDateTime::now(),
creation_date: Zoned::now(),
deletion_date: None,
origin: "KMS".to_string(),
key_manager: "CUSTOMER".to_string(),
@@ -376,7 +376,7 @@ mod tests {
key_state: KeyState::Enabled,
key_usage: KeyUsage::EncryptDecrypt,
description: Some("TTL test key".to_string()),
creation_date: ZonedDateTime::now(),
creation_date: Zoned::now(),
deletion_date: None,
origin: "KMS".to_string(),
key_manager: "CUSTOMER".to_string(),
@@ -406,7 +406,7 @@ mod tests {
key_state: KeyState::Enabled,
key_usage: KeyUsage::EncryptDecrypt,
description: None,
creation_date: ZonedDateTime::now(),
creation_date: Zoned::now(),
deletion_date: None,
origin: "KMS".to_string(),
key_manager: "CUSTOMER".to_string(),
+16 -16
View File
@@ -27,7 +27,7 @@ use crate::backends::{ExpiredKeyRemoval, KmsBackend};
use crate::error::Result;
use crate::types::{KeyInfo, KeyStatus, ListKeysRequest, OperationContext};
use async_trait::async_trait;
use jiff::ZonedDateTime;
use jiff::Zoned;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;
@@ -117,7 +117,7 @@ struct KeyCensus {
}
impl KeyCensus {
fn observe(&mut self, key: &KeyInfo, now: &ZonedDateTime) {
fn observe(&mut self, key: &KeyInfo, now: &Zoned) {
match key.status {
KeyStatus::PendingDeletion => self.pending_deletion += 1,
KeyStatus::Deleted => self.tombstones += 1,
@@ -145,7 +145,7 @@ impl KeyCensus {
/// Seconds from `earlier` to `now`, clamped at zero so clock skew (or a
/// timestamp persisted by a node running ahead) cannot produce a negative age.
fn seconds_between(earlier: &ZonedDateTime, now: &ZonedDateTime) -> f64 {
fn seconds_between(earlier: &Zoned, now: &Zoned) -> f64 {
(now.timestamp().as_second() - earlier.timestamp().as_second()).max(0) as f64
}
@@ -273,7 +273,7 @@ impl DeletionWorker {
}
_ = ticker.tick() => {}
}
let report = self.sweep(&ZonedDateTime::now()).await;
let report = self.sweep(&Zoned::now()).await;
if !report.removed.is_empty() || !report.blocked.is_empty() || report.failed > 0 || report.unreadable > 0 {
info!(
removed = ?report.removed,
@@ -293,7 +293,7 @@ impl DeletionWorker {
/// Also feeds the lifecycle gauges: every key it lists is counted into a
/// [`KeyCensus`] on the way past, so the observability comes out of the
/// pages the sweep already had to fetch.
pub(crate) async fn sweep(&self, now: &ZonedDateTime) -> SweepReport {
pub(crate) async fn sweep(&self, now: &Zoned) -> SweepReport {
let mut report = SweepReport::default();
let mut census = KeyCensus::default();
let mut marker: Option<String> = None;
@@ -359,7 +359,7 @@ impl DeletionWorker {
}
/// Handle one key that is on its way out; reports whether it was removed.
async fn process_key(&self, key_id: &str, now: &ZonedDateTime, report: &mut SweepReport) -> bool {
async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) -> bool {
// Never remove a key that live configuration still points at. The
// default key check is built in; broader references (bucket
// encryption settings, ...) come from the injected checker.
@@ -464,8 +464,8 @@ mod tests {
DeletionWorker::new(backend, None, None, "local")
}
fn after_window() -> ZonedDateTime {
ZonedDateTime::now() + Duration::from_secs(8 * 86400)
fn after_window() -> Zoned {
Zoned::now() + Duration::from_secs(8 * 86400)
}
async fn assert_key_gone(backend: &LocalKmsBackend, key_id: &str) {
@@ -488,7 +488,7 @@ mod tests {
let worker = worker(backend.clone());
// Not yet due: nothing happens.
let report = worker.sweep(&ZonedDateTime::now()).await;
let report = worker.sweep(&Zoned::now()).await;
assert!(report.removed.is_empty());
assert_eq!(report.skipped, 1);
assert_eq!(report.failed, 0);
@@ -635,7 +635,7 @@ mod tests {
let worker = DeletionWorker::new(backend.clone(), None, None, "local").with_audit_sink(Some(sink.clone()));
// A key that is not yet due was never at risk, so it produces no record.
worker.sweep(&ZonedDateTime::now()).await;
worker.sweep(&Zoned::now()).await;
assert!(
sink.records.lock().expect("sink lock").is_empty(),
"a key the sweep declined to touch must not be audited as a deletion"
@@ -780,7 +780,7 @@ mod tests {
.sum()
}
fn key_info(key_id: &str, status: KeyStatus, created_at: ZonedDateTime, rotated_at: Option<ZonedDateTime>) -> KeyInfo {
fn key_info(key_id: &str, status: KeyStatus, created_at: Zoned, rotated_at: Option<Zoned>) -> KeyInfo {
KeyInfo {
key_id: key_id.to_string(),
description: None,
@@ -801,7 +801,7 @@ mod tests {
#[test]
fn census_ages_from_rotation_and_ignores_departing_keys() {
let now = ZonedDateTime::now();
let now = Zoned::now();
let day = Duration::from_secs(86400);
let mut census = KeyCensus::default();
@@ -833,7 +833,7 @@ mod tests {
/// material never wraps again, so its consumed nonce budget is moot.
#[test]
fn census_takes_the_max_wrap_reservation_of_usable_keys_only() {
let now = ZonedDateTime::now();
let now = Zoned::now();
let mut census = KeyCensus::default();
census.observe(&key_info("uncounted", KeyStatus::Active, now.clone(), None), &now);
@@ -859,7 +859,7 @@ mod tests {
fn wrap_budget_gauge_is_aggregate_and_carries_no_key_label() {
let (snapshot, ()) = record_metrics(|| {
Box::pin(async {
let now = ZonedDateTime::now();
let now = Zoned::now();
let mut census = KeyCensus::default();
let mut wrapped = key_info("wrapped-key-id", KeyStatus::Active, now.clone(), None);
wrapped.wrap_budget_reserved = Some(2_000_000);
@@ -897,7 +897,7 @@ mod tests {
// Three days in, still inside the seven-day window: the sweep
// observes the scheduled key instead of removing it.
let report = worker(backend.clone())
.sweep(&(ZonedDateTime::now() + Duration::from_secs(3 * 86400)))
.sweep(&(Zoned::now() + Duration::from_secs(3 * 86400)))
.await;
assert_eq!(report.skipped, 1);
assert!(report.removed.is_empty());
@@ -950,7 +950,7 @@ mod tests {
// Well inside the seven-day window: every key is observed as
// pending rather than removed.
let report = worker(backend.clone()).sweep(&ZonedDateTime::now()).await;
let report = worker(backend.clone()).sweep(&Zoned::now()).await;
assert_eq!(report.skipped, total, "every scheduled key must be inspected");
assert!(report.removed.is_empty());
})
+9 -9
View File
@@ -21,7 +21,7 @@
use crate::error::{KmsError, Result};
use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary};
use async_trait::async_trait;
use jiff::ZonedDateTime;
use jiff::Zoned;
use rand::Rng;
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
use serde::{Deserialize, Serialize};
@@ -66,7 +66,7 @@ pub struct DataKeyEnvelope {
pub nonce: Vec<u8>,
pub encryption_context: HashMap<String, String>,
#[serde(with = "crate::time_serde::zoned")]
pub created_at: ZonedDateTime,
pub created_at: Zoned,
/// KEK version that wrapped `encrypted_key`; `None` on pre-versioning envelopes.
///
/// Optional and omitted when `None` so envelopes from non-rotating backends stay
@@ -205,7 +205,7 @@ impl<'de> Deserialize<'de> for DataKeyEnvelope {
}
#[derive(Deserialize)]
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] ZonedDateTime);
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] Zoned);
struct DataKeyEnvelopeVisitor;
@@ -591,7 +591,7 @@ mod tests {
encrypted_key: vec![1, 2, 3, 4],
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
encryption_context: HashMap::from([("bucket".to_string(), "b".to_string())]),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: None,
context_binding: None,
};
@@ -625,7 +625,7 @@ mod tests {
encrypted_key: vec![1, 2, 3, 4],
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
encryption_context: HashMap::new(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: None,
context_binding: Some(CONTEXT_BINDING_AAD_V1),
};
@@ -676,7 +676,7 @@ mod tests {
map.insert("bucket".to_string(), "test-bucket".to_string());
map
},
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: None,
context_binding: None,
};
@@ -694,7 +694,7 @@ mod tests {
#[tokio::test]
async fn test_data_key_envelope_backward_compatibility() {
// Test deserialization with current ZonedDateTime format (with timezone annotation)
// Test deserialization with current Zoned format (with timezone annotation)
let envelope_json = r#"{
"key_id": "test-key-id",
"master_key_id": "master-key-id",
@@ -812,7 +812,7 @@ mod tests {
encrypted_key: vec![1, 2, 3, 4],
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
encryption_context: HashMap::new(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: None,
context_binding: None,
};
@@ -832,7 +832,7 @@ mod tests {
encrypted_key: vec![1, 2, 3, 4],
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
encryption_context: HashMap::new(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
master_key_version: Some(7),
context_binding: None,
};
+9 -9
View File
@@ -28,7 +28,7 @@ use crate::types::{
MAX_PENDING_DELETION_WINDOW_DAYS, MIN_PENDING_DELETION_WINDOW_DAYS, OperationContext, RewrapDataKeyRequest,
RewrapDataKeyResponse, RotationDueReason,
};
use jiff::ZonedDateTime;
use jiff::Zoned;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
@@ -335,7 +335,7 @@ impl KmsManager {
let mut result = self.backend.list_keys(request).await;
if let Ok(response) = result.as_mut() {
let rotates = self.backend.capabilities().rotate;
let now = ZonedDateTime::now();
let now = Zoned::now();
for key in &mut response.keys {
self.apply_rotation_readiness(key, rotates, &now);
}
@@ -351,7 +351,7 @@ impl KmsManager {
/// about what "overdue" means, and so a backend that cannot rotate is never
/// the one deciding whether it should. Nothing consults the verdict before
/// encrypting or decrypting: a key reported as due keeps serving traffic.
fn apply_rotation_readiness(&self, key: &mut KeyInfo, backend_rotates: bool, now: &ZonedDateTime) {
fn apply_rotation_readiness(&self, key: &mut KeyInfo, backend_rotates: bool, now: &Zoned) {
if !backend_rotates {
// Reported, not silently omitted: an operator chasing an overdue key
// needs to know the answer is "this backend cannot rotate at all",
@@ -699,7 +699,7 @@ mod tests {
use crate::error::KmsError;
use crate::types::{KeyMetadata, KeySpec, KeyState, KeyStatus, KeyUsage};
use async_trait::async_trait;
use jiff::ZonedDateTime;
use jiff::Zoned;
use std::collections::HashMap;
use std::sync::Mutex;
use std::time::Duration;
@@ -764,7 +764,7 @@ mod tests {
key_state: KeyState::Enabled,
key_usage: KeyUsage::EncryptDecrypt,
description: None,
creation_date: ZonedDateTime::now(),
creation_date: Zoned::now(),
deletion_date: None,
origin: "RUSTFS_KMS".to_string(),
key_manager: "RUSTFS".to_string(),
@@ -1810,7 +1810,7 @@ mod tests {
manager
}
fn aged_key(rotated_at: Option<ZonedDateTime>, created_at: ZonedDateTime) -> KeyInfo {
fn aged_key(rotated_at: Option<Zoned>, created_at: Zoned) -> KeyInfo {
KeyInfo {
key_id: "key-a".to_string(),
description: None,
@@ -1833,7 +1833,7 @@ mod tests {
/// a backend that cannot rotate is never told to rotate.
#[test]
fn rotation_readiness_reports_but_never_asks_the_impossible() {
let now = ZonedDateTime::now();
let now = Zoned::now();
let long_ago = &now - jiff::Span::new().days(400);
let recently = &now - jiff::Span::new().hours(1);
let day = Duration::from_secs(86_400);
@@ -1887,13 +1887,13 @@ mod tests {
/// independently of the age policy and reported under its own reason.
#[test]
fn rotation_readiness_reports_an_exhausted_wrap_budget() {
let now = ZonedDateTime::now();
let now = Zoned::now();
let recently = &now - jiff::Span::new().hours(1);
let long_ago = &now - jiff::Span::new().days(400);
let day = Duration::from_secs(86_400);
let budget = 2_000_000;
let with_wraps = |manager: &KmsManager, wraps: Option<u64>, rotated_at: Option<ZonedDateTime>| {
let with_wraps = |manager: &KmsManager, wraps: Option<u64>, rotated_at: Option<Zoned>| {
let mut key = aged_key(rotated_at, recently.clone());
key.wrap_budget_reserved = wraps;
manager.apply_rotation_readiness(&mut key, true, &now);
+1 -1
View File
@@ -595,7 +595,7 @@ mod tests {
key_state: KeyState::Enabled,
key_usage: KeyUsage::EncryptDecrypt,
description: None,
creation_date: jiff::ZonedDateTime::now(),
creation_date: jiff::Zoned::now(),
deletion_date: None,
origin: "test".to_string(),
key_manager: "test".to_string(),
+5 -5
View File
@@ -23,7 +23,7 @@ use crate::encryption::context_aad;
use crate::error::{KmsError, Result};
use crate::manager::KmsManager;
use crate::types::*;
use jiff::ZonedDateTime;
use jiff::Zoned;
use md5::{Digest as Md5Digest, Md5};
use rand::random;
use rustfs_utils::http::object_encryption_keys::{
@@ -543,7 +543,7 @@ impl ObjectEncryptionService {
iv,
tag: Some(tag),
encryption_context: context,
encrypted_at: ZonedDateTime::now(),
encrypted_at: Zoned::now(),
// Pinned to the bytes actually fed to the AEAD, so the projection
// below can store them verbatim instead of re-deriving them.
context_aad: Some(aad),
@@ -702,7 +702,7 @@ impl ObjectEncryptionService {
iv,
tag: Some(tag),
encryption_context: context,
encrypted_at: ZonedDateTime::now(),
encrypted_at: Zoned::now(),
// Pinned to the bytes actually fed to the AEAD, so the projection
// below can store them verbatim instead of re-deriving them.
context_aad: Some(aad),
@@ -953,7 +953,7 @@ impl ObjectEncryptionService {
iv,
tag,
encryption_context,
encrypted_at: ZonedDateTime::now(),
encrypted_at: Zoned::now(),
original_size: 0, // Not available from headers
encrypted_data_key,
context_aad,
@@ -1070,7 +1070,7 @@ mod tests {
iv: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
tag: Some(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
encryption_context: HashMap::from([("bucket".to_string(), "test-bucket".to_string())]),
encrypted_at: ZonedDateTime::now(),
encrypted_at: Zoned::now(),
original_size: 100,
encrypted_data_key: vec![1, 2, 3, 4],
// A hand-built record with no sealed bytes to defer to.
+8 -8
View File
@@ -1,17 +1,17 @@
use jiff::{Timestamp, ZonedDateTime, tz::TimeZone};
use jiff::{Timestamp, Zoned, tz::TimeZone};
use serde::{Deserialize, Deserializer, Serializer};
pub(crate) mod zoned {
use super::*;
pub(crate) fn serialize<S>(value: &ZonedDateTime, serializer: S) -> Result<S::Ok, S::Error>
pub(crate) fn serialize<S>(value: &Zoned, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&value.to_string())
}
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<ZonedDateTime, D::Error>
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Zoned, D::Error>
where
D: Deserializer<'de>,
{
@@ -23,7 +23,7 @@ pub(crate) mod zoned {
pub(crate) mod option_zoned {
use super::*;
pub(crate) fn serialize<S>(value: &Option<ZonedDateTime>, serializer: S) -> Result<S::Ok, S::Error>
pub(crate) fn serialize<S>(value: &Option<Zoned>, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
@@ -33,7 +33,7 @@ pub(crate) mod option_zoned {
}
}
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Option<ZonedDateTime>, D::Error>
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Option<Zoned>, D::Error>
where
D: Deserializer<'de>,
{
@@ -44,15 +44,15 @@ pub(crate) mod option_zoned {
}
}
fn parse_zoned_compat(value: &str) -> Result<ZonedDateTime, String> {
if let Ok(zoned) = value.parse::<ZonedDateTime>() {
fn parse_zoned_compat(value: &str) -> Result<Zoned, String> {
if let Ok(zoned) = value.parse::<Zoned>() {
return Ok(zoned);
}
let timestamp = value
.parse::<Timestamp>()
.map_err(|err| format!("failed to parse legacy timestamp '{value}': {err}"))?;
Ok(timestamp.to_zoned_datetime(TimeZone::UTC))
Ok(timestamp.to_zoned(TimeZone::UTC))
}
#[cfg(test)]
+13 -13
View File
@@ -14,7 +14,7 @@
//! Core type definitions for KMS operations
use jiff::ZonedDateTime;
use jiff::Zoned;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use uuid::Uuid;
@@ -37,7 +37,7 @@ pub struct DataKeyInfo {
/// Associated metadata
pub metadata: HashMap<String, String>,
/// Key creation timestamp
pub created_at: ZonedDateTime,
pub created_at: Zoned,
}
impl DataKeyInfo {
@@ -61,7 +61,7 @@ impl DataKeyInfo {
ciphertext,
key_spec,
metadata: HashMap::new(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
}
}
@@ -112,14 +112,14 @@ pub struct MasterKeyInfo {
/// Associated metadata
pub metadata: HashMap<String, String>,
/// Key creation timestamp
pub created_at: ZonedDateTime,
pub created_at: Zoned,
/// Key last rotation timestamp
pub rotated_at: Option<ZonedDateTime>,
pub rotated_at: Option<Zoned>,
/// Key creator/owner
pub created_by: Option<String>,
/// Scheduled deletion deadline while the key is pending deletion
#[serde(default)]
pub deletion_date: Option<ZonedDateTime>,
pub deletion_date: Option<Zoned>,
}
impl MasterKeyInfo {
@@ -142,7 +142,7 @@ impl MasterKeyInfo {
status: KeyStatus::Active,
description: None,
metadata: HashMap::new(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
rotated_at: None,
created_by,
deletion_date: None,
@@ -174,7 +174,7 @@ impl MasterKeyInfo {
status: KeyStatus::Active,
description,
metadata: HashMap::new(),
created_at: ZonedDateTime::now(),
created_at: Zoned::now(),
rotated_at: None,
created_by,
deletion_date: None,
@@ -247,9 +247,9 @@ pub struct KeyInfo {
/// Key tags
pub tags: HashMap<String, String>,
/// Key creation timestamp
pub created_at: ZonedDateTime,
pub created_at: Zoned,
/// Key last rotation timestamp
pub rotated_at: Option<ZonedDateTime>,
pub rotated_at: Option<Zoned>,
/// Key creator
pub created_by: Option<String>,
/// Whether the key has outlived the configured rotation age.
@@ -690,7 +690,7 @@ pub struct EncryptionMetadata {
/// Encryption context
pub encryption_context: HashMap<String, String>,
/// Timestamp when encrypted
pub encrypted_at: ZonedDateTime,
pub encrypted_at: Zoned,
/// Size of original data
pub original_size: u64,
/// Encrypted data key
@@ -789,9 +789,9 @@ pub struct KeyMetadata {
/// Key description
pub description: Option<String>,
/// Key creation timestamp
pub creation_date: ZonedDateTime,
pub creation_date: Zoned,
/// Key deletion timestamp
pub deletion_date: Option<ZonedDateTime>,
pub deletion_date: Option<Zoned>,
/// Key origin
pub origin: String,
/// Key manager
+4 -4
View File
@@ -39,7 +39,7 @@ use std::sync::Arc;
use std::time::Duration;
use common::{TestKms, assert_key_not_found, assert_unsupported_capability, ctx};
use jiff::ZonedDateTime;
use jiff::Zoned;
use rustfs_kms::backends::local::LocalKmsBackend;
use rustfs_kms::backends::static_kms::StaticKmsBackend;
use rustfs_kms::backends::{ExpiredKeyRemoval, KmsBackend};
@@ -79,8 +79,8 @@ async fn schedule(backend: &LocalKmsBackend, key_id: &str, days: u32) {
.expect("deletion should be scheduled");
}
fn days_from_now(days: u64) -> ZonedDateTime {
ZonedDateTime::now() + Duration::from_secs(days * 86_400)
fn days_from_now(days: u64) -> Zoned {
Zoned::now() + Duration::from_secs(days * 86_400)
}
async fn exists(backend: &LocalKmsBackend, key_id: &str) -> bool {
@@ -422,7 +422,7 @@ async fn a_backend_without_deletion_support_reports_the_capability_gap() {
"the static backend must not advertise deletion scheduling"
);
assert_unsupported_capability(
backend.remove_expired_key(common::STATIC_KEY_ID, &ZonedDateTime::now()).await,
backend.remove_expired_key(common::STATIC_KEY_ID, &Zoned::now()).await,
"remove_expired_key",
);
}
+1 -1
View File
@@ -39,7 +39,7 @@ hotpath-alloc = ["hotpath"]
hotpath-cpu = ["hotpath"]
[dependencies]
jiff = { workspace = true, features = ["serde"] }
chrono = { workspace = true, features = ["serde"] }
flate2 = { workspace = true }
regex = { workspace = true }
serde = { workspace = true, features = ["derive"] }
+36 -41
View File
@@ -20,8 +20,7 @@ use crate::ingest::{IngestReport, SkipReason};
use crate::model::{EventKind, LogEvent, LogLevel, ParseStats};
use crate::redact;
use crate::rules::{Finding, FindingsCollector, RuleEngine, Severity};
use crate::time::Rfc3339;
use jiff::{SignedDuration, ZonedDateTime};
use chrono::{DateTime, Duration, FixedOffset, Utc};
use regex::Regex;
use serde::Serialize;
use std::collections::{BTreeMap, BTreeSet, HashMap};
@@ -36,9 +35,9 @@ const BUCKET_WIDTHS_MIN: [i64; 6] = [1, 5, 15, 60, 360, 1440];
#[derive(Debug, Clone)]
pub struct AnalyzeOptions {
/// Keep events at/after this time. Timestamp-less events always pass.
pub since: Option<ZonedDateTime>,
pub since: Option<DateTime<FixedOffset>>,
/// Keep events at/before this time. Timestamp-less events always pass.
pub until: Option<ZonedDateTime>,
pub until: Option<DateTime<FixedOffset>>,
/// Drop events below this level; level-less events always pass.
pub min_level: Option<LogLevel>,
pub max_samples: usize,
@@ -65,7 +64,7 @@ pub struct Summary {
pub events_after_filter: u64,
/// "ERROR" -> n, …, "no_level" -> n.
pub level_counts: BTreeMap<String, u64>,
pub time_range: Option<(ZonedDateTime, ZonedDateTime)>,
pub time_range: Option<(DateTime<FixedOffset>, DateTime<FixedOffset>)>,
pub nodes: Vec<String>,
pub parse: ParseStats,
pub files_parsed: u64,
@@ -89,7 +88,7 @@ pub struct UnmatchedCluster {
#[derive(Debug, Clone, Serialize)]
pub struct TimeBucket {
/// Bucket start, normalized to UTC.
pub start: ZonedDateTime,
pub start: DateTime<Utc>,
pub error: u64,
pub warn: u64,
pub other: u64,
@@ -106,8 +105,8 @@ pub struct TimelineAnomaly {
/// Involved node labels (node_ranges_disjoint); empty otherwise.
pub nodes: Vec<String>,
/// Gap boundaries in UTC (log_gap); null otherwise.
pub gap_start: Option<ZonedDateTime>,
pub gap_end: Option<ZonedDateTime>,
pub gap_start: Option<DateTime<Utc>>,
pub gap_end: Option<DateTime<Utc>>,
/// log_gap only: a startup-class finding begins within 5min after the gap.
pub restart_evidence: bool,
}
@@ -130,7 +129,7 @@ pub struct AnalysisReport {
}
/// (min ts, max ts, timestamped event count) for one node label.
type NodeRange = (ZonedDateTime, ZonedDateTime, u64);
type NodeRange = (DateTime<FixedOffset>, DateTime<FixedOffset>, u64);
#[derive(Default)]
struct BucketCounts {
@@ -155,7 +154,7 @@ pub struct Analyzer {
events_total: u64,
events_after_filter: u64,
level_counts: BTreeMap<String, u64>,
time_range: Option<(ZonedDateTime, ZonedDateTime)>,
time_range: Option<(DateTime<FixedOffset>, DateTime<FixedOffset>)>,
nodes: BTreeSet<String>,
offsets: BTreeSet<String>,
/// node label -> (min ts, max ts, timestamped event count); feeds the
@@ -190,9 +189,8 @@ impl Analyzer {
pub fn observe(&mut self, mut event: LogEvent) {
self.events_total += 1;
if let Some(ts) = event.timestamp.as_ref()
&& (self.opts.since.as_ref().is_some_and(|since| ts < since)
|| self.opts.until.as_ref().is_some_and(|until| ts > until))
if let Some(ts) = event.timestamp
&& (self.opts.since.is_some_and(|since| ts < since) || self.opts.until.is_some_and(|until| ts > until))
{
return;
}
@@ -219,26 +217,23 @@ impl Analyzer {
self.nodes.insert(node.to_string());
}
if let Some(ts) = event.timestamp.clone() {
self.time_range = Some(match self.time_range.take() {
None => (ts.clone(), ts.clone()),
Some((first, last)) => (first.min(ts.clone()), last.max(ts.clone())),
if let Some(ts) = event.timestamp {
self.time_range = Some(match self.time_range {
None => (ts, ts),
Some((first, last)) => (first.min(ts), last.max(ts)),
});
self.offsets.insert(format_offset(&ts));
if let Some(node) = event.node.as_deref() {
self.node_ranges
.entry(node.to_string())
.and_modify(|(min, max, count)| {
*min = min.clone().min(ts.clone());
*max = max.clone().max(ts.clone());
*min = (*min).min(ts);
*max = (*max).max(ts);
*count += 1;
})
.or_insert((ts.clone(), ts.clone(), 1));
.or_insert((ts, ts, 1));
}
let bucket = self
.minute_buckets
.entry(ts.timestamp().as_second().div_euclid(60))
.or_default();
let bucket = self.minute_buckets.entry(ts.timestamp().div_euclid(60)).or_default();
match event.level {
Some(LogLevel::Error) => bucket.error += 1,
Some(LogLevel::Warn) => bucket.warn += 1,
@@ -361,8 +356,8 @@ impl Analyzer {
}
}
fn format_offset(ts: &ZonedDateTime) -> String {
let seconds = crate::time::offset_seconds(ts);
fn format_offset(ts: &DateTime<FixedOffset>) -> String {
let seconds = ts.offset().local_minus_utc();
if seconds == 0 {
return "Z".to_string();
}
@@ -395,7 +390,7 @@ fn merge_timeline(minute_buckets: &BTreeMap<i64, BucketCounts>) -> (Vec<TimeBuck
let buckets = merged
.into_iter()
.map(|(slot, counts)| TimeBucket {
start: crate::time::utc_from_unix(slot * width * 60),
start: DateTime::<Utc>::from_timestamp(slot * width * 60, 0).expect("valid timestamp"),
error: counts.error,
warn: counts.warn,
other: counts.other,
@@ -476,13 +471,13 @@ fn detect_timeline_anomalies(
// span of at least max(15min, 3x display bucket width), then activity
// again — the shape of a process restart or hang.
let threshold = (3 * bucket_width_min).max(15);
let startup_after_gap = |gap_end: &ZonedDateTime| {
let startup_after_gap = |gap_end: DateTime<Utc>| {
findings.iter().chain(low_confidence).find_map(|f| {
if !STARTUP_RULE_IDS.contains(&f.rule_id.as_str()) {
return None;
}
let first = f.first_seen.as_ref()?;
(first >= gap_end && first <= gap_end + SignedDuration::from_mins(5)).then(|| f.rule_id.clone())
let first = f.first_seen?.with_timezone(&Utc);
(first >= gap_end && first <= gap_end + Duration::minutes(5)).then(|| f.rule_id.clone())
})
};
let minutes: Vec<i64> = minute_buckets.keys().copied().collect();
@@ -495,9 +490,9 @@ fn detect_timeline_anomalies(
continue;
}
if run_len >= 3 && delta > threshold {
let gap_start = crate::time::utc_from_unix((prev + 1) * 60);
let gap_end = crate::time::utc_from_unix(next * 60);
let restart = startup_after_gap(&gap_end);
let gap_start = DateTime::<Utc>::from_timestamp((prev + 1) * 60, 0).expect("valid timestamp");
let gap_end = DateTime::<Utc>::from_timestamp(next * 60, 0).expect("valid timestamp");
let restart = startup_after_gap(gap_end);
let span = human_duration_minutes(delta - 1);
let message = match &restart {
Some(rule_id) => format!(
@@ -550,7 +545,7 @@ fn collapse_findings(findings: &mut [Finding]) {
let index_of: HashMap<String, usize> = findings.iter().enumerate().map(|(i, f)| (f.rule_id.clone(), i)).collect();
// Among several qualifying roots pick the earliest first_seen;
// timestamp-less roots come last, rule id breaks remaining ties.
let root_key = |f: &Finding| (f.first_seen.is_none(), f.first_seen.clone(), f.rule_id.clone());
let root_key = |f: &Finding| (f.first_seen.is_none(), f.first_seen, f.rule_id.clone());
let mut chosen: Vec<Option<usize>> = vec![None; findings.len()];
for (i, symptom) in findings.iter().enumerate() {
@@ -560,10 +555,10 @@ fn collapse_findings(findings: &mut [Finding]) {
continue;
}
let root = &findings[r];
let qualifies = match (root.first_seen.as_ref(), symptom.first_seen.as_ref()) {
let qualifies = match (root.first_seen, symptom.first_seen) {
(Some(root_first), Some(symptom_first)) => {
root_first <= symptom_first + SignedDuration::from_mins(5)
&& root.last_seen.as_ref().unwrap_or(root_first) >= symptom_first - SignedDuration::from_mins(30)
root_first <= symptom_first + Duration::minutes(5)
&& root.last_seen.unwrap_or(root_first) >= symptom_first - Duration::minutes(30)
}
_ => true,
};
@@ -704,7 +699,7 @@ mod tests {
fn event(message: &str, level: Option<LogLevel>, ts: Option<&str>) -> LogEvent {
LogEvent {
timestamp: ts.map(|t| crate::time::parse_rfc3339(t).expect("ts")),
timestamp: ts.map(|t| DateTime::parse_from_rfc3339(t).expect("ts")),
level,
target: Some("rustfs::server::http".to_string()),
message: message.to_string(),
@@ -820,8 +815,8 @@ mod tests {
assert_eq!(gaps.len(), 1, "{:?}", report.timeline_anomalies);
assert!(gaps[0].restart_evidence, "startup-fatal right after the gap: {:?}", gaps[0]);
assert!(gaps[0].message.contains("重启证据"));
assert_eq!(gaps[0].gap_start.as_ref().expect("start").to_rfc3339(), "2026-07-15T03:04:00+00:00");
assert_eq!(gaps[0].gap_end.as_ref().expect("end").to_rfc3339(), "2026-07-15T03:44:00+00:00");
assert_eq!(gaps[0].gap_start.expect("start").to_rfc3339(), "2026-07-15T03:04:00+00:00");
assert_eq!(gaps[0].gap_end.expect("end").to_rfc3339(), "2026-07-15T03:44:00+00:00");
// Same hole but no startup finding afterwards: still a gap, no upgrade.
let mut a = analyzer(AnalyzeOptions::default());
@@ -1016,7 +1011,7 @@ mod tests {
#[test]
fn since_until_filters_timestamped_events_only() {
let mut a = analyzer(AnalyzeOptions {
since: Some(crate::time::parse_rfc3339("2026-07-15T03:00:00+08:00").expect("ts")),
since: Some(DateTime::parse_from_rfc3339("2026-07-15T03:00:00+08:00").expect("ts")),
..Default::default()
});
a.observe(event("Disk full", Some(LogLevel::Error), Some("2026-07-15T02:00:00+08:00")));
-1
View File
@@ -27,7 +27,6 @@ pub mod parse;
mod redact;
pub mod report;
pub mod rules;
pub(crate) mod time;
pub use analyze::{AnalysisReport, AnalyzeOptions, Analyzer, Summary, TimeBucket, TimelineAnomaly, UnmatchedCluster};
pub use ingest::{IngestOptions, IngestReport, SkipReason, ingest_path, ingest_reader};
+2 -2
View File
@@ -14,7 +14,7 @@
//! Unified event model shared by every stage of the analyzer.
use jiff::ZonedDateTime;
use chrono::{DateTime, FixedOffset};
use serde::{Deserialize, Serialize, Serializer};
use std::fmt;
use std::sync::Arc;
@@ -102,7 +102,7 @@ pub enum EventKind {
#[derive(Debug, Clone, Serialize)]
pub struct LogEvent {
/// RFC-3339 timestamp with offset. None for panic/text lines that carry none.
pub timestamp: Option<ZonedDateTime>,
pub timestamp: Option<DateTime<FixedOffset>>,
/// None when the line carries no recognizable level (text lines).
pub level: Option<LogLevel>,
/// tracing `target`, e.g. "rustfs::scanner::io". None for panic/text.
+1 -1
View File
@@ -36,7 +36,7 @@ pub(crate) fn try_parse(line: &str, file: &Arc<str>, line_no: u64, node: &Option
}
let timestamp = match map.remove("timestamp") {
Some(Value::String(raw)) => match crate::time::parse_rfc3339(&raw) {
Some(Value::String(raw)) => match chrono::DateTime::parse_from_rfc3339(&raw) {
Ok(ts) => Some(ts),
Err(_) => {
map.insert("timestamp_raw".to_string(), Value::String(raw));
+2 -2
View File
@@ -156,8 +156,8 @@ mod tests {
assert_eq!(ev.level, Some(LogLevel::Error));
assert_eq!(ev.target.as_deref(), Some("rustfs::scanner::io"));
assert_eq!(ev.message, "Disk health check marked disk faulty");
let ts = ev.timestamp.as_ref().expect("timestamp");
assert_eq!(crate::time::offset_seconds(ts), 8 * 3600);
let ts = ev.timestamp.expect("timestamp");
assert_eq!(ts.offset().local_minus_utc(), 8 * 3600);
assert_eq!(ev.field_str("reason"), Some("faulty_disk"));
assert_eq!(ev.field_str("disk"), Some("/data/disk3"));
assert!(ev.fields.contains_key("filename"));
+2 -3
View File
@@ -17,7 +17,6 @@
use super::{human_bytes, parse_percentage};
use crate::analyze::AnalysisReport;
use crate::time::Rfc3339;
use std::io;
/// GFM parses a table row into cells before inline code, so a literal `|`
@@ -34,7 +33,7 @@ pub(super) fn render(report: &AnalysisReport, w: &mut dyn io::Write) -> io::Resu
writeln!(w)?;
writeln!(w, "| 项 | 值 |")?;
writeln!(w, "|---|---|")?;
match report.summary.time_range.as_ref() {
match report.summary.time_range {
Some((first, last)) => writeln!(w, "| 时间范围 | {} → {} |", first.to_rfc3339(), last.to_rfc3339())?,
None => writeln!(w, "| 时间范围 | (无带时间戳的事件) |")?,
}
@@ -105,7 +104,7 @@ pub(super) fn render(report: &AnalysisReport, w: &mut dyn io::Write) -> io::Resu
finding.count
)?;
writeln!(w)?;
if let (Some(first), Some(last)) = (finding.first_seen.as_ref(), finding.last_seen.as_ref()) {
if let (Some(first), Some(last)) = (finding.first_seen, finding.last_seen) {
writeln!(w, "- 首次: {} · 末次: {}", first.to_rfc3339(), last.to_rfc3339())?;
}
if !finding.caused.is_empty() {
+4 -4
View File
@@ -74,13 +74,13 @@ mod tests {
use crate::ingest::IngestReport;
use crate::model::{EventKind, LogEvent, LogLevel, SourceRef};
use crate::rules::{RuleEngine, seed_rule_set};
use crate::time::parse_rfc3339;
use chrono::DateTime;
use std::sync::Arc;
fn fixed_report() -> AnalysisReport {
let mut analyzer = Analyzer::new(RuleEngine::new(seed_rule_set()), AnalyzeOptions::default());
let mut ev = LogEvent {
timestamp: Some(parse_rfc3339("2026-07-15T03:12:44+08:00").expect("ts")),
timestamp: Some(DateTime::parse_from_rfc3339("2026-07-15T03:12:44+08:00").expect("ts")),
level: Some(LogLevel::Error),
target: Some("rustfs_ecstore".to_string() + "::erasure"),
message: "erasure write quorum (required=8, achieved=5, failed=3)".to_string(),
@@ -96,11 +96,11 @@ mod tests {
.insert("required".to_string(), serde_json::Value::String("8".into()));
analyzer.observe(ev.clone());
analyzer.observe(LogEvent {
timestamp: Some(parse_rfc3339("2026-07-15T03:20:44+08:00").expect("ts")),
timestamp: Some(DateTime::parse_from_rfc3339("2026-07-15T03:20:44+08:00").expect("ts")),
..ev
});
analyzer.observe(LogEvent {
timestamp: Some(parse_rfc3339("2026-07-15T03:25:00+08:00").expect("ts")),
timestamp: Some(DateTime::parse_from_rfc3339("2026-07-15T03:25:00+08:00").expect("ts")),
level: Some(LogLevel::Error),
target: None,
message: "failed to sync /data/x9 after 5 retries".to_string(),
+4 -8
View File
@@ -17,7 +17,6 @@
use super::{human_bytes, human_duration, parse_percentage};
use crate::analyze::AnalysisReport;
use crate::rules::Finding;
use crate::time::Rfc3339;
use std::io;
const HEAVY_RULE: &str = "════════════════════════════════════════";
@@ -27,9 +26,9 @@ pub(super) fn render(report: &AnalysisReport, w: &mut dyn io::Write) -> io::Resu
writeln!(w, "RustFS 日志诊断报告")?;
writeln!(w, "{HEAVY_RULE}")?;
match report.summary.time_range.as_ref() {
match report.summary.time_range {
Some((first, last)) => {
let span = human_duration(last.duration_since(first).as_secs());
let span = human_duration(last.signed_duration_since(first).num_seconds());
writeln!(w, "时间范围 {} → {} ({span})", first.to_rfc3339(), last.to_rfc3339())?;
}
None => writeln!(w, "时间范围 (无带时间戳的事件)")?,
@@ -158,7 +157,7 @@ fn render_finding(finding: &Finding, counts: &std::collections::HashMap<&str, u6
finding.count,
nodes
)?;
if let (Some(first), Some(last)) = (finding.first_seen.as_ref(), finding.last_seen.as_ref()) {
if let (Some(first), Some(last)) = (finding.first_seen, finding.last_seen) {
writeln!(w, " 首次 {} 末次 {}", first.to_rfc3339(), last.to_rfc3339())?;
}
if !finding.caused.is_empty() {
@@ -181,10 +180,7 @@ fn render_finding(finding: &Finding, counts: &std::collections::HashMap<&str, u6
writeln!(w, " 证据: {field} = {}{overflow}", shown.join(", "))?;
}
for sample in &finding.samples {
let ts = sample
.timestamp
.as_ref()
.map_or_else(|| "(无时间戳)".to_string(), |t| t.to_rfc3339());
let ts = sample.timestamp.map_or_else(|| "(无时间戳)".to_string(), |t| t.to_rfc3339());
let node = sample.node.as_deref().unwrap_or("-");
writeln!(w, " 样本: {ts} {node} {}:{}", sample.source.file, sample.source.line)?;
writeln!(w, " {}", sample.message)?;
+13 -13
View File
@@ -18,7 +18,7 @@
use super::model::{Rule, Severity};
use crate::model::LogEvent;
use jiff::ZonedDateTime;
use chrono::{DateTime, FixedOffset};
use serde::Serialize;
use std::collections::{BTreeMap, BTreeSet};
@@ -44,8 +44,8 @@ pub struct Finding {
pub suggestion: String,
pub count: u64,
/// Earliest / latest timestamps among hits that carry one.
pub first_seen: Option<ZonedDateTime>,
pub last_seen: Option<ZonedDateTime>,
pub first_seen: Option<DateTime<FixedOffset>>,
pub last_seen: Option<DateTime<FixedOffset>>,
/// Node labels of hits; label-less events are recorded as "-".
pub nodes: BTreeSet<String>,
/// The earliest `max_samples` hit events (timestamp-less ones sort last).
@@ -79,28 +79,28 @@ pub struct FindingsCollector {
/// Sort key for sample selection: earliest first, timestamp-less last,
/// (file, line) as the deterministic tiebreaker — line numbers alone are only
/// unique within one file, so ties across inputs would be order-dependent.
fn sample_key(event: &LogEvent) -> (bool, Option<ZonedDateTime>, &str, u64) {
(event.timestamp.is_none(), event.timestamp.clone(), &event.source.file, event.source.line)
fn sample_key(event: &LogEvent) -> (bool, Option<DateTime<FixedOffset>>, &str, u64) {
(event.timestamp.is_none(), event.timestamp, &event.source.file, event.source.line)
}
/// `ZonedDateTime` orders by instant, so two equal instants carrying
/// `DateTime<FixedOffset>` orders by instant, so two equal instants carrying
/// different offsets compare equal and `min`/`max` keep whichever arrived
/// first — an input-order dependency that then leaks into the serialized
/// RFC3339 offset. Break the tie on the offset itself to stay deterministic.
fn earliest(a: ZonedDateTime, b: ZonedDateTime) -> ZonedDateTime {
fn earliest(a: DateTime<FixedOffset>, b: DateTime<FixedOffset>) -> DateTime<FixedOffset> {
match a.cmp(&b) {
std::cmp::Ordering::Less => a,
std::cmp::Ordering::Greater => b,
std::cmp::Ordering::Equal if crate::time::offset_seconds(&a) <= crate::time::offset_seconds(&b) => a,
std::cmp::Ordering::Equal if a.offset().local_minus_utc() <= b.offset().local_minus_utc() => a,
std::cmp::Ordering::Equal => b,
}
}
fn latest(a: ZonedDateTime, b: ZonedDateTime) -> ZonedDateTime {
fn latest(a: DateTime<FixedOffset>, b: DateTime<FixedOffset>) -> DateTime<FixedOffset> {
match a.cmp(&b) {
std::cmp::Ordering::Greater => a,
std::cmp::Ordering::Less => b,
std::cmp::Ordering::Equal if crate::time::offset_seconds(&a) >= crate::time::offset_seconds(&b) => a,
std::cmp::Ordering::Equal if a.offset().local_minus_utc() >= b.offset().local_minus_utc() => a,
std::cmp::Ordering::Equal => b,
}
}
@@ -140,9 +140,9 @@ impl FindingsCollector {
let finding = &mut acc.finding;
finding.count += 1;
if let Some(ts) = event.timestamp.clone() {
finding.first_seen = Some(finding.first_seen.take().map_or(ts.clone(), |cur| earliest(cur, ts.clone())));
finding.last_seen = Some(finding.last_seen.take().map_or(ts.clone(), |cur| latest(cur, ts)));
if let Some(ts) = event.timestamp {
finding.first_seen = Some(finding.first_seen.map_or(ts, |cur| earliest(cur, ts)));
finding.last_seen = Some(finding.last_seen.map_or(ts, |cur| latest(cur, ts)));
}
finding.nodes.insert(event.node.as_deref().unwrap_or("-").to_string());
+3 -4
View File
@@ -40,7 +40,6 @@ pub fn seed_rule_set() -> RuleSet {
mod tests {
use super::*;
use crate::model::{EventKind, LogEvent, LogLevel, SourceRef};
use crate::time::Rfc3339;
use std::sync::Arc;
fn event(message: &str, level: Option<LogLevel>, line: u64) -> LogEvent {
@@ -61,7 +60,7 @@ mod tests {
fn event_at(message: &str, ts: &str, line: u64) -> LogEvent {
LogEvent {
timestamp: Some(crate::time::parse_rfc3339(ts).expect("ts")),
timestamp: Some(chrono::DateTime::parse_from_rfc3339(ts).expect("ts")),
..event(message, Some(LogLevel::Error), line)
}
}
@@ -225,8 +224,8 @@ mod tests {
};
let f = &findings[0];
assert_eq!(f.count, 5);
assert_eq!(f.first_seen.as_ref().expect("first").to_rfc3339(), "2026-07-15T01:00:00+08:00");
assert_eq!(f.last_seen.as_ref().expect("last").to_rfc3339(), "2026-07-15T04:00:00+08:00");
assert_eq!(f.first_seen.expect("first").to_rfc3339(), "2026-07-15T01:00:00+08:00");
assert_eq!(f.last_seen.expect("last").to_rfc3339(), "2026-07-15T04:00:00+08:00");
let sample_messages: Vec<_> = f.samples.iter().map(|s| s.message.as_str()).collect();
assert_eq!(sample_messages, vec!["quorum e2", "quorum e3", "quorum e1"]);
}
-38
View File
@@ -1,38 +0,0 @@
use jiff::{Timestamp, ZonedDateTime, tz::TimeZone};
pub(crate) trait Rfc3339 {
fn to_rfc3339(&self) -> String;
fn format(&self, pattern: &str) -> String;
}
impl Rfc3339 for ZonedDateTime {
fn to_rfc3339(&self) -> String {
format_rfc3339(self)
}
fn format(&self, pattern: &str) -> String {
self.strftime(pattern).to_string()
}
}
pub(crate) fn parse_rfc3339(value: &str) -> Result<ZonedDateTime, jiff::Error> {
if let Some(prefix) = value.strip_suffix('Z') {
let normalized = format!("{prefix}+00:00");
return ZonedDateTime::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", normalized);
}
ZonedDateTime::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", value)
}
pub(crate) fn format_rfc3339(value: &ZonedDateTime) -> String {
value.strftime("%Y-%m-%dT%H:%M:%S%.f%:z").to_string()
}
pub(crate) fn utc_from_unix(seconds: i64) -> ZonedDateTime {
Timestamp::new(seconds, 0)
.expect("unix timestamp must fit jiff")
.to_zoned_datetime(TimeZone::UTC)
}
pub(crate) fn offset_seconds(value: &ZonedDateTime) -> i32 {
value.offset().seconds()
}
+4 -5
View File
@@ -30,13 +30,12 @@ workspace = true
[features]
default = []
# Kept as no-op compatibility features for dependents that forward the
# workspace profiling feature set to every crate.
hotpath = []
hotpath-alloc = []
hotpath-cpu = []
hotpath = ["hotpath/hotpath"]
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
[dependencies]
hotpath.workspace = true
humantime.workspace = true
http.workspace = true
hyper = { workspace = true, features = ["http2", "http1", "server"] }
+1
View File
@@ -74,6 +74,7 @@ jiff = { workspace = true, features = ["serde"] }
percent-encoding = { workspace = true }
rayon = { workspace = true }
rustc-hash = { workspace = true }
uuid = { workspace = true }
serde = { workspace = true, features = ["derive"] }
starshard = { workspace = true, features = ["rayon", "async", "serde"] }
thiserror = { workspace = true }
+19 -27
View File
@@ -13,9 +13,8 @@
// limitations under the License.
use crate::{
BucketNotificationConfig, NotificationError, config_manager::notify_configuration_hint,
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
rules::ParseConfigError,
BucketNotificationConfig, NotificationError, config_manager::notify_configuration_hint, notifier::EventNotifier,
rule_engine::NotifyRuleEngine, rules::ParseConfigError,
};
use rustfs_s3_types::EventName;
use std::sync::Arc;
@@ -30,26 +29,14 @@ const EVENT_NOTIFY_BUCKET_CONFIG_LOADED: &str = "notify_bucket_config_loaded";
pub struct NotifyBucketConfigManager {
notifier: Arc<EventNotifier>,
rule_engine: NotifyRuleEngine,
subscriber_view: Arc<NotificationSystemSubscriberView>,
}
impl NotifyBucketConfigManager {
pub fn new(
notifier: Arc<EventNotifier>,
rule_engine: NotifyRuleEngine,
subscriber_view: Arc<NotificationSystemSubscriberView>,
) -> Self {
Self {
notifier,
rule_engine,
subscriber_view,
}
pub fn new(notifier: Arc<EventNotifier>, rule_engine: NotifyRuleEngine) -> Self {
Self { notifier, rule_engine }
}
pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
if !self.subscriber_view.has_subscriber(bucket, event) {
return false;
}
self.rule_engine.has_subscriber(bucket, event).await
}
@@ -105,7 +92,6 @@ impl NotifyBucketConfigManager {
/// Publishes an already validated `cfg` to the runtime rule state.
pub async fn apply_bucket_notification_config(&self, bucket: &str, cfg: &BucketNotificationConfig) {
self.subscriber_view.apply_bucket_config(bucket, cfg);
self.rule_engine.set_bucket_rules(bucket, cfg.get_rules_map().clone()).await;
info!(
event = EVENT_NOTIFY_BUCKET_CONFIG_LOADED,
@@ -128,8 +114,16 @@ impl NotifyBucketConfigManager {
Ok(())
}
pub(crate) fn publish_bucket_notification_config(&self, bucket: &str, cfg: Option<(uuid::Uuid, &BucketNotificationConfig)>) {
if let Some((incarnation_id, cfg)) = cfg {
self.rule_engine
.publish_bucket_rules(bucket, Some(incarnation_id), cfg.get_rules_map().clone());
} else {
self.rule_engine.withdraw_bucket_rules(bucket);
}
}
pub async fn remove_bucket_notification_config(&self, bucket: &str) {
self.subscriber_view.clear_bucket(bucket);
self.rule_engine.clear_bucket_rules(bucket).await;
}
}
@@ -138,8 +132,8 @@ impl NotifyBucketConfigManager {
mod tests {
use super::NotifyBucketConfigManager;
use crate::{
BucketNotificationConfig, NotificationError, integration::NotificationMetrics,
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
BucketNotificationConfig, NotificationError, integration::NotificationMetrics, notifier::EventNotifier,
rule_engine::NotifyRuleEngine,
};
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
@@ -149,8 +143,7 @@ mod tests {
let metrics = Arc::new(NotificationMetrics::new());
let rule_engine = NotifyRuleEngine::new();
let notifier = Arc::new(EventNotifier::new(metrics, rule_engine.clone()));
let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
NotifyBucketConfigManager::new(notifier, rule_engine, subscriber_view)
NotifyBucketConfigManager::new(notifier, rule_engine)
}
#[tokio::test]
@@ -166,11 +159,11 @@ mod tests {
let mut cfg = BucketNotificationConfig::new("us-east-1");
cfg.add_rule(&[EventName::ObjectCreatedPut], "*".to_string(), target_id);
manager.subscriber_view.apply_bucket_config("bucket", &cfg);
assert!(manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
manager.apply_bucket_notification_config("bucket", &cfg).await;
assert!(manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
manager.remove_bucket_notification_config("bucket").await;
assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
}
#[tokio::test]
@@ -186,7 +179,6 @@ mod tests {
.expect_err("validation must fail when the runtime has no notify targets");
assert!(matches!(err, NotificationError::Configuration(_)));
assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
}
#[tokio::test]
+32 -1
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::rules::TargetIdSet;
use hashbrown::HashMap;
use jiff::{Timestamp, tz::TimeZone};
use rustfs_s3_ops::is_object_removed_event;
@@ -170,6 +171,10 @@ pub struct Event {
pub glacier_event_data: Option<GlacierEventData>,
/// Information about the source of the event
pub source: Source,
#[serde(skip)]
pub(crate) bucket_incarnation_id: Option<uuid::Uuid>,
#[serde(skip)]
pub(crate) bucket_removed_targets: Option<TargetIdSet>,
}
impl Event {
@@ -195,6 +200,8 @@ impl Event {
user_metadata.insert("x-request-time".to_string(), format_timestamp_rfc3339_millis(&Timestamp::now()));
Event {
bucket_incarnation_id: None,
bucket_removed_targets: None,
event_version: event_schema_version(event_name).to_string(),
event_source: "rustfs:s3".to_string(),
aws_region: "us-east-1".to_string(),
@@ -320,6 +327,8 @@ impl Event {
};
Self {
bucket_incarnation_id: args.bucket_incarnation_id,
bucket_removed_targets: args.bucket_removed_targets,
event_version: event_schema_version(args.event_name).to_string(),
event_source: "rustfs:s3".to_string(),
aws_region: args.req_params.get("region").cloned().unwrap_or_default(),
@@ -359,7 +368,7 @@ where
}
fn format_timestamp_rfc3339_millis(value: &Timestamp) -> String {
let utc = value.to_zoned_datetime(TimeZone::UTC);
let utc = value.to_zoned(TimeZone::UTC);
format!(
"{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:03}Z",
utc.year(),
@@ -387,6 +396,10 @@ fn initialize_response_elements(elements: &mut HashMap<String, String>, keys: &[
#[derive(Debug, Clone)]
pub struct EventArgs {
/// Bucket identity established by the producing operation.
pub bucket_incarnation_id: Option<uuid::Uuid>,
/// Delivery targets captured before a successful bucket removal withdraws its rules.
pub bucket_removed_targets: Option<TargetIdSet>,
pub event_name: EventName,
pub bucket_name: String,
pub object: NotifyObjectInfo,
@@ -436,6 +449,8 @@ impl EventArgs {
/// ```
#[derive(Debug, Clone, Default)]
pub struct EventArgsBuilder {
bucket_incarnation_id: Option<uuid::Uuid>,
bucket_removed_targets: Option<TargetIdSet>,
event_name: EventName,
bucket_name: String,
object: NotifyObjectInfo,
@@ -458,6 +473,18 @@ impl EventArgsBuilder {
}
}
/// Carries the operation's confirmed bucket identity to event dispatch.
pub fn bucket_incarnation_id(mut self, incarnation_id: Option<uuid::Uuid>) -> Self {
self.bucket_incarnation_id = incarnation_id;
self
}
/// Uses the subscription targets captured before deletion.
pub fn bucket_removed_targets(mut self, targets: TargetIdSet) -> Self {
self.bucket_removed_targets = Some(targets);
self
}
/// Sets the event name.
pub fn event_name(mut self, event_name: EventName) -> Self {
self.event_name = event_name;
@@ -529,6 +556,8 @@ impl EventArgsBuilder {
/// This method consumes the builder and returns the constructed `EventArgs`.
pub fn build(self) -> EventArgs {
EventArgs {
bucket_incarnation_id: self.bucket_incarnation_id,
bucket_removed_targets: self.bucket_removed_targets,
event_name: self.event_name,
bucket_name: self.bucket_name,
object: self.object,
@@ -823,6 +852,8 @@ mod event_args_tests {
req_params.insert((*k).to_string(), (*v).to_string());
}
EventArgs {
bucket_incarnation_id: None,
bucket_removed_targets: None,
event_name: EventName::ObjectRemovedDelete,
bucket_name: "b".to_string(),
object: ObjectInfo::default(),
+2 -1
View File
@@ -239,7 +239,8 @@ pub mod notifier_global {
.await
}
fn bucket_notification_config_for_rules(
/// Compiles event rules into the bucket configuration used by validation and publication.
pub fn bucket_notification_config_for_rules(
region: &str,
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
) -> BucketNotificationConfig {
+7 -3
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::notification_system_subscriber::NotificationSystemSubscriberView;
use crate::notifier::{EventNotifier, TargetList};
use crate::services::NotifyServices;
use crate::{
@@ -182,7 +181,6 @@ impl NotificationSystem {
rustfs_utils::get_env_usize(ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY);
let (live_event_sender, _) = broadcast::channel(1024);
let metrics = Arc::new(NotificationMetrics::new());
let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
let rule_engine = NotifyRuleEngine::new();
let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
let target_list = notifier.target_list();
@@ -200,7 +198,6 @@ impl NotificationSystem {
stream_cancellers,
concurrency_limiter,
metrics,
subscriber_view,
live_event_sender,
live_event_history,
);
@@ -413,6 +410,13 @@ impl NotificationSystem {
.await
}
/// Applies the current committed bucket rules while the caller holds the metadata publication fence.
pub fn publish_bucket_notification_config(&self, bucket: &str, cfg: Option<(uuid::Uuid, &BucketNotificationConfig)>) {
self.services
.bucket_config_manager
.publish_bucket_notification_config(bucket, cfg);
}
/// Sends an event
pub async fn send_event(&self, event: Arc<Event>) {
self.services.pipeline.send_event(event).await;
-1
View File
@@ -26,7 +26,6 @@ pub mod factory;
mod global;
pub mod integration;
mod lifecycle;
mod notification_system_subscriber;
pub mod notifier;
mod pipeline;
pub mod registry;
@@ -1,74 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::BucketNotificationConfig;
use crate::rules::{BucketRulesSnapshot, DynRulesContainer, SubscriberIndex};
use rustfs_s3_types::EventName;
/// NotificationSystemSubscriberView - Provides an interface to manage and query
/// the subscription status of buckets in the notification system.
#[derive(Debug)]
pub struct NotificationSystemSubscriberView {
index: SubscriberIndex,
}
impl NotificationSystemSubscriberView {
/// Creates a new NotificationSystemSubscriberView with an empty SubscriberIndex.
///
/// Returns a new instance of NotificationSystemSubscriberView.
pub fn new() -> Self {
Self {
index: SubscriberIndex::default(),
}
}
/// Checks if a bucket has any subscribers for a specific event.
/// This is a quick check using the event mask in the snapshot.
///
/// # Arguments
/// * `bucket` - The name of the bucket to check.
/// * `event` - The event name to check for subscriptions.
///
/// Returns `true` if there are subscribers for the event, `false` otherwise.
#[inline]
pub fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
self.index.has_subscriber(bucket, event)
}
/// Builds and atomically replaces a bucket's subscription snapshot from the configuration.
///
/// Core principle: masks and rules are calculated and stored together in the same update.
///
/// # Arguments
/// * `bucket` - The name of the bucket to update.
/// * `cfg` - The bucket notification configuration to compile into a snapshot.
pub fn apply_bucket_config(&self, bucket: &str, cfg: &BucketNotificationConfig) {
// *It is recommended to merge compile into one function to ensure the same origin.
let snapshot: BucketRulesSnapshot<DynRulesContainer> = cfg.compile_snapshot();
// *debug to prevent inconsistencies from being introduced when modifying the compile logic in the future.
snapshot.debug_assert_mask_consistent();
self.index.store_snapshot(bucket, snapshot);
}
/// Clears a bucket's subscription snapshot.
///
/// #Arguments
/// * `bucket` - The name of the bucket to clear.
#[inline]
pub fn clear_bucket(&self, bucket: &str) {
self.index.clear_bucket(bucket);
}
}
+67 -2
View File
@@ -13,8 +13,9 @@
// limitations under the License.
use crate::error::NotificationError;
use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine, rules::TargetIdSet};
use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
use rustfs_s3_types::EventName;
use rustfs_targets::Target;
use rustfs_targets::arn::TargetID;
use rustfs_targets::target::EntityTarget;
@@ -235,6 +236,12 @@ impl EventNotifier {
);
}
/// Captures the deleted bucket's final delivery targets before its rules are withdrawn.
pub fn bucket_removed_targets(&self, bucket: &str, incarnation_id: uuid::Uuid) -> TargetIdSet {
self.rule_engine
.match_bucket_targets(bucket, Some(incarnation_id), EventName::BucketRemoved, "")
}
/// Sends an event to the appropriate targets based on the bucket rules
///
/// # Arguments
@@ -245,7 +252,25 @@ impl EventNotifier {
let object_key = &event.s3.object.key;
let event_name = event.event_name;
let target_ids = self.rule_engine.match_targets(bucket_name, event_name, object_key).await;
let target_ids = match &event.bucket_removed_targets {
Some(targets) if event_name == EventName::BucketRemoved => targets.clone(),
_ => {
let match_targets = || {
self.rule_engine
.match_bucket_targets(bucket_name, event.bucket_incarnation_id, event_name, object_key)
};
let targets = match_targets();
if targets.is_empty()
&& let Some(id) = event.bucket_incarnation_id
&& self.rule_engine.request_incarnation_refresh(bucket_name, id)
&& crate::storage_api::refresh_notify_bucket_metadata(bucket_name).await
{
match_targets()
} else {
targets
}
}
};
if target_ids.is_empty() {
debug!(
event = EVENT_NOTIFY_DISPATCH_SKIPPED,
@@ -1110,6 +1135,46 @@ mod tests {
assert_eq!(disabled_target.save_call_count(), 0);
}
#[tokio::test]
async fn bucket_removal_targets_survive_withdrawal_without_routing_recreated_objects() {
let engine = NotifyRuleEngine::new();
let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), engine.clone());
let target = MockTarget::new("removed-bucket", "webhook");
let old_id = uuid::Uuid::new_v4();
let new_id = uuid::Uuid::new_v4();
let mut rules = RulesMap::new();
rules.add_rule_config(
&[EventName::BucketRemoved, EventName::ObjectCreatedPut],
"*".to_owned(),
target.target_id(),
);
engine.publish_bucket_rules("bucket", Some(old_id), rules);
notifier
.target_list()
.write()
.await
.add(Arc::new(target.clone()))
.expect("register target");
let targets = notifier.bucket_removed_targets("bucket", old_id);
assert!(targets.contains(&target.target_id()));
engine.withdraw_bucket_rules("bucket");
let mut removed = Event::new_test_event("bucket", "", EventName::BucketRemoved);
removed.bucket_incarnation_id = Some(old_id);
removed.bucket_removed_targets = Some(targets);
let encoded = serde_json::to_value(&removed).expect("serialize webhook event");
assert!(encoded.get("bucketIncarnationId").is_none());
assert!(encoded.get("bucketRemovedTargets").is_none());
notifier.send(Arc::new(removed)).await;
assert_eq!(target.save_call_count(), 1, "the deleted bucket earns its final removal event");
let mut created = Event::new_test_event("bucket", "fresh.txt", EventName::ObjectCreatedPut);
created.bucket_incarnation_id = Some(new_id);
notifier.send(Arc::new(created)).await;
assert_eq!(target.save_call_count(), 1, "the new bucket has no subscription");
}
#[tokio::test]
async fn send_event_respects_prefix_suffix_filters() {
let rule_engine = NotifyRuleEngine::new();
+110 -27
View File
@@ -16,9 +16,13 @@ use crate::rules::{RulesMap, TargetIdSet};
use percent_encoding::percent_decode_str;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode};
use std::sync::Arc;
use starshard::{DEFAULT_SHARDS, ShardedHashMap, SnapshotMode};
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
use tracing::info;
use uuid::Uuid;
const LOG_COMPONENT_NOTIFY: &str = "notify";
const LOG_SUBSYSTEM_RULE_ENGINE: &str = "rule_engine";
@@ -33,34 +37,56 @@ fn decoded_object_key_for_matching(object_key: &str) -> Option<String> {
(decoded != object_key).then(|| decoded.into_owned())
}
struct BucketRules {
incarnation_id: Option<Uuid>,
rules: RulesMap,
refresh_requested: Arc<AtomicBool>,
}
#[derive(Clone)]
pub struct NotifyRuleEngine {
bucket_rules_map: Arc<AsyncShardedHashMap<String, RulesMap, rustc_hash::FxBuildHasher>>,
bucket_rules_map: Arc<ShardedHashMap<String, Arc<BucketRules>, rustc_hash::FxBuildHasher>>,
}
impl NotifyRuleEngine {
pub fn new() -> Self {
Self {
bucket_rules_map: Arc::new(AsyncShardedHashMap::with_snapshot_mode(DEFAULT_SHARDS, SnapshotMode::Cached)),
bucket_rules_map: Arc::new(ShardedHashMap::with_snapshot_mode(DEFAULT_SHARDS, SnapshotMode::Cached)),
}
}
pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool {
let items = self.bucket_rules_map.iter().await;
for (_bucket, rules_map) in items {
if rules_map.contains_target_id(target_id) {
return true;
}
}
false
self.bucket_rules_map
.iter()
.any(|(_, snapshot)| snapshot.rules.contains_target_id(target_id))
}
pub async fn set_bucket_rules(&self, bucket: &str, rules_map: RulesMap) {
self.publish_bucket_rules(bucket, None, rules_map);
}
/// Atomically publishes the rules and the identity used by event dispatch.
pub(crate) fn publish_bucket_rules(&self, bucket: &str, incarnation_id: Option<Uuid>, rules_map: RulesMap) {
let event_count = rules_map.iter_events().count();
let bucket_key = bucket.to_owned();
if rules_map.is_empty() {
self.bucket_rules_map.remove(&bucket.to_string()).await;
self.bucket_rules_map.remove(&bucket_key);
} else {
self.bucket_rules_map.insert(bucket.to_string(), rules_map).await;
// A reload that confirms the current incarnation must not re-arm
// the refresh triggered by delayed events from an older bucket.
let refresh_requested = self
.bucket_rules_map
.get(&bucket_key)
.filter(|current| current.incarnation_id == incarnation_id)
.map_or_else(|| Arc::new(AtomicBool::new(false)), |current| current.refresh_requested.clone());
self.bucket_rules_map.insert(
bucket_key,
Arc::new(BucketRules {
incarnation_id,
rules: rules_map,
refresh_requested,
}),
);
}
info!(
event = EVENT_NOTIFY_RULES_UPDATED,
@@ -74,11 +100,17 @@ impl NotifyRuleEngine {
}
pub async fn get_bucket_rules(&self, bucket: &str) -> Option<RulesMap> {
self.bucket_rules_map.get(&bucket.to_string()).await
self.bucket_rules_map
.get(&bucket.to_string())
.map(|snapshot| snapshot.rules.clone())
}
pub async fn clear_bucket_rules(&self, bucket: &str) {
if self.bucket_rules_map.remove(&bucket.to_string()).await.is_some() {
self.withdraw_bucket_rules(bucket);
}
pub(crate) fn withdraw_bucket_rules(&self, bucket: &str) {
if self.bucket_rules_map.remove(&bucket.to_string()).is_some() {
info!(
event = EVENT_NOTIFY_RULES_UPDATED,
component = LOG_COMPONENT_NOTIFY,
@@ -91,21 +123,43 @@ impl NotifyRuleEngine {
}
pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
self.get_bucket_rules(bucket)
.await
.is_some_and(|rules_map| rules_map.has_subscriber(event))
self.bucket_rules_map
.get(&bucket.to_string())
.is_some_and(|snapshot| snapshot.rules.has_subscriber(event))
}
pub async fn match_targets(&self, bucket: &str, event_name: EventName, object_key: &str) -> TargetIdSet {
self.get_bucket_rules(bucket)
.await
.map_or_else(TargetIdSet::new, |rules_map| {
let mut target_ids = rules_map.match_rules(event_name, object_key);
if let Some(decoded_key) = decoded_object_key_for_matching(object_key) {
target_ids.extend(rules_map.match_rules(event_name, &decoded_key));
}
target_ids
})
self.match_bucket_targets(bucket, None, event_name, object_key)
}
/// Allow one targeted reload per installed bucket incarnation. The ordinary
/// metadata refresh loop can still recover a failed or interrupted reload.
pub(crate) fn request_incarnation_refresh(&self, bucket: &str, incarnation_id: Uuid) -> bool {
self.bucket_rules_map.get(&bucket.to_string()).is_some_and(|snapshot| {
snapshot.incarnation_id.is_some_and(|current| current != incarnation_id)
&& !snapshot.refresh_requested.swap(true, Ordering::Relaxed)
})
}
pub(crate) fn match_bucket_targets(
&self,
bucket: &str,
incarnation_id: Option<Uuid>,
event_name: EventName,
object_key: &str,
) -> TargetIdSet {
let Some(snapshot) = self
.bucket_rules_map
.get(&bucket.to_string())
.filter(|snapshot| snapshot.incarnation_id == incarnation_id)
else {
return TargetIdSet::new();
};
let mut targets = snapshot.rules.match_rules(event_name, object_key);
if let Some(decoded_key) = decoded_object_key_for_matching(object_key) {
targets.extend(snapshot.rules.match_rules(event_name, &decoded_key));
}
targets
}
}
@@ -122,6 +176,35 @@ mod tests {
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
#[test]
fn bucket_rules_route_events_only_with_the_matching_incarnation() {
let engine = NotifyRuleEngine::new();
let old_id = uuid::Uuid::new_v4();
let new_id = uuid::Uuid::new_v4();
let old_target = TargetID::new("old".to_string(), "webhook".to_string());
let new_target = TargetID::new("new".to_string(), "webhook".to_string());
let rules = |target: TargetID| {
let mut rules = RulesMap::new();
rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_owned(), target);
rules
};
let matched = |id| engine.match_bucket_targets("bucket", id, EventName::ObjectCreatedPut, "object");
engine.publish_bucket_rules("bucket", Some(old_id), rules(old_target.clone()));
assert!(matched(Some(old_id)).contains(&old_target));
assert!(matched(Some(new_id)).is_empty(), "a peer may still have the old rules");
assert!(engine.request_incarnation_refresh("bucket", new_id));
assert!(!engine.request_incarnation_refresh("bucket", new_id));
assert!(matched(None).is_empty());
engine.withdraw_bucket_rules("bucket");
assert!(matched(Some(old_id)).is_empty());
assert!(matched(Some(new_id)).is_empty());
engine.publish_bucket_rules("bucket", Some(new_id), rules(new_target.clone()));
assert!(matched(Some(old_id)).is_empty());
assert!(matched(Some(new_id)).contains(&new_target));
}
#[tokio::test]
async fn rule_engine_tracks_bucket_rule_lifecycle() {
let engine = NotifyRuleEngine::new();
+3 -7
View File
@@ -17,7 +17,6 @@ use crate::{
bucket_config_manager::NotifyBucketConfigManager,
config_manager::NotifyConfigManager,
integration::NotificationMetrics,
notification_system_subscriber::NotificationSystemSubscriberView,
notifier::{EventNotifier, SharedNotifyTargetList},
pipeline::{LiveEventHistory, NotifyPipeline},
registry::TargetRegistry,
@@ -52,7 +51,6 @@ impl NotifyServices {
stream_cancellers: Arc<RwLock<ReplayWorkerManager>>,
concurrency_limiter: Arc<Semaphore>,
metrics: Arc<NotificationMetrics>,
subscriber_view: Arc<NotificationSystemSubscriberView>,
live_event_sender: broadcast::Sender<Arc<Event>>,
live_event_history: Arc<RwLock<LiveEventHistory>>,
) -> Self {
@@ -65,7 +63,7 @@ impl NotifyServices {
metrics.clone(),
);
let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone());
let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view);
let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine);
let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history);
let status_view = NotifyStatusView::new(metrics);
@@ -84,8 +82,8 @@ impl NotifyServices {
mod tests {
use super::NotifyServices;
use crate::{
integration::NotificationMetrics, notification_system_subscriber::NotificationSystemSubscriberView,
notifier::EventNotifier, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
integration::NotificationMetrics, notifier::EventNotifier, pipeline::LiveEventHistory, registry::TargetRegistry,
rule_engine::NotifyRuleEngine,
};
use rustfs_config::server_config::Config;
use rustfs_targets::ReplayWorkerManager;
@@ -102,7 +100,6 @@ mod tests {
let config = Arc::new(RwLock::new(Config::default()));
let stream_cancellers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
let concurrency_limiter = Arc::new(Semaphore::new(4));
let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
let (live_event_sender, _) = broadcast::channel(16);
let live_event_history = Arc::new(RwLock::new(LiveEventHistory::default()));
@@ -115,7 +112,6 @@ mod tests {
stream_cancellers,
concurrency_limiter,
metrics,
subscriber_view,
live_event_sender,
live_event_history,
);
+9
View File
@@ -27,6 +27,15 @@ pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
resolve_notify_object_store_handle_from_backend()
}
pub(crate) async fn refresh_notify_bucket_metadata(bucket: &str) -> bool {
let Some(store) = resolve_notify_object_store_handle() else {
return false;
};
rustfs_ecstore::api::bucket::metadata_sys::reload_bucket_metadata(store, bucket)
.await
.is_ok()
}
pub(crate) type NotifyServerConfigSnapshot = rustfs_ecstore::api::config::com::ServerConfigSnapshot;
pub(crate) async fn read_notify_server_config_snapshot(store: Arc<NotifyStore>) -> Result<NotifyServerConfigSnapshot, String> {
+1
View File
@@ -127,6 +127,7 @@ rustfs-security-governance = { workspace = true }
rustfs-storage-api = { workspace = true }
rustfs-utils = { workspace = true, features = ["ip"] }
rustix = { workspace = true, features = ["fs", "stdio"] }
chrono = { workspace = true, features = ["serde"] }
flate2 = { workspace = true }
glob = { workspace = true }
jiff = { workspace = true, features = ["serde"] }
+10 -14
View File
@@ -23,7 +23,8 @@ use crate::global::{
METRIC_LOG_CLEANER_ACTIVE_FILE_SIZE_BYTES, METRIC_LOG_CLEANER_ROTATION_DURATION_SECONDS,
METRIC_LOG_CLEANER_ROTATION_FAILURES_TOTAL, METRIC_LOG_CLEANER_ROTATION_TOTAL,
};
use jiff::{Timestamp, ZonedDateTime, tz::TimeZone};
use chrono::{Local, TimeZone as _};
use jiff::Zoned;
use metrics::{counter, gauge, histogram};
use std::fs::{self, File};
use std::io::{self, Write};
@@ -57,12 +58,8 @@ impl Rotation {
Rotation::Minutely => now / 60 != last / 60,
Rotation::Hourly => now / 3600 != last / 3600,
Rotation::Daily => {
let last_day = Timestamp::new(last, 0)
.ok()
.map(|ts| ts.to_zoned_datetime(TimeZone::system()).date());
let now_day = Timestamp::new(now, 0)
.ok()
.map(|ts| ts.to_zoned_datetime(TimeZone::system()).date());
let last_day = Local.timestamp_opt(last, 0).single().map(|ts| ts.date_naive());
let now_day = Local.timestamp_opt(now, 0).single().map(|ts| ts.date_naive());
match (last_day, now_day) {
(Some(last_day), Some(now_day)) => last_day != now_day,
@@ -134,7 +131,7 @@ impl RollingAppender {
match_mode,
file: None,
size: 0,
last_roll_ts: ZonedDateTime::now().timestamp().as_second(),
last_roll_ts: Zoned::now().timestamp().as_second(),
rotation_retry_cooldown_until: None,
roll_signal: None,
};
@@ -228,7 +225,7 @@ impl RollingAppender {
// 2. Time-based check
// We check this after size check to avoid unnecessary time calls if size forces a roll.
let now = ZonedDateTime::now().timestamp().as_second();
let now = Zoned::now().timestamp().as_second();
self.rotation.check_should_roll(self.last_roll_ts, now)
}
@@ -252,7 +249,7 @@ impl RollingAppender {
// 2. Generate archive name.
// Format: YYYYMMDDHHMMSS.uuuuuu (Microsecond/Nanosecond precision)
// We use jiff's strftime. "%Y%m%d%H%M%S%.6f" gives microsecond precision.
let now = ZonedDateTime::now();
let now = Zoned::now();
let timestamp_str = now.strftime("%Y%m%d%H%M%S%.6f").to_string();
// Add a unique counter to prevent collisions in high-concurrency/fast-rotation scenarios.
@@ -670,12 +667,11 @@ mod tests {
/// The fixtures below use mid-January dates, which sit on no DST or
/// offset transition in any zone, so every wall-clock time exists exactly once.
fn local_ts(year: i32, month: u32, day: u32, hour: u32, min: u32, sec: u32) -> i64 {
jiff::civil::DateTime::new(year as i16, month as i8, day as i8, hour as i8, min as i8, sec as i8, 0)
.expect("fixture local time must be valid")
.to_zoned_datetime(TimeZone::system())
Local
.with_ymd_and_hms(year, month, day, hour, min, sec)
.single()
.expect("fixture local time must exist exactly once")
.timestamp()
.as_second()
}
#[test]

Some files were not shown because too many files have changed in this diff Show More