mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-18 14:05:27 +00:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bd8855d0e4 | |||
| f57449197b | |||
| c4a50138ea | |||
| a5a5d0aac0 | |||
| 5e04936a88 | |||
| d098f82bf5 |
Generated
+390
-443
File diff suppressed because it is too large
Load Diff
+33
-33
@@ -151,16 +151,16 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.2.0", default-features = false }
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.66.0", default-features = false }
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.66.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.13.0", default-features = false }
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.13.0", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -168,13 +168,13 @@ async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.53.0", default-features = false }
|
||||
async-nats = { version = "0.52.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.2"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "=6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.3" }
|
||||
hyper = { version = "1.12.0" }
|
||||
@@ -209,8 +209,8 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
] }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.23.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.2" }
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
@@ -242,7 +242,7 @@ jsonwebtoken = { version = "12.0.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "0.10.0", features = ["pkcs1v15"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
rustls = { default-features = false, version = "0.23.46" }
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.16.0"
|
||||
@@ -250,13 +250,13 @@ x509-parser = "0.18.1"
|
||||
sha1 = "0.11.0"
|
||||
sha2 = "0.11.1"
|
||||
subtle = "2.6"
|
||||
zeroize = { version = "1.9.2" }
|
||||
zeroize = { version = "1.9.1" }
|
||||
proptest = "1"
|
||||
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "1.0.1" }
|
||||
jiff = { version = "0.2.43" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -273,18 +273,18 @@ tar-codec = "0.0.16"
|
||||
tar-framing = "0.0.16"
|
||||
atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.3" }
|
||||
aws-config = { version = "1.13.2" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.133.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.160.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.127.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.131.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.159.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.126.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.3" }
|
||||
aws-smithy-runtime-api = { version = "1.22.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.21.0" }
|
||||
aws-smithy-types = { version = "1.9.4" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.1.3"
|
||||
clap = { version = "4.6.8" }
|
||||
brotli = "9.1.1"
|
||||
clap = { version = "4.6.7" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -292,7 +292,7 @@ crossbeam-queue = "0.3.14"
|
||||
crossbeam-channel = "0.5.17"
|
||||
crossbeam-deque = "0.8.8"
|
||||
crossbeam-utils = "0.8.23"
|
||||
datafusion = { default-features = false, version = "55.3.0" }
|
||||
datafusion = { default-features = false, version = "55.2.0" }
|
||||
derive_builder = "0.20.2"
|
||||
ed25519-dalek = "3.0.0"
|
||||
enumset = "1.1.15"
|
||||
@@ -318,7 +318,7 @@ log = "0.4"
|
||||
memmap2 = "0.9.12"
|
||||
nixl-sys = { git = "https://github.com/houseme/nixl", rev = "cc636c83f13d692e85b67e55aed092413d16fb41" }
|
||||
lz4 = "1.28.1"
|
||||
lz4_flex = { version = "0.14.2", default-features = false, features = ["std", "frame"] }
|
||||
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
mime_guess = "2.0.5"
|
||||
@@ -337,14 +337,14 @@ percent-encoding = "2.3.2"
|
||||
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.4" }
|
||||
rand = { version = "0.10.3" }
|
||||
ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.1.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.2" }
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.2" }
|
||||
redis = { version = "1.8.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.14.0" }
|
||||
@@ -364,7 +364,7 @@ snap = "1.1.2"
|
||||
starshard = { version = "2.4.1" }
|
||||
strum = { version = "0.28.0" }
|
||||
sync_wrapper = "1.0.2"
|
||||
sysinfo = "0.40.4"
|
||||
sysinfo = "0.40.3"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.1"
|
||||
@@ -378,15 +378,15 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.29.3" }
|
||||
uuid = { version = "1.29.2" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.21" }
|
||||
zip = "9.7.0"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.5.1"
|
||||
zstd = "0.14.4"
|
||||
|
||||
# Observability and Metrics
|
||||
@@ -405,9 +405,9 @@ pyroscope = { version = "2.1.5" }
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.25.0" }
|
||||
unftp-core = "0.1.1"
|
||||
suppaftp = { version = "13.0.0" }
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.2" }
|
||||
russh = { version = "0.65.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
@@ -416,7 +416,7 @@ dav-server = "0.12.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.4" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.32.1", default-features = false }
|
||||
hotpath = { version = "0.31.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
|
||||
base64-simd = { workspace = true }
|
||||
pbkdf2 = { workspace = true, optional = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
rsa = { workspace = true }
|
||||
rsa = { workspace = true, features = ["sha2"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
thiserror.workspace = true
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use rsa::{
|
||||
Pkcs1v15Encrypt, PrivateKey, PublicKey,
|
||||
Pkcs1v15Encrypt, RsaPrivateKey, RsaPublicKey,
|
||||
pkcs8::{DecodePrivateKey, DecodePublicKey},
|
||||
pss::{Signature, SigningKey, VerifyingKey},
|
||||
pss::{BlindedSigningKey, Signature, VerifyingKey},
|
||||
sha2::Sha256,
|
||||
signature::{RandomizedSigner, Verifier},
|
||||
traits::PublicKeyParts,
|
||||
@@ -37,7 +37,7 @@ pub struct Token {
|
||||
pub fn gencode(token: &Token, key: &str) -> Result<String> {
|
||||
let data = serde_json::to_vec(token)?;
|
||||
let mut rng = rand::rng();
|
||||
let public_key = PublicKey::from_public_key_pem(key).map_err(Error::other)?;
|
||||
let public_key = RsaPublicKey::from_public_key_pem(key).map_err(Error::other)?;
|
||||
let encrypted_data = public_key.encrypt(&mut rng, Pkcs1v15Encrypt, &data).map_err(Error::other)?;
|
||||
Ok(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&encrypted_data))
|
||||
}
|
||||
@@ -51,7 +51,7 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
|
||||
let encrypted_data = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(Error::other)?;
|
||||
let private_key = PrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
|
||||
let private_key = RsaPrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
|
||||
let decrypted_data = private_key.decrypt(Pkcs1v15Encrypt, &encrypted_data).map_err(Error::other)?;
|
||||
serde_json::from_slice(&decrypted_data).map_err(Error::other)
|
||||
}
|
||||
@@ -63,8 +63,8 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
|
||||
pub fn sign_license_token(token: &Token, private_key_pem: &str) -> Result<String> {
|
||||
let payload = serde_json::to_vec(token)?;
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
|
||||
let signing_key = SigningKey::<Sha256>::new(private_key);
|
||||
let private_key = RsaPrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
|
||||
let signing_key = BlindedSigningKey::<Sha256>::new(private_key);
|
||||
let signature: Signature = signing_key.try_sign_with_rng(&mut rng, &payload).map_err(Error::other)?;
|
||||
let signature: Box<[u8]> = signature.into();
|
||||
|
||||
@@ -80,7 +80,7 @@ pub fn parse_signed_license_token(token: &str, public_key_pem: &str) -> Result<T
|
||||
let signed_payload = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(Error::other)?;
|
||||
let public_key = PublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
|
||||
let public_key = RsaPublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
|
||||
let signature_len = public_key.size();
|
||||
|
||||
if signed_payload.len() <= signature_len {
|
||||
@@ -103,7 +103,7 @@ pub fn parse_license_with_public_key(license: &str, public_key: &str) -> Result<
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rsa::{
|
||||
PrivateKey,
|
||||
RsaPrivateKey,
|
||||
pkcs8::{EncodePrivateKey, EncodePublicKey, LineEnding},
|
||||
};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -112,8 +112,8 @@ mod tests {
|
||||
fn test_sign_license_token_and_parse_signed_license_token() {
|
||||
let mut rng = rand::rng();
|
||||
let bits = 2048;
|
||||
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
@@ -144,8 +144,8 @@ mod tests {
|
||||
fn test_legacy_gencode_and_parse_roundtrip() {
|
||||
let mut rng = rand::rng();
|
||||
let bits = 2048;
|
||||
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
@@ -173,8 +173,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_signed_license_token_rejects_tampered_payload() {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
.expect("failed to encode private key pem");
|
||||
@@ -206,8 +206,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_signed_license_token_rejects_invalid_token() {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let public_key_pem = public_key
|
||||
.to_public_key_pem(LineEnding::LF)
|
||||
.expect("failed to encode public key pem");
|
||||
|
||||
@@ -98,6 +98,7 @@ rustfs-madmin.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
aws-smithy-types.workspace = true
|
||||
async-trait = { workspace = true }
|
||||
@@ -123,7 +124,7 @@ urlencoding.workspace = true
|
||||
walkdir.workspace = true
|
||||
base64-simd = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
hex-simd = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
prost.workspace = true
|
||||
@@ -141,7 +142,6 @@ clap = { workspace = true, features = ["derive", "env"] }
|
||||
[dev-dependencies]
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
|
||||
@@ -170,15 +170,6 @@ impl DiskFaultHarness {
|
||||
/// Start (or restart after [`Self::kill_server`]) the RustFS server with
|
||||
/// all volume directories and the same address.
|
||||
pub async fn start_server(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(false).await
|
||||
}
|
||||
|
||||
/// Start a process once its node read-readiness probe passes.
|
||||
pub async fn start_server_at_read_quorum(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(true).await
|
||||
}
|
||||
|
||||
async fn start_server_with_readiness(&mut self, read_quorum_only: bool) -> ChaosResult<()> {
|
||||
// The common helper always appends `env.temp_dir` as the final storage
|
||||
// path: point it at the last disk for startup and pass the remaining
|
||||
// disks explicitly, then restore it to the root so Drop cleans up
|
||||
@@ -190,13 +181,7 @@ impl DiskFaultHarness {
|
||||
let arg_refs: Vec<&str> = disk_args.iter().map(String::as_str).collect();
|
||||
let env_refs: Vec<(&str, &str)> = self.extra_env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
|
||||
|
||||
let result = if read_quorum_only {
|
||||
self.env
|
||||
.start_rustfs_server_with_env_at_read_quorum(arg_refs, &env_refs)
|
||||
.await
|
||||
} else {
|
||||
self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await
|
||||
};
|
||||
let result = self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await;
|
||||
self.env.temp_dir = self.root.to_string_lossy().to_string();
|
||||
result
|
||||
}
|
||||
|
||||
@@ -722,7 +722,7 @@ impl RustFSTestEnvironment {
|
||||
cleanup_existing: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing, false)
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -732,7 +732,6 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
cleanup_existing: bool,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
if cleanup_existing {
|
||||
self.cleanup_existing_processes().await?;
|
||||
@@ -757,7 +756,7 @@ impl RustFSTestEnvironment {
|
||||
self.process = Some(process);
|
||||
|
||||
// Wait for server to be ready
|
||||
self.wait_for_server_ready_with_probe(read_quorum_only).await?;
|
||||
self.wait_for_server_ready().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -771,7 +770,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -783,7 +782,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -801,18 +800,6 @@ impl RustFSTestEnvironment {
|
||||
self.start_rustfs_server_inner(extra_args, extra_env, true).await
|
||||
}
|
||||
|
||||
/// Start a degraded read-serving process using the node readiness probe.
|
||||
/// ListBuckets retains its namespace write-quorum requirement.
|
||||
pub async fn start_rustfs_server_with_env_at_read_quorum(
|
||||
&mut self,
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, true, true)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start RustFS server without cleaning up other running RustFS processes.
|
||||
///
|
||||
/// This is useful for tests that need multiple independent RustFS instances
|
||||
@@ -841,16 +828,8 @@ impl RustFSTestEnvironment {
|
||||
/// Fails immediately if the spawned server process has already exited
|
||||
/// (e.g. a port bind failure) instead of polling out the full timeout.
|
||||
pub async fn wait_for_server_ready(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_server_ready_with_probe(false).await
|
||||
}
|
||||
|
||||
async fn wait_for_server_ready_with_probe(
|
||||
&mut self,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
info!("Waiting for RustFS server to be ready on {}", self.address);
|
||||
let client = self.create_s3_client();
|
||||
let health_client = HttpClient::builder().no_proxy().timeout(Duration::from_secs(2)).build()?;
|
||||
|
||||
for i in 0..60 {
|
||||
if let Some(process) = self.process.as_mut()
|
||||
@@ -859,16 +838,7 @@ impl RustFSTestEnvironment {
|
||||
return Err(format!("RustFS server process exited before becoming ready: {status}").into());
|
||||
}
|
||||
|
||||
let ready = if read_quorum_only {
|
||||
health_client
|
||||
.get(format!("{}/minio/health/ready", self.url))
|
||||
.send()
|
||||
.await
|
||||
.is_ok_and(|response| response.status().is_success())
|
||||
} else {
|
||||
TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok()
|
||||
};
|
||||
if ready {
|
||||
if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
|
||||
info!("✅ RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! Over-the-wire smoke net for the embedded console listener (backlog#1154
|
||||
//! peri-4). The console is a second HTTP listener for the control-plane and
|
||||
//! unauthenticated console routes under `/rustfs/console`;
|
||||
//! peri-4). The console is a second HTTP listener that serves the full S3/admin
|
||||
//! surface plus the unauthenticated console routes under `/rustfs/console`;
|
||||
//! before this test its behaviour was only covered by in-process router tests
|
||||
//! (`rustfs/src/admin/console.rs`), so a misconfiguration that exposed
|
||||
//! credentials through the public console endpoints or fail-opened the
|
||||
@@ -26,13 +26,13 @@
|
||||
//! * `/rustfs/console/license` exposes only the coarse `licensed` flag.
|
||||
//! * the console SPA prefix dispatches to the static handler, never to the
|
||||
//! S3 API (no S3 error XML), whether or not console assets are embedded.
|
||||
//! * unauthenticated admin API requests are denied with 403 AccessDenied.
|
||||
//! * S3 root/object requests on the console listener return an empty 404,
|
||||
//! including authenticated requests, before S3 dispatch.
|
||||
//! * unauthenticated requests to the admin API and the S3 root on the console
|
||||
//! listener are denied (403 AccessDenied) — the extra listener does not
|
||||
//! fail-open the protected surface.
|
||||
//! * a listener with the console disabled (the main S3 port here) does not
|
||||
//! serve the unauthenticated console endpoints at all.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, signed_request_with_client};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
@@ -166,38 +166,18 @@ async fn test_console_over_the_wire_smoke() -> TestResult {
|
||||
let admin_body = admin_response.text().await?;
|
||||
assert!(admin_body.contains("AccessDenied"), "admin denial must be AccessDenied: {admin_body}");
|
||||
|
||||
let signed_data_root = signed_request_with_client(
|
||||
&client,
|
||||
http::Method::GET,
|
||||
&format!("{}/", env.url),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
// The console listener serves no S3 surface at all: an object route is not found there,
|
||||
// signed or not, instead of reaching the S3 service and being denied by it.
|
||||
let s3_root_response = client.get(format!("{console_base}/")).send().await?;
|
||||
assert_eq!(
|
||||
signed_data_root.status(),
|
||||
reqwest::StatusCode::OK,
|
||||
"the same credentials must reach S3 on the data listener"
|
||||
s3_root_response.status(),
|
||||
reqwest::StatusCode::NOT_FOUND,
|
||||
"the S3 root must not be served on the console listener"
|
||||
);
|
||||
assert!(
|
||||
s3_root_response.text().await?.is_empty(),
|
||||
"the console listener must not answer an object route with an S3 document"
|
||||
);
|
||||
|
||||
for path in ["/", "/console-object-bucket/key"] {
|
||||
let url = format!("{console_base}{path}");
|
||||
for signed in [false, true] {
|
||||
let response = if signed {
|
||||
signed_request_with_client(&client, http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?
|
||||
} else {
|
||||
client.get(&url).send().await?
|
||||
};
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
reqwest::StatusCode::NOT_FOUND,
|
||||
"S3 route {path}, signed={signed} must be isolated from the console listener"
|
||||
);
|
||||
assert!(response.bytes().await?.is_empty(), "isolated S3 routes must not reach a handler");
|
||||
}
|
||||
}
|
||||
|
||||
// --- console disabled on a listener means no console surface at all ------
|
||||
// The main S3 listener of this same process runs with the console disabled,
|
||||
|
||||
@@ -26,7 +26,7 @@ use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ObjectLockRule,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
|
||||
fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
@@ -66,7 +66,7 @@ async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let retain_until = Timestamp::now() + SignedDuration::from_hours(24);
|
||||
let retain_until = Utc::now() + ChronoDuration::days(1);
|
||||
|
||||
let compliance_key = "compliance.bin";
|
||||
let compliance_version = put_object_with_retention(
|
||||
@@ -176,10 +176,7 @@ async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_conf
|
||||
let retain_until = retention
|
||||
.retain_until_date()
|
||||
.ok_or("default retention omitted retain-until date")?;
|
||||
assert!(
|
||||
retain_until.secs() > Timestamp::now().as_second(),
|
||||
"default retention is not in the future"
|
||||
);
|
||||
assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
|
||||
|
||||
let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
|
||||
assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
|
||||
|
||||
@@ -38,8 +38,8 @@ use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use http::{Method, StatusCode};
|
||||
use jiff::Timestamp;
|
||||
use serde_json::Value;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -526,8 +526,8 @@ async fn put_multipart(client: &Client, bucket: &str, key: &str) -> TestResult<V
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
fn retention_deadline() -> TestResult<Timestamp> {
|
||||
Ok(Timestamp::now() + jiff::SignedDuration::from_hours(24))
|
||||
fn retention_deadline() -> TestResult<DateTime<Utc>> {
|
||||
DateTime::<Utc>::from_timestamp(Utc::now().timestamp() + 24 * 3600, 0).ok_or_else(|| "retention deadline out of range".into())
|
||||
}
|
||||
|
||||
/// Seed the deterministic inventory into the seed pool and record it.
|
||||
@@ -654,7 +654,7 @@ pub(crate) async fn seed_manifest(run: &TopologyRun) -> TestResult<TopologyManif
|
||||
let mut entry = ManifestEntry::body(&locked, &key, "compliance", Some(version_id.clone()), Some(version_id), &body);
|
||||
entry.retention = Some(RetentionRecord {
|
||||
mode: "COMPLIANCE".to_string(),
|
||||
retain_until_unix: retain_until.as_second(),
|
||||
retain_until_unix: retain_until.timestamp(),
|
||||
});
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
@@ -29,7 +29,6 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use jiff::Timestamp;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
@@ -387,7 +386,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
|
||||
"usage": "EncryptDecrypt",
|
||||
"status": "Active",
|
||||
"metadata": HashMap::<String, String>::new(),
|
||||
"created_at": format!("{}[UTC]", Timestamp::now().strftime("%Y-%m-%dT%H:%M:%S%.fZ")),
|
||||
"created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
|
||||
"rotated_at": serde_json::Value::Null,
|
||||
"created_by": "e2e-test",
|
||||
"encrypted_key_material": BASE64.encode_to_string(key_data),
|
||||
|
||||
@@ -23,13 +23,13 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use flate2::{
|
||||
Compression,
|
||||
write::{GzEncoder, ZlibEncoder},
|
||||
};
|
||||
use http::HeaderValue;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
@@ -42,8 +42,8 @@ use tokio::io::AsyncWriteExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
|
||||
let expiration = (Timestamp::now() + SignedDuration::from_hours(1))
|
||||
.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
let expiration = (Utc::now() + ChronoDuration::hours(1))
|
||||
.format("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
.to_string();
|
||||
let policy = serde_json::json!({
|
||||
"expiration": expiration,
|
||||
|
||||
@@ -283,7 +283,6 @@ async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> Te
|
||||
#[tokio::test]
|
||||
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
assert_single_node_cold_start_with_missing_volume().await?;
|
||||
|
||||
for (node_count, parity) in [(3, 1), (4, 2), (6, 3), (6, 2), (4, 0)] {
|
||||
let read_quorum = node_count - parity;
|
||||
@@ -431,138 +430,6 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_single_node_cold_start_with_missing_volume() -> TestResult {
|
||||
for drive_count in [2, 3, 4] {
|
||||
let mut server = crate::chaos::DiskFaultHarness::new(drive_count).await?;
|
||||
server.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{}", drive_count / 2));
|
||||
server.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
server.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
server.set_env("RUSTFS_HEALTH_ENDPOINT_ENABLE", "true");
|
||||
server.set_env("RUST_LOG", "warn");
|
||||
server.start_server_at_read_quorum().await?;
|
||||
let client = server.env.create_s3_client();
|
||||
let client = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
);
|
||||
let bucket = format!("missing-volume-{drive_count}");
|
||||
client.create_bucket().bucket(&bucket).send().await?;
|
||||
let small = vec![17; 4096];
|
||||
let large = vec![73; 8 * 1024 * 1024];
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
for disk in 0..drive_count {
|
||||
let census = crate::chaos::census_object_version_on_disk(server.disk_path(disk), &bucket, key, None)?;
|
||||
assert!(census.is_complete(), "seed shard must exist before restart: {census:?}");
|
||||
}
|
||||
}
|
||||
server.kill_server();
|
||||
let missing = drive_count - 1;
|
||||
server.take_disk_offline(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
let head = client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
|
||||
}
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
let keys: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(keys, ["large", "small"]);
|
||||
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("degraded-write")
|
||||
.body(Bytes::from_static(b"degraded write").into())
|
||||
.send()
|
||||
.await;
|
||||
if drive_count == 2 {
|
||||
let error = put.expect_err("one of two drives cannot commit a new object");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
let error = client
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key("large")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("one of two drives cannot commit an object deletion");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_quorum_object_body(&client, &bucket, "large", &large).await?;
|
||||
} else {
|
||||
put?;
|
||||
assert_quorum_object_body(&client, &bucket, "degraded-write", b"degraded write").await?;
|
||||
client.delete_object().bucket(&bucket).key("degraded-write").send().await?;
|
||||
}
|
||||
assert!(
|
||||
!server.disk_path(missing).exists(),
|
||||
"degraded startup and S3 operations must leave the root absent"
|
||||
);
|
||||
|
||||
if drive_count == 2 {
|
||||
server.kill_server();
|
||||
std::fs::create_dir(server.disk_path(missing))?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
}
|
||||
assert_eq!(client.list_objects_v2().bucket(&bucket).send().await?.contents().len(), 2);
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("empty-root-write")
|
||||
.body(Bytes::from_static(b"must remain unavailable").into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an empty replacement cannot restore write quorum");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert!(
|
||||
!server
|
||||
.disk_path(missing)
|
||||
.join(crate::storage_api::RUSTFS_META_BUCKET)
|
||||
.join("format.json")
|
||||
.exists(),
|
||||
"read admission must not format the empty replacement"
|
||||
);
|
||||
}
|
||||
server.kill_server();
|
||||
server.bring_disk_online(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
}
|
||||
for key in ["degraded-write", "empty-root-write"] {
|
||||
let error = client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("rejected or deleted degraded write must stay absent after recovery");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("recovery-write")
|
||||
.body(Bytes::from_static(b"recovered write").into())
|
||||
.send()
|
||||
.await?;
|
||||
assert_quorum_object_body(&client, &bucket, "recovery-write", b"recovered write").await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cold_start_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
@@ -864,6 +864,70 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_webhook_subscriptions_follow_bucket_incarnation() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let (endpoint, mut rx, collector) = spawn_event_collector().await?;
|
||||
let allowed_origin = endpoint_origin(&endpoint)?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[(ENV_OUTBOUND_ALLOW_ORIGINS, allowed_origin.as_str())])
|
||||
.await?;
|
||||
enable_notify_module(&env).await?;
|
||||
let target = "incarnation";
|
||||
configure_webhook_target(&env, target, &endpoint).await?;
|
||||
wait_for_target_registered(&env, target).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "notify-incarnation";
|
||||
let control_bucket = "notify-incarnation-control";
|
||||
for name in [bucket, control_bucket] {
|
||||
client.create_bucket().bucket(name).send().await?;
|
||||
put_notification_config(&client, name, target, "events/", ".txt").await?;
|
||||
}
|
||||
let put = |bucket, key| {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"event fixture"))
|
||||
.send()
|
||||
};
|
||||
let wait = Duration::from_secs(20);
|
||||
let original_key = "events/original.txt";
|
||||
put(bucket, original_key).await?;
|
||||
wait_for_event(&mut rx, original_key, "s3:ObjectCreated:", wait).await?;
|
||||
|
||||
assert!(client.delete_bucket().bucket(bucket).send().await.is_err());
|
||||
let after_failed_delete = "events/retained.txt";
|
||||
put(bucket, after_failed_delete).await?;
|
||||
wait_for_event(&mut rx, after_failed_delete, "s3:ObjectCreated:", wait).await?;
|
||||
|
||||
for key in [original_key, after_failed_delete] {
|
||||
client.delete_object().bucket(bucket).key(key).send().await?;
|
||||
}
|
||||
client.delete_bucket().bucket(bucket).send().await?;
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
let config = client.get_bucket_notification_configuration().bucket(bucket).send().await?;
|
||||
assert!(config.queue_configurations().is_empty());
|
||||
|
||||
let new_key = "events/fresh.txt";
|
||||
let sentinel_key = "events/sentinel.txt";
|
||||
put(bucket, new_key).await?;
|
||||
put(control_bucket, sentinel_key).await?;
|
||||
let events = collect_until(&mut rx, sentinel_key, wait, Duration::from_secs(2)).await;
|
||||
assert!(events.iter().any(|event| event_key(event).as_deref() == Some(sentinel_key)));
|
||||
assert!(events.iter().all(|event| event_key(event).as_deref() != Some(new_key)));
|
||||
|
||||
put_notification_config(&client, bucket, target, "events/", ".txt").await?;
|
||||
let subscribed_key = "events/subscribed.txt";
|
||||
put(bucket, subscribed_key).await?;
|
||||
wait_for_event(&mut rx, subscribed_key, "s3:ObjectCreated:", wait).await?;
|
||||
env.stop_server();
|
||||
collector.abort();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// An event queued while the target endpoint rejects delivery survives on the
|
||||
/// durable store and is redelivered once the endpoint comes back.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -29,7 +29,7 @@ use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockMode,
|
||||
ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use tracing::info;
|
||||
|
||||
use crate::common::RustFSTestEnvironment;
|
||||
@@ -126,10 +126,10 @@ pub async fn put_object_with_retention(
|
||||
key: &str,
|
||||
data: &[u8],
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: Timestamp,
|
||||
retain_until: DateTime<Utc>,
|
||||
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
|
||||
// AWS SDK requires UTC time without timezone offset (e.g., "2026-01-24T11:20:14Z")
|
||||
let retain_until_str = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_str = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_datetime =
|
||||
aws_sdk_s3::primitives::DateTime::from_str(&retain_until_str, aws_sdk_s3::primitives::DateTimeFormat::DateTime)?;
|
||||
|
||||
@@ -181,10 +181,10 @@ pub async fn put_object_retention(
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: Timestamp,
|
||||
retain_until: DateTime<Utc>,
|
||||
bypass_governance: bool,
|
||||
) -> Result<(), Box<SdkError<PutObjectRetentionError>>> {
|
||||
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.as_second());
|
||||
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.timestamp());
|
||||
|
||||
let retention = ObjectLockRetention::builder()
|
||||
.mode(mode.clone())
|
||||
@@ -253,6 +253,6 @@ pub async fn delete_object_with_bypass(
|
||||
}
|
||||
|
||||
/// Calculate a future retain_until date
|
||||
pub fn future_retain_until(days: i64) -> Timestamp {
|
||||
Timestamp::now() + SignedDuration::from_hours(days * 24)
|
||||
pub fn future_retain_until(days: i64) -> DateTime<Utc> {
|
||||
Utc::now() + Duration::days(days)
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use aws_sdk_s3::types::{
|
||||
CompletedMultipartUpload, CompletedPart, Delete, MetadataDirective, ObjectIdentifier, ObjectLockLegalHoldStatus,
|
||||
ObjectLockMode, ObjectLockRetentionMode,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use tracing::info;
|
||||
|
||||
/// Initialize test logging
|
||||
@@ -67,7 +67,7 @@ async fn put_bucket_deny_policy(
|
||||
}
|
||||
|
||||
fn retention_timestamp(days: i64) -> aws_sdk_s3::primitives::DateTime {
|
||||
let retain_until = future_retain_until(days).strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until = future_retain_until(days).format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
aws_sdk_s3::primitives::DateTime::from_str(&retain_until, aws_sdk_s3::primitives::DateTimeFormat::DateTime)
|
||||
.expect("retention timestamp should parse")
|
||||
}
|
||||
@@ -118,14 +118,13 @@ where
|
||||
);
|
||||
}
|
||||
|
||||
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> Timestamp {
|
||||
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> DateTime<Utc> {
|
||||
let formatted = value
|
||||
.fmt(DateTimeFormat::DateTime)
|
||||
.expect("S3 timestamp should format as RFC3339");
|
||||
let normalized = formatted
|
||||
.strip_suffix('Z')
|
||||
.map_or(formatted.clone(), |prefix| format!("{prefix}+00:00"));
|
||||
Timestamp::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", normalized).expect("S3 timestamp should parse as RFC3339")
|
||||
DateTime::parse_from_rfc3339(&formatted)
|
||||
.expect("S3 timestamp should parse as RFC3339")
|
||||
.with_timezone(&Utc)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -471,7 +470,7 @@ async fn test_get_object_retention_returns_configured_values() {
|
||||
let bucket = "test-get-retention";
|
||||
let key = "retained-object";
|
||||
let retain_until = future_retain_until(30);
|
||||
let retain_until_expected = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_expected = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
|
||||
env.create_object_lock_bucket(bucket).await.unwrap();
|
||||
|
||||
@@ -1925,7 +1924,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let copy_started = Timestamp::now();
|
||||
let copy_started = Utc::now();
|
||||
client
|
||||
.copy_object()
|
||||
.copy_source(format!("{src_bucket}/{src_key}"))
|
||||
@@ -1948,7 +1947,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
|
||||
assert!(
|
||||
retain_until < copy_started + SignedDuration::from_hours(3 * 24),
|
||||
retain_until < copy_started + Duration::days(3),
|
||||
"destination default retention should not inherit the source's longer retention"
|
||||
);
|
||||
|
||||
@@ -2009,7 +2008,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
.expect("explicit retain-until date should be present"),
|
||||
);
|
||||
assert!(
|
||||
explicit_retain_until > Timestamp::now() + SignedDuration::from_hours(20 * 24),
|
||||
explicit_retain_until > Utc::now() + Duration::days(20),
|
||||
"explicit retention should override the shorter bucket default"
|
||||
);
|
||||
|
||||
@@ -2089,7 +2088,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let create_started = Timestamp::now();
|
||||
let create_started = Utc::now();
|
||||
let create_output = client.create_multipart_upload().bucket(bucket).key(key).send().await.unwrap();
|
||||
let upload_id = create_output.upload_id().unwrap();
|
||||
|
||||
@@ -2132,7 +2131,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
|
||||
assert!(
|
||||
retain_until < create_started + SignedDuration::from_hours(3 * 24),
|
||||
retain_until < create_started + Duration::days(3),
|
||||
"CompleteMultipartUpload should keep the default retention calculated at create time"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -573,15 +573,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
type RenameDataBatchStream = ResponseStream<rustfs_protos::proto_gen::node_service::RenameDataBatchResult>;
|
||||
|
||||
async fn rename_data_batch(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataBatchRequest>,
|
||||
) -> Result<Response<Self::RenameDataBatchStream>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_multipart_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -154,6 +154,7 @@ arc-swap.workspace = true
|
||||
async-trait.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
byteorder = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
thiserror.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
|
||||
@@ -90,9 +90,7 @@ fn remote_credentials(credentials: &Credentials, account_id: &str) -> RemoteCred
|
||||
access_key: credentials.access_key.clone(),
|
||||
secret_key: credentials.secret_key.clone(),
|
||||
session_token: credentials.effective_session_token().map(str::to_string),
|
||||
expiration: credentials
|
||||
.effective_expiration()
|
||||
.and_then(|timestamp| SystemTime::try_from(timestamp).ok()),
|
||||
expiration: credentials.effective_expiration().map(SystemTime::from),
|
||||
account_id: account_id.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,15 +51,20 @@ impl LcAuditEvent {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn emit_transition_failed_event(object: ObjectInfo) {
|
||||
emit_lifecycle_event(EventName::ObjectTransitionFailed, object, LIFECYCLE_TRANSITION_USER_AGENT);
|
||||
pub(crate) fn emit_transition_failed_event(object: ObjectInfo, incarnation_id: Option<uuid::Uuid>) {
|
||||
emit_lifecycle_event(EventName::ObjectTransitionFailed, object, LIFECYCLE_TRANSITION_USER_AGENT, incarnation_id);
|
||||
}
|
||||
|
||||
pub(crate) fn emit_transition_complete_event(object: ObjectInfo) {
|
||||
emit_lifecycle_event(EventName::ObjectTransitionComplete, object, LIFECYCLE_TRANSITION_USER_AGENT);
|
||||
pub(crate) fn emit_transition_complete_event(object: ObjectInfo, incarnation_id: Option<uuid::Uuid>) {
|
||||
emit_lifecycle_event(
|
||||
EventName::ObjectTransitionComplete,
|
||||
object,
|
||||
LIFECYCLE_TRANSITION_USER_AGENT,
|
||||
incarnation_id,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn emit_transitioned_expiration_event(source: &ObjectInfo, deleted: &ObjectInfo) {
|
||||
pub(crate) fn emit_transitioned_expiration_event(source: &ObjectInfo, deleted: &ObjectInfo, incarnation_id: uuid::Uuid) {
|
||||
let event_name = transitioned_expiration_event_name(source.delete_marker, deleted.delete_marker);
|
||||
let object = ObjectInfo {
|
||||
bucket: source.bucket.clone(),
|
||||
@@ -69,12 +74,17 @@ pub(crate) fn emit_transitioned_expiration_event(source: &ObjectInfo, deleted: &
|
||||
delete_marker: source.delete_marker,
|
||||
..Default::default()
|
||||
};
|
||||
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT);
|
||||
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT, Some(incarnation_id));
|
||||
}
|
||||
|
||||
pub(crate) fn emit_non_transitioned_expiration_event(action: IlmAction, source: &ObjectInfo, deleted: ObjectInfo) {
|
||||
pub(crate) fn emit_non_transitioned_expiration_event(
|
||||
action: IlmAction,
|
||||
source: &ObjectInfo,
|
||||
deleted: ObjectInfo,
|
||||
incarnation_id: uuid::Uuid,
|
||||
) {
|
||||
let event_name = non_transitioned_expiration_event_name(action, source.delete_marker, deleted.delete_marker);
|
||||
emit_lifecycle_event(event_name, deleted, LIFECYCLE_EXPIRY_USER_AGENT);
|
||||
emit_lifecycle_event(event_name, deleted, LIFECYCLE_EXPIRY_USER_AGENT, Some(incarnation_id));
|
||||
}
|
||||
|
||||
/// Emit the lifecycle expiration event for one version removed by the batch
|
||||
@@ -83,9 +93,15 @@ pub(crate) fn emit_non_transitioned_expiration_event(action: IlmAction, source:
|
||||
/// That path never sent events, so a successful noncurrent-version expiry was
|
||||
/// invisible to notification subscribers even though the equivalent
|
||||
/// current-version path emits one (backlog#2202).
|
||||
pub(crate) fn emit_noncurrent_expiration_event(bucket: &str, target: &ObjectToDelete, deleted: &DeletedObject, failed: bool) {
|
||||
pub(crate) fn emit_noncurrent_expiration_event(
|
||||
bucket: &str,
|
||||
target: &ObjectToDelete,
|
||||
deleted: &DeletedObject,
|
||||
failed: bool,
|
||||
incarnation_id: uuid::Uuid,
|
||||
) {
|
||||
if let Some((event_name, object)) = noncurrent_expiration_event(bucket, target, deleted, failed) {
|
||||
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT);
|
||||
emit_lifecycle_event(event_name, object, LIFECYCLE_EXPIRY_USER_AGENT, Some(incarnation_id));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,9 +147,10 @@ fn noncurrent_expiration_event(
|
||||
Some((event_name, object))
|
||||
}
|
||||
|
||||
fn emit_lifecycle_event(event_name: EventName, object: ObjectInfo, user_agent: &str) {
|
||||
fn emit_lifecycle_event(event_name: EventName, object: ObjectInfo, user_agent: &str, incarnation_id: Option<uuid::Uuid>) {
|
||||
send_event(EventArgs {
|
||||
event_name: event_name.to_string(),
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
bucket_name: object.bucket.clone(),
|
||||
object,
|
||||
user_agent: user_agent.to_string(),
|
||||
|
||||
@@ -2166,9 +2166,10 @@ impl TransitionState {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if let Err(err) =
|
||||
let (incarnation_id, transition_result) =
|
||||
transition_object(api.clone(), &task.obj_info, LcAuditEvent::new(task.event.clone(), task.src.clone()))
|
||||
.await
|
||||
.await;
|
||||
if let Err(err) = transition_result
|
||||
{
|
||||
if let (Some(job_id), Some(result_key)) = (task.manual_job_id, task.manual_result_key.as_deref()) {
|
||||
record_manual_transition_worker_result_for_task_with_reason(
|
||||
@@ -2195,7 +2196,7 @@ impl TransitionState {
|
||||
"Lifecycle tier operation failed"
|
||||
);
|
||||
}
|
||||
emit_transition_failed_event(obj_info_for_event);
|
||||
emit_transition_failed_event(obj_info_for_event, incarnation_id);
|
||||
} else {
|
||||
if let (Some(job_id), Some(result_key)) = (task.manual_job_id, task.manual_result_key.as_deref()) {
|
||||
record_manual_transition_worker_result_for_task(
|
||||
@@ -2217,7 +2218,7 @@ impl TransitionState {
|
||||
}
|
||||
transition_state.add_lastday_stats(&task.event.storage_class, ts);
|
||||
|
||||
emit_transition_complete_event(obj_info_for_event);
|
||||
emit_transition_complete_event(obj_info_for_event, incarnation_id);
|
||||
}
|
||||
// Release the dedup claim so a later lifecycle pass can
|
||||
// re-transition this object if needed (rustfs/backlog#1268).
|
||||
@@ -5121,7 +5122,7 @@ async fn expire_transitioned_object_with_lock_lost_signal(
|
||||
}
|
||||
};
|
||||
|
||||
schedule_lifecycle_replication_delete_if_needed(oi, &dobj).await;
|
||||
schedule_lifecycle_replication_delete_if_needed(oi, &dobj, bucket_incarnation_id).await;
|
||||
|
||||
// The transitioned version is gone; evict any cached body for this object
|
||||
// so it does not linger until TTL (ODC-26).
|
||||
@@ -5134,7 +5135,7 @@ async fn expire_transitioned_object_with_lock_lost_signal(
|
||||
|
||||
//audit_log_lifecycle(oi, ILMExpiry, tags);
|
||||
|
||||
emit_transitioned_expiration_event(oi, &dobj);
|
||||
emit_transitioned_expiration_event(oi, &dobj, bucket_incarnation_id);
|
||||
|
||||
Ok(dobj)
|
||||
}
|
||||
@@ -5152,12 +5153,12 @@ pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
|
||||
Ok(obj)
|
||||
}
|
||||
|
||||
pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> Result<(), Error> {
|
||||
async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditEvent) -> (Option<Uuid>, Result<(), Error>) {
|
||||
let time_ilm = Metrics::time_ilm(lae.event.action);
|
||||
|
||||
let (publication_guard, bucket_incarnation_id) = lifecycle_transition_publication_guard(&api, oi)
|
||||
.await
|
||||
.ok_or_else(|| Error::other("lifecycle transition is not allowed for this bucket"))?;
|
||||
let Some((publication_guard, bucket_incarnation_id)) = lifecycle_transition_publication_guard(&api, oi).await else {
|
||||
return (None, Err(Error::other("lifecycle transition is not allowed for this bucket")));
|
||||
};
|
||||
|
||||
let etag = if let Some(etag) = &oi.etag { etag } else { "" };
|
||||
let etag = etag.to_string();
|
||||
@@ -5180,7 +5181,7 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
|
||||
opts.add_namespace_lock_guard(&publication_guard);
|
||||
let result = api.transition_object(&oi.bucket, &oi.name, &opts).await;
|
||||
time_ilm(1)();
|
||||
result
|
||||
(Some(bucket_incarnation_id), result)
|
||||
}
|
||||
|
||||
#[allow(
|
||||
@@ -6034,7 +6035,7 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
|
||||
{
|
||||
accounting.commit(size).await;
|
||||
}
|
||||
schedule_lifecycle_replication_delete_if_needed(oi, &dobj).await;
|
||||
schedule_lifecycle_replication_delete_if_needed(oi, &dobj, bucket_incarnation_id).await;
|
||||
|
||||
// The object (or all its versions, for delete_all) was expired; evict any
|
||||
// cached body so dead bytes do not sit resident until TTL (ODC-26). The
|
||||
@@ -6050,7 +6051,7 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
|
||||
//let tags = LcAuditEvent::new(lc_event.clone(), src.clone()).tags();
|
||||
//tags["version-id"] = dobj.version_id;
|
||||
|
||||
emit_non_transitioned_expiration_event(lc_event.action, oi, dobj);
|
||||
emit_non_transitioned_expiration_event(lc_event.action, oi, dobj, bucket_incarnation_id);
|
||||
|
||||
if lc_event.action != IlmAction::NoneAction {
|
||||
let mut num_versions = 1_u64;
|
||||
@@ -6202,12 +6203,12 @@ fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> DeletedObject
|
||||
}
|
||||
}
|
||||
|
||||
async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo, dobj: &ObjectInfo) {
|
||||
async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo, dobj: &ObjectInfo, bucket_incarnation_id: Uuid) {
|
||||
let delete_object = lifecycle_deleted_object(oi, dobj);
|
||||
if delete_object.replication_state.is_none() {
|
||||
return;
|
||||
}
|
||||
replication_sink::schedule_delete(oi.bucket.clone(), delete_object).await;
|
||||
replication_sink::schedule_delete(oi.bucket.clone(), delete_object, Some(bucket_incarnation_id)).await;
|
||||
}
|
||||
|
||||
async fn lifecycle_delete_config_snapshot(api: &ECStore, oi: &ObjectInfo) -> Result<DeleteReplicationConfigSnapshot, Error> {
|
||||
|
||||
@@ -128,7 +128,7 @@ pub async fn delete_object_versions(
|
||||
// order and admission; the event is derived from the committed
|
||||
// result, and a send failure never rolls back a delete that
|
||||
// already happened (backlog#2202).
|
||||
emit_noncurrent_expiration_event(bucket, target, deleted_obj, false);
|
||||
emit_noncurrent_expiration_event(bucket, target, deleted_obj, false, bucket_incarnation_id);
|
||||
}
|
||||
if !deleted_obj
|
||||
.replication_state
|
||||
@@ -137,7 +137,7 @@ pub async fn delete_object_versions(
|
||||
{
|
||||
continue;
|
||||
}
|
||||
replication_sink::schedule_delete(bucket.to_string(), deleted_obj.clone()).await;
|
||||
replication_sink::schedule_delete(bucket.to_string(), deleted_obj.clone(), Some(bucket_incarnation_id)).await;
|
||||
}
|
||||
|
||||
for (i, err) in errors.iter().enumerate() {
|
||||
|
||||
@@ -66,8 +66,12 @@ pub(crate) fn lifecycle_action_waits_for_replication(action: IlmAction) -> bool
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
|
||||
ReplicationLifecycleBridge::schedule_delete(bucket, delete_object).await;
|
||||
pub(crate) async fn schedule_delete(
|
||||
bucket: String,
|
||||
delete_object: DeletedObject,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) {
|
||||
ReplicationLifecycleBridge::schedule_delete(bucket, delete_object, notification_bucket_incarnation).await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -460,7 +460,7 @@ fn clear_bucket_overrides(bucket: &str) {
|
||||
}
|
||||
|
||||
/// Publish application-owned bytes on every cache install path.
|
||||
fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
|
||||
fn sync_application_bucket_configs(bucket: &str, bm: &BucketMetadata) {
|
||||
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
|
||||
hook(
|
||||
bucket,
|
||||
@@ -468,12 +468,22 @@ fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
|
||||
bm.on_demand_migration_config()
|
||||
.map(|(bytes, stamp)| (bytes, stamp, bm.bucket_incarnation_id)),
|
||||
);
|
||||
hook(
|
||||
bucket,
|
||||
super::metadata::BUCKET_NOTIFICATION_CONFIG,
|
||||
(!bm.notification_config_xml.is_empty()).then_some((
|
||||
bm.notification_config_xml.as_slice(),
|
||||
bm.notification_config_updated_at,
|
||||
bm.bucket_incarnation_id,
|
||||
)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn clear_on_demand_migration(bucket: &str) {
|
||||
fn clear_application_bucket_configs(bucket: &str) {
|
||||
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
|
||||
hook(bucket, super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, None);
|
||||
hook(bucket, super::metadata::BUCKET_NOTIFICATION_CONFIG, None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2253,6 +2263,7 @@ impl BucketMetadataSys {
|
||||
.await?;
|
||||
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, persisted, &guard)
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2486,7 +2497,7 @@ impl BucketMetadataSys {
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_overrides(bucket);
|
||||
clear_on_demand_migration(bucket);
|
||||
clear_application_bucket_configs(bucket);
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
@@ -2524,7 +2535,7 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &bm).await;
|
||||
sync_bucket_overrides(bucket, &bm);
|
||||
sync_on_demand_migration(bucket, &bm);
|
||||
sync_application_bucket_configs(bucket, &bm);
|
||||
}
|
||||
MetadataLoadMode::Initial => {
|
||||
let _publish_guard = self
|
||||
@@ -2550,7 +2561,7 @@ impl BucketMetadataSys {
|
||||
metadata: BucketMetadata,
|
||||
persisted: bool,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
) -> Result<bool> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
if !persisted {
|
||||
let _publish_guard = self
|
||||
@@ -2563,7 +2574,7 @@ impl BucketMetadataSys {
|
||||
_ => false,
|
||||
};
|
||||
if !unchanged {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
}
|
||||
let removed = map.remove(bucket).is_some();
|
||||
drop(map);
|
||||
@@ -2572,9 +2583,9 @@ impl BucketMetadataSys {
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_overrides(bucket);
|
||||
clear_on_demand_migration(bucket);
|
||||
clear_application_bucket_configs(bucket);
|
||||
}
|
||||
return Ok(());
|
||||
return Ok(true);
|
||||
}
|
||||
let _publish_guard = self
|
||||
.lock_metadata_publish(bucket, namespace_guard, "refreshed bucket metadata publish")
|
||||
@@ -2587,7 +2598,7 @@ impl BucketMetadataSys {
|
||||
_ => false,
|
||||
};
|
||||
if !unchanged {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
}
|
||||
map.insert(bucket.to_string(), Arc::clone(&metadata));
|
||||
drop(map);
|
||||
@@ -2595,8 +2606,8 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &metadata).await;
|
||||
sync_bucket_overrides(bucket, &metadata);
|
||||
sync_on_demand_migration(bucket, &metadata);
|
||||
Ok(())
|
||||
sync_application_bucket_configs(bucket, &metadata);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub async fn get(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
@@ -2625,7 +2636,7 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(&bucket).await;
|
||||
sync_bucket_target_sys(&bucket, &bm).await;
|
||||
sync_bucket_overrides(&bucket, &bm);
|
||||
sync_on_demand_migration(&bucket, &bm);
|
||||
sync_application_bucket_configs(&bucket, &bm);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2649,7 +2660,7 @@ impl BucketMetadataSys {
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_overrides(bucket);
|
||||
clear_on_demand_migration(bucket);
|
||||
clear_application_bucket_configs(bucket);
|
||||
}
|
||||
removed || removed_fabricated
|
||||
}
|
||||
@@ -2876,25 +2887,35 @@ impl BucketMetadataSys {
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
self.revoke_pending_bucket_creation(bucket).await;
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
if !self
|
||||
.bucket_exists(bucket, namespace_guard, "peer bucket metadata existence check")
|
||||
.await?
|
||||
{
|
||||
return Err(Error::other(PEER_METADATA_NOT_PERSISTED));
|
||||
}
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"peer bucket metadata load",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if !persisted {
|
||||
return Err(Error::other(PEER_METADATA_NOT_PERSISTED));
|
||||
// A concurrent publication can reject the snapshot we read. Re-read
|
||||
// from storage so a newer committed config does not lose its reload to
|
||||
// an older one that happened to publish first.
|
||||
for _ in 0..3 {
|
||||
let expected = self.metadata_map.read().await.get(bucket).cloned();
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"peer bucket metadata load",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
if !persisted {
|
||||
return Err(Error::other(PEER_METADATA_NOT_PERSISTED));
|
||||
}
|
||||
if self
|
||||
.publish_if_unchanged(bucket, expected.as_ref(), metadata, true, namespace_guard)
|
||||
.await?
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, true, namespace_guard)
|
||||
.await
|
||||
Err(Error::other("bucket metadata changed repeatedly during peer reload"))
|
||||
}
|
||||
|
||||
pub async fn get_config(&self, bucket: &str) -> Result<(Arc<BucketMetadata>, bool)> {
|
||||
@@ -2992,7 +3013,7 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &bm).await;
|
||||
sync_bucket_overrides(bucket, &bm);
|
||||
sync_on_demand_migration(bucket, &bm);
|
||||
sync_application_bucket_configs(bucket, &bm);
|
||||
return Ok((bm, true));
|
||||
} else {
|
||||
let exists = self
|
||||
@@ -3443,7 +3464,7 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &metadata).await;
|
||||
sync_bucket_overrides(bucket, &metadata);
|
||||
sync_on_demand_migration(bucket, &metadata);
|
||||
sync_application_bucket_configs(bucket, &metadata);
|
||||
Ok(BucketMetadataAuthority::Authoritative(metadata))
|
||||
}
|
||||
|
||||
@@ -6792,31 +6813,33 @@ mod tests {
|
||||
|
||||
const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
|
||||
|
||||
type RecordedOdmConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
|
||||
type RecordedOdmHookCall = (String, RecordedOdmConfig);
|
||||
const NOTIFY_XML: &[u8] = br#"<NotificationConfiguration><QueueConfiguration><Id>test</Id><Queue>arn:rustfs:sqs:us-east-1:primary:webhook</Queue><Event>s3:ObjectCreated:*</Event></QueueConfiguration></NotificationConfiguration>"#;
|
||||
const PUBLISH_TEST_BUCKET: &str = "application-config-hook-paths";
|
||||
|
||||
/// Every `(bucket, config)` the recording hook has seen. Tests filter by
|
||||
/// their own bucket name; the hook is process-wide and set once.
|
||||
static ODM_HOOK_CALLS: std::sync::Mutex<Vec<RecordedOdmHookCall>> = std::sync::Mutex::new(Vec::new());
|
||||
type RecordedConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
|
||||
type RecordedConfigCall = (String, RecordedConfig);
|
||||
|
||||
fn install_recording_odm_hook() {
|
||||
static CONFIG_HOOK_CALLS: std::sync::Mutex<Vec<RecordedConfigCall>> = std::sync::Mutex::new(Vec::new());
|
||||
|
||||
fn install_recording_config_hook() {
|
||||
BUCKET_CONFIG_PUBLISH_HOOK.get_or_init(|| {
|
||||
Box::new(|bucket, config_file, config| {
|
||||
assert_eq!(config_file, super::super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG);
|
||||
ODM_HOOK_CALLS.lock().unwrap().push((
|
||||
bucket.to_string(),
|
||||
config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
|
||||
));
|
||||
if bucket == PUBLISH_TEST_BUCKET {
|
||||
CONFIG_HOOK_CALLS.lock().unwrap().push((
|
||||
config_file.to_string(),
|
||||
config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
|
||||
));
|
||||
}
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
fn odm_hook_calls(bucket: &str) -> Vec<RecordedOdmConfig> {
|
||||
ODM_HOOK_CALLS
|
||||
fn config_hook_calls(config_file: &str) -> Vec<RecordedConfig> {
|
||||
CONFIG_HOOK_CALLS
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|(name, _)| name == bucket)
|
||||
.filter(|(file, _)| file == config_file)
|
||||
.map(|(_, config)| config.clone())
|
||||
.collect()
|
||||
}
|
||||
@@ -6826,51 +6849,56 @@ mod tests {
|
||||
/// reload, refresh loop, lazy load) and withdraws on removal, mirroring
|
||||
/// `sync_bucket_durability`.
|
||||
#[tokio::test]
|
||||
async fn on_demand_migration_hook_fires_on_every_cache_install_path() {
|
||||
install_recording_odm_hook();
|
||||
async fn application_config_hook_fires_on_every_cache_install_path() {
|
||||
use crate::bucket::metadata::{BUCKET_NOTIFICATION_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
|
||||
|
||||
install_recording_config_hook();
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "odm-hook-paths";
|
||||
let bucket = PUBLISH_TEST_BUCKET;
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
|
||||
}
|
||||
|
||||
let incarnation = Uuid::new_v4();
|
||||
let expect_publish = |before: usize, label: &str| {
|
||||
let calls = odm_hook_calls(bucket);
|
||||
assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
|
||||
assert_eq!(
|
||||
calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
|
||||
Some(ODM_JSON),
|
||||
"{label} must publish the stored bytes"
|
||||
);
|
||||
assert_eq!(calls.last().unwrap().as_ref().map(|(_, _, id)| *id), Some(incarnation));
|
||||
for (file, bytes) in [
|
||||
(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON),
|
||||
(BUCKET_NOTIFICATION_CONFIG, NOTIFY_XML),
|
||||
] {
|
||||
let calls = config_hook_calls(file);
|
||||
assert_eq!(calls.len(), before + 1, "{label} must publish {file} exactly once");
|
||||
let (stored, _, id) = calls.last().unwrap().as_ref().expect("stored configuration");
|
||||
assert_eq!(stored, bytes, "{label} must publish the stored config");
|
||||
assert_eq!(*id, incarnation);
|
||||
}
|
||||
};
|
||||
|
||||
// set (via persist_new_and_set, which installs through `set`).
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.bucket_incarnation_id = incarnation;
|
||||
bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.unwrap();
|
||||
bm.update_config(BUCKET_NOTIFICATION_CONFIG, NOTIFY_XML.to_vec()).unwrap();
|
||||
let writer = BucketMetadataSys::new(ecstore.clone());
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
writer.persist_new_and_set(bm).await.expect("metadata should persist");
|
||||
expect_publish(before, "set");
|
||||
|
||||
// init (initial load on a cold system).
|
||||
let mut cold = BucketMetadataSys::new(ecstore.clone());
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
cold.init(vec![bucket.to_string()]).await;
|
||||
assert!(cold.get(bucket).await.is_ok(), "initial load must cache the bucket");
|
||||
expect_publish(before, "init");
|
||||
|
||||
// peer reload.
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
cold.reload_from_store(bucket).await.expect("peer reload should publish");
|
||||
expect_publish(before, "peer reload");
|
||||
|
||||
// refresh loop.
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
let mut failed = HashSet::new();
|
||||
cold.concurrent_load(&[bucket.to_string()], &mut failed, MetadataLoadMode::Refresh)
|
||||
.await;
|
||||
@@ -6879,24 +6907,28 @@ mod tests {
|
||||
|
||||
// lazy load on another cold system.
|
||||
let lazy = BucketMetadataSys::new(ecstore);
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
let (_, loaded) = lazy.get_config(bucket).await.expect("lazy load should publish");
|
||||
assert!(loaded, "the lazy path must have gone to disk");
|
||||
expect_publish(before, "lazy load");
|
||||
|
||||
// Removal withdraws the config.
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
assert!(lazy.remove(bucket).await);
|
||||
let calls = odm_hook_calls(bucket);
|
||||
let calls = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG);
|
||||
assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
|
||||
assert_eq!(calls.last().unwrap(), &None);
|
||||
|
||||
let notify_calls = config_hook_calls(BUCKET_NOTIFICATION_CONFIG);
|
||||
assert_eq!(notify_calls.len(), before + 1);
|
||||
assert_eq!(notify_calls.last(), Some(&None));
|
||||
|
||||
// Opaque bytes reach the application even if they are not valid JSON.
|
||||
let mut corrupt = BucketMetadata::new(bucket);
|
||||
corrupt.on_demand_migration_config_json = b"not-json".to_vec();
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let before = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG).len();
|
||||
lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
|
||||
let calls = odm_hook_calls(bucket);
|
||||
let calls = config_hook_calls(BUCKET_ON_DEMAND_MIGRATION_CONFIG);
|
||||
assert_eq!(calls.len(), before + 1);
|
||||
assert_eq!(
|
||||
calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
|
||||
|
||||
@@ -1070,8 +1070,8 @@ mod tests {
|
||||
.unwrap_or_else(|| panic!("signed request must contain {name}"))
|
||||
}
|
||||
|
||||
fn signing_time(headers: &[(String, String)]) -> jiff::Timestamp {
|
||||
jiff::Timestamp::strptime("%Y%m%dT%H%M%SZ", recorded_header(headers, "x-amz-date"))
|
||||
fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
|
||||
chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
|
||||
.expect("SDK signing timestamp must use the SigV4 format")
|
||||
}
|
||||
|
||||
@@ -1086,8 +1086,8 @@ mod tests {
|
||||
.map(|(key, value)| (key.to_string(), value.to_string()))
|
||||
.collect(),
|
||||
);
|
||||
let server_time = jiff::Timestamp::try_from(self.clock.now()).expect("clock timestamp")
|
||||
+ jiff::SignedDuration::from_secs(self.skew_seconds);
|
||||
let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
|
||||
+ chrono::Duration::seconds(self.skew_seconds);
|
||||
let (status, body) = if headers.len() == 1 {
|
||||
(
|
||||
403,
|
||||
@@ -1098,7 +1098,7 @@ mod tests {
|
||||
};
|
||||
let response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("date", server_time.strftime("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len())
|
||||
.body(SdkBody::from(body))
|
||||
@@ -1136,7 +1136,7 @@ mod tests {
|
||||
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
|
||||
for skew_seconds in [-600, 600] {
|
||||
let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
|
||||
let initial = jiff::Timestamp::try_from(clock.now()).expect("clock timestamp");
|
||||
let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
|
||||
client
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
@@ -1163,12 +1163,12 @@ mod tests {
|
||||
assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
|
||||
assert_eq!(
|
||||
signing_time(&headers[1]),
|
||||
initial + jiff::SignedDuration::from_secs(skew_seconds),
|
||||
initial + chrono::Duration::seconds(skew_seconds),
|
||||
"{error_code}: retry must apply the measured offset exactly"
|
||||
);
|
||||
assert_eq!(
|
||||
signing_time(&headers[2]),
|
||||
initial + jiff::SignedDuration::from_secs(skew_seconds + 17),
|
||||
initial + chrono::Duration::seconds(skew_seconds + 17),
|
||||
"{error_code}: the next operation must apply cached skew to the advanced signing clock"
|
||||
);
|
||||
let signature = |index: usize| {
|
||||
|
||||
@@ -13,6 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use crate::services::event_notification::EventArgs;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::services::event_notification::test_recorder;
|
||||
|
||||
use super::runtime_boundary;
|
||||
|
||||
|
||||
@@ -89,7 +89,11 @@ impl ReplicationLifecycleBridge {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
|
||||
pub(crate) async fn schedule_delete(
|
||||
bucket: String,
|
||||
delete_object: DeletedObject,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) {
|
||||
#[cfg(test)]
|
||||
{
|
||||
SCHEDULED_DELETE_OBJECTS
|
||||
@@ -98,6 +102,7 @@ impl ReplicationLifecycleBridge {
|
||||
.push(delete_object.clone());
|
||||
}
|
||||
super::replication_pool::schedule_replication_delete(DeletedObjectReplicationInfo {
|
||||
notification_bucket_incarnation,
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
bucket,
|
||||
event_type: REPLICATE_INCOMING_DELETE.to_string(),
|
||||
|
||||
@@ -22,6 +22,10 @@ use rustfs_utils::path::path_join_buf;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use super::replication_error_boundary::{Error, Result};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::metadata_sys::{
|
||||
bucket_metadata_sys_of, init_bucket_metadata_sys, set_bucket_metadata_in, test_support,
|
||||
};
|
||||
|
||||
pub(crate) type ReplicationInstanceContext = InstanceContext;
|
||||
|
||||
|
||||
@@ -110,13 +110,16 @@ impl ReplicationObjectBridge {
|
||||
check_replicate_delete_with_snapshot(object, source, opts, source_error, snapshot)
|
||||
}
|
||||
|
||||
/// Admits replication with the bucket identity verified by the originating mutation.
|
||||
/// An absent identity suppresses external notifications; workers never infer one by name.
|
||||
pub async fn schedule_object<S: ReplicationStorage>(
|
||||
object: ObjectInfo,
|
||||
storage: Arc<S>,
|
||||
decision: ReplicateDecision,
|
||||
op_type: ReplicationType,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) {
|
||||
schedule_replication(object, storage, decision, op_type).await;
|
||||
schedule_replication(object, storage, decision, op_type, notification_bucket_incarnation).await;
|
||||
}
|
||||
|
||||
pub async fn schedule_delete(delete_object: DeletedObjectReplicationInfo) {
|
||||
@@ -148,8 +151,16 @@ impl ReplicationObjectBridge {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn schedule_storage_delete(delete_object: DeletedObject, bucket: String, event_type: String) {
|
||||
/// Schedules a committed delete with its originating bucket identity for notifications.
|
||||
/// An absent identity leaves external notifications unbound.
|
||||
pub async fn schedule_storage_delete(
|
||||
delete_object: DeletedObject,
|
||||
bucket: String,
|
||||
event_type: String,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) {
|
||||
Self::schedule_delete(DeletedObjectReplicationInfo {
|
||||
notification_bucket_incarnation,
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
bucket,
|
||||
event_type,
|
||||
@@ -158,10 +169,18 @@ impl ReplicationObjectBridge {
|
||||
.await;
|
||||
}
|
||||
|
||||
pub async fn schedule_storage_deletes(delete_objects: Vec<DeletedObject>, bucket: String, event_type: String) {
|
||||
/// Schedules committed deletes from one bucket instance, retaining that notification identity.
|
||||
/// An absent identity leaves external notifications unbound.
|
||||
pub async fn schedule_storage_deletes(
|
||||
delete_objects: Vec<DeletedObject>,
|
||||
bucket: String,
|
||||
event_type: String,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) {
|
||||
let delete_objects = delete_objects
|
||||
.into_iter()
|
||||
.map(|delete_object| DeletedObjectReplicationInfo {
|
||||
notification_bucket_incarnation,
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
bucket: bucket.clone(),
|
||||
event_type: event_type.clone(),
|
||||
|
||||
@@ -1016,7 +1016,9 @@ async fn replay_mrf_metadata_entry<S: ReplicationStorage>(
|
||||
/// admitted targets, carrying over the entry's retry count.
|
||||
fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_type: ReplicationType) -> ReplicateObjectInfo {
|
||||
let dsc = replicate_decision_for_admitted_targets(&entry.target_arns);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type);
|
||||
// The journal does not bind this task to a bucket instance. Keep external
|
||||
// notifications unbound instead of attaching an old task to a reused name.
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, op_type, None);
|
||||
roi.retry_count = entry.retry_count.max(0) as u32;
|
||||
roi
|
||||
}
|
||||
@@ -3171,12 +3173,13 @@ pub(crate) async fn schedule_replication<S: ReplicationStorage>(
|
||||
o: Arc<S>,
|
||||
dsc: ReplicateDecision,
|
||||
op_type: ReplicationType,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) {
|
||||
let (synchronous, asynchronous) = dsc.partition_by_sync();
|
||||
let mut async_oi = oi;
|
||||
|
||||
if synchronous.replicate_any() {
|
||||
let ri = replicate_object_info_from_object_info(async_oi.clone(), synchronous, op_type);
|
||||
let ri = replicate_object_info_from_object_info(async_oi.clone(), synchronous, op_type, notification_bucket_incarnation);
|
||||
let state = replicate_object(ri, o.clone()).await;
|
||||
async_oi.replication_status_internal = state.replication_status_internal;
|
||||
async_oi.version_purge_status_internal = state.version_purge_status_internal;
|
||||
@@ -3185,7 +3188,7 @@ pub(crate) async fn schedule_replication<S: ReplicationStorage>(
|
||||
if asynchronous.replicate_any()
|
||||
&& let Some(pool) = runtime_sources::replication_pool()
|
||||
{
|
||||
let ri = replicate_object_info_from_object_info(async_oi, asynchronous, op_type);
|
||||
let ri = replicate_object_info_from_object_info(async_oi, asynchronous, op_type, notification_bucket_incarnation);
|
||||
let _ = pool.queue_replica_task(ri).await;
|
||||
}
|
||||
}
|
||||
@@ -3194,6 +3197,7 @@ fn replicate_object_info_from_object_info(
|
||||
oi: ObjectInfo,
|
||||
dsc: ReplicateDecision,
|
||||
op_type: ReplicationType,
|
||||
notification_bucket_incarnation: Option<uuid::Uuid>,
|
||||
) -> ReplicateObjectInfo {
|
||||
let tgt_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default());
|
||||
let purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default());
|
||||
@@ -3209,6 +3213,7 @@ fn replicate_object_info_from_object_info(
|
||||
let checksum = if ssec { oi.checksum.clone() } else { None };
|
||||
|
||||
ReplicateObjectInfo {
|
||||
notification_bucket_incarnation,
|
||||
name: oi.name,
|
||||
size: oi.size,
|
||||
actual_size: asz,
|
||||
@@ -3350,7 +3355,7 @@ pub async fn queue_replication_metadata(bucket: &str, oi: ObjectInfo, retry_coun
|
||||
return ReplicationQueueAdmission::Skipped;
|
||||
}
|
||||
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata);
|
||||
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata, None);
|
||||
roi.retry_count = retry_count;
|
||||
if let Some(pool) = runtime_sources::replication_pool() {
|
||||
pool.queue_replica_task(roi).await
|
||||
@@ -5205,6 +5210,117 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_admission_keeps_original_incarnation_and_mrf_replay_does_not_guess() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let target = "arn:rustfs:replication:target-a";
|
||||
for op_type in [ReplicationType::Object, ReplicationType::Metadata] {
|
||||
let object = ObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
let admitted = replicate_object_info_from_object_info(
|
||||
object.clone(),
|
||||
test_replicate_decision(&[target]),
|
||||
op_type,
|
||||
Some(incarnation),
|
||||
);
|
||||
assert_eq!(admitted.notification_bucket_incarnation, Some(incarnation));
|
||||
let encoded = encode_mrf_file(&[admitted.to_mrf_entry()]).expect("encode queued replication");
|
||||
let entries = decode_mrf_file(&encoded).expect("decode queued replication");
|
||||
let replayed = admitted_mrf_replicate_object(object, &entries[0], op_type);
|
||||
assert_eq!(
|
||||
replayed.notification_bucket_incarnation, None,
|
||||
"MRF replay must not infer a bucket instance from a reused name"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial_test::serial]
|
||||
async fn queued_delete_notifications_keep_origin_across_bucket_recreation() {
|
||||
use super::super::replication_event_sink::test_recorder;
|
||||
use super::super::replication_metadata_boundary::{
|
||||
bucket_metadata_sys_of, init_bucket_metadata_sys, set_bucket_metadata_in,
|
||||
test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
use super::super::replication_storage_boundary::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
runtime_sources::set_test_object_layer(store.clone()).await;
|
||||
test_recorder::install();
|
||||
let bucket = "queued-delete-notification-origin";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create original bucket");
|
||||
let original = store.bucket_incarnation_id(bucket).await.expect("original incarnation");
|
||||
let queued = DeletedObjectReplicationInfo {
|
||||
notification_bucket_incarnation: Some(original),
|
||||
bucket: bucket.to_string(),
|
||||
target_arn: "arn:rustfs:replication:target-a".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "original-object".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: "malformed-decision".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let encoded = encode_mrf_file(&[queued.to_mrf_entry()]).expect("persist old delete task");
|
||||
let entries = decode_mrf_file(&encoded).expect("recover old delete task");
|
||||
let mut replayed = reconstructed_heal_delete_info(
|
||||
&entries[0],
|
||||
&ObjectInfo::default(),
|
||||
&replicate_decision_for_admitted_targets(&entries[0].target_arns),
|
||||
);
|
||||
replayed
|
||||
.delete_object
|
||||
.replication_state
|
||||
.as_mut()
|
||||
.expect("reconstructed replication state")
|
||||
.replicate_decision_str = "malformed-decision".to_string();
|
||||
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create replacement bucket");
|
||||
let replacement = store.bucket_incarnation_id(bucket).await.expect("replacement incarnation");
|
||||
assert_ne!(original, replacement);
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
|
||||
let mut metadata = (*sys.read().await.get(bucket).await.expect("replacement metadata")).clone();
|
||||
metadata.notification_config_xml = br#"<NotificationConfiguration><QueueConfiguration><Id>replacement</Id><Queue>arn:rustfs:sqs:us-east-1:replacement:webhook</Queue><Event>s3:Replication:*</Event></QueueConfiguration></NotificationConfiguration>"#.to_vec();
|
||||
set_bucket_metadata_in(&store.ctx, metadata)
|
||||
.await
|
||||
.expect("install replacement notification configuration");
|
||||
|
||||
assert!(!replicate_delete_with_outcome(queued, store.clone()).await);
|
||||
assert!(!replicate_delete_with_outcome(replayed, store).await);
|
||||
let events = test_recorder::recorded_for_bucket(bucket);
|
||||
assert_eq!(events.len(), 2, "both worker failures must emit an observable local event");
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.all(|event| event.object == "original-object" && event.event_name == "s3:Replication:OperationNotTracked")
|
||||
);
|
||||
assert_eq!(events[0].bucket_incarnation_id, Some(original));
|
||||
assert_eq!(
|
||||
events[1].bucket_incarnation_id, None,
|
||||
"a replay without provenance must not acquire the replacement bucket's identity"
|
||||
);
|
||||
assert!(events.iter().all(|event| event.bucket_incarnation_id != Some(replacement)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replicate_object_info_from_object_info_preserves_ssec_checksum() {
|
||||
let checksum = bytes::Bytes::from_static(b"ssec-checksum");
|
||||
@@ -5219,7 +5335,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let ri = replicate_object_info_from_object_info(oi, ReplicateDecision::default(), ReplicationType::Object);
|
||||
let ri = replicate_object_info_from_object_info(oi, ReplicateDecision::default(), ReplicationType::Object, None);
|
||||
|
||||
assert!(ri.ssec);
|
||||
assert_eq!(ri.checksum, Some(checksum));
|
||||
@@ -5235,8 +5351,12 @@ mod tests {
|
||||
user_tags: Arc::new("owner=a3".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let live =
|
||||
replicate_object_info_from_object_info(object.clone(), test_replicate_decision(&[target]), ReplicationType::Metadata);
|
||||
let live = replicate_object_info_from_object_info(
|
||||
object.clone(),
|
||||
test_replicate_decision(&[target]),
|
||||
ReplicationType::Metadata,
|
||||
None,
|
||||
);
|
||||
let persisted = live.to_mrf_entry();
|
||||
let encoded = encode_mrf_file(std::slice::from_ref(&persisted)).expect("metadata MRF entry should encode");
|
||||
let decoded = decode_mrf_file(&encoded).expect("metadata MRF entry should decode");
|
||||
|
||||
@@ -2275,6 +2275,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
let replication_generation = oi.replication_generation_snapshot();
|
||||
|
||||
Ok(ReplicateObjectInfo {
|
||||
notification_bucket_incarnation: None,
|
||||
name: oi.name.clone(),
|
||||
size: oi.size,
|
||||
actual_size,
|
||||
@@ -2449,6 +2450,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
"Failed to parse replicate decision"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -2530,6 +2532,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
"Failed to verify source delete-marker state before replication"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -2564,6 +2567,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
"Skipping replication delete"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -2594,6 +2598,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
"Skipping replication delete"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -2693,6 +2698,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
"Replication resync task failed"
|
||||
);
|
||||
send_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -2801,6 +2807,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
{
|
||||
Ok(object) => {
|
||||
send_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
object,
|
||||
@@ -2826,6 +2833,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
);
|
||||
}
|
||||
send_event(EventArgs {
|
||||
bucket_incarnation_id: dobj.notification_bucket_incarnation,
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3157,6 +3165,7 @@ async fn purge_stale_delete_marker_targets(bucket: &str, dobj: &DeletedObjectRep
|
||||
}
|
||||
|
||||
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) -> bool {
|
||||
let incarnation_id = dobj.notification_bucket_incarnation;
|
||||
let bucket = &dobj.bucket;
|
||||
let object_name = &dobj.delete_object.object_name;
|
||||
let admitted_target_arns = dobj.admitted_target_arns();
|
||||
@@ -3177,6 +3186,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Skipping replication force-delete because replication config is missing"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3200,6 +3210,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Skipping replication force-delete because replication config lookup failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3234,6 +3245,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Skipping replication force-delete"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3262,6 +3274,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Skipping replication force-delete"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3309,6 +3322,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Skipping replication force-delete because target client is unavailable"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3339,6 +3353,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Skipping replication force-delete"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationFailed.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -3375,6 +3390,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationFailed.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
@@ -4239,6 +4255,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
"Skipping replication object"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: roi.to_object_info(),
|
||||
@@ -4262,6 +4279,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
"Skipping replication object"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: roi.to_object_info(),
|
||||
@@ -4328,6 +4346,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
"Replication resync task failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: roi.to_object_info(),
|
||||
@@ -4472,6 +4491,7 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
|
||||
if publication.emit_event {
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
@@ -4614,6 +4634,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
"Skipping replication object target"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: self.to_object_info(),
|
||||
@@ -4629,6 +4650,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await
|
||||
else {
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: self.to_object_info(),
|
||||
@@ -4672,6 +4694,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
);
|
||||
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: self.to_object_info(),
|
||||
@@ -4703,6 +4726,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
"Skipping replication object target"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
@@ -4728,6 +4752,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
"Skipping replication object target"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
@@ -4862,6 +4887,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
@@ -4890,6 +4916,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
@@ -4987,6 +5014,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let Some(ssec_audit_required) = resolve_ssec_passthrough_gate(self.ssec, &tgt_client, &bucket, &object, &mut rinfo).await
|
||||
else {
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: self.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: self.to_object_info(),
|
||||
@@ -5020,7 +5048,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let size = match object_info.get_actual_size() {
|
||||
Ok(size) => size,
|
||||
Err(e) => {
|
||||
note_replicate_all_size_unavailable(&bucket, &tgt_client, object_info, &e);
|
||||
note_replicate_all_size_unavailable(&bucket, &tgt_client, object_info, &e, self.notification_bucket_incarnation);
|
||||
return rinfo;
|
||||
}
|
||||
};
|
||||
@@ -5029,7 +5057,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let transfer_size = if self.ssec { object_info.size } else { size };
|
||||
|
||||
if tgt_client.bucket.is_empty() {
|
||||
note_replicate_all_target_bucket_empty(&bucket, &tgt_client, object_info);
|
||||
note_replicate_all_target_bucket_empty(&bucket, &tgt_client, object_info, self.notification_bucket_incarnation);
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
@@ -5075,7 +5103,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let (put_opts, is_multipart) = match replication_put_object_options_for_target(&tgt_client, &object_info) {
|
||||
Ok((put_opts, is_mp)) => (put_opts, is_mp),
|
||||
Err(e) => {
|
||||
fail_replicate_all_put_options(&mut rinfo, &tgt_client, &bucket, object_info, &e, start_time);
|
||||
fail_replicate_all_put_options(
|
||||
&mut rinfo,
|
||||
&tgt_client,
|
||||
&bucket,
|
||||
object_info,
|
||||
&e,
|
||||
start_time,
|
||||
self.notification_bucket_incarnation,
|
||||
);
|
||||
return rinfo;
|
||||
}
|
||||
};
|
||||
@@ -5179,6 +5215,7 @@ fn note_replicate_all_target_offline(roi: &ReplicateObjectInfo, bucket: &str, tg
|
||||
"Skipped replication because target is offline"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: roi.to_object_info(),
|
||||
@@ -5217,6 +5254,7 @@ fn note_replicate_all_reader_unavailable(roi: &ReplicateObjectInfo, bucket: &str
|
||||
"Skipped replication because object reader is unavailable"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: roi.to_object_info(),
|
||||
@@ -5228,7 +5266,13 @@ fn note_replicate_all_reader_unavailable(roi: &ReplicateObjectInfo, bucket: &str
|
||||
|
||||
/// Log and notify that replication was skipped because the actual object size
|
||||
/// is unavailable.
|
||||
fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient, object_info: ObjectInfo, e: &std::io::Error) {
|
||||
fn note_replicate_all_size_unavailable(
|
||||
bucket: &str,
|
||||
tgt_client: &TargetClient,
|
||||
object_info: ObjectInfo,
|
||||
e: &std::io::Error,
|
||||
incarnation_id: Option<uuid::Uuid>,
|
||||
) {
|
||||
debug!(
|
||||
event = EVENT_RESYNC_RUNTIME_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -5241,6 +5285,7 @@ fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient,
|
||||
"Skipped replication because actual object size is unavailable"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: object_info,
|
||||
@@ -5251,7 +5296,12 @@ fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient,
|
||||
|
||||
/// Log and notify that replication was skipped because the target bucket is
|
||||
/// empty.
|
||||
fn note_replicate_all_target_bucket_empty(bucket: &str, tgt_client: &TargetClient, object_info: ObjectInfo) {
|
||||
fn note_replicate_all_target_bucket_empty(
|
||||
bucket: &str,
|
||||
tgt_client: &TargetClient,
|
||||
object_info: ObjectInfo,
|
||||
incarnation_id: Option<uuid::Uuid>,
|
||||
) {
|
||||
debug!(
|
||||
event = EVENT_RESYNC_RUNTIME_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -5263,6 +5313,7 @@ fn note_replicate_all_target_bucket_empty(bucket: &str, tgt_client: &TargetClien
|
||||
"Skipped replication because target bucket is empty"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: object_info,
|
||||
@@ -5376,6 +5427,7 @@ async fn resolve_replicate_all_action(
|
||||
"Skipping replication because newer target version exists"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: object_info.clone(),
|
||||
@@ -5499,6 +5551,7 @@ async fn resolve_replicate_all_action(
|
||||
"Failed replication head-object fallback"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: object_info,
|
||||
@@ -5531,6 +5584,7 @@ async fn resolve_replicate_all_action(
|
||||
);
|
||||
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: roi.notification_bucket_incarnation,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: object_info,
|
||||
@@ -5559,6 +5613,7 @@ fn fail_replicate_all_put_options(
|
||||
object_info: ObjectInfo,
|
||||
e: &Error,
|
||||
start_time: OffsetDateTime,
|
||||
incarnation_id: Option<uuid::Uuid>,
|
||||
) {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(e.to_string());
|
||||
@@ -5574,6 +5629,7 @@ fn fail_replicate_all_put_options(
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
bucket_incarnation_id: incarnation_id,
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.to_string(),
|
||||
object: object_info,
|
||||
|
||||
@@ -24,6 +24,8 @@ pub(crate) use crate::object_api::{
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage_api_contracts::bucket::{BucketOperations, DeleteBucketOptions, MakeBucketOptions};
|
||||
pub(crate) use crate::storage_api_contracts::list::{
|
||||
ListOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
|
||||
};
|
||||
|
||||
@@ -18,6 +18,8 @@ use super::replication_bandwidth_boundary::ReplicationBucketMonitor;
|
||||
use super::replication_pool::DynReplicationPool;
|
||||
use super::replication_state::ReplicationStats;
|
||||
use super::replication_storage_boundary::ReplicationObjectStore;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::runtime::global::set_object_layer as set_test_object_layer;
|
||||
use crate::runtime::sources;
|
||||
pub use crate::runtime::sources::{
|
||||
ScannerDirtyUsageMutationObserver, ScannerDirtyUsageMutationSource, set_scanner_dirty_usage_mutation_observer,
|
||||
|
||||
@@ -73,11 +73,10 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataBatchItem, RenameDataBatchRequest,
|
||||
RenameDataBatchResult, RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
|
||||
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest,
|
||||
VerifyFileRequest, WriteAllRequest, WriteAllResponse, WriteFormatControlRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteAllResponse,
|
||||
WriteFormatControlRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use rustfs_protos::{
|
||||
ChannelClass, ChannelRebuildReason, ConnectionEvictionLogLevel, evict_failed_connection_with_reason, prewarm_channel_class,
|
||||
@@ -105,7 +104,7 @@ use tokio::{
|
||||
time::timeout,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::{Code, Request, Streaming, service::interceptor::InterceptedService};
|
||||
use tonic::{Code, Request, service::interceptor::InterceptedService};
|
||||
use tracing::{debug, trace, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -1046,7 +1045,6 @@ pub struct RemoteDisk {
|
||||
/// Optional per-peer gate held through complete bulk-unary decoding and validation.
|
||||
bulk_body_admission: Option<Arc<Semaphore>>,
|
||||
bulk_body_wait_timeout: Duration,
|
||||
rename_admission: Option<Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
/// Shortest gap between two failure warnings of one kind from one remote disk.
|
||||
@@ -1062,251 +1060,6 @@ struct BulkBodyAdmissionRegistry {
|
||||
static BULK_BODY_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<BulkBodyAdmissionRegistry>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(BulkBodyAdmissionRegistry::default()));
|
||||
|
||||
const ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH: &str = "RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH";
|
||||
const RENAME_ADMISSION_REGISTRY_CAPACITY: usize = 512;
|
||||
|
||||
#[derive(Default)]
|
||||
struct RenameAdmissionRegistry {
|
||||
peers: HashMap<String, Weak<Semaphore>>,
|
||||
overflow: Option<Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
static RENAME_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<RenameAdmissionRegistry>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(RenameAdmissionRegistry::default()));
|
||||
|
||||
fn peer_rename_admission(addr: &str, enabled: bool) -> Option<Arc<Semaphore>> {
|
||||
if !enabled {
|
||||
return None;
|
||||
}
|
||||
let mut registry = RENAME_ADMISSION_REGISTRY
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(semaphore) = registry.peers.get(addr).and_then(Weak::upgrade) {
|
||||
return Some(semaphore);
|
||||
}
|
||||
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
|
||||
registry.peers.retain(|_, semaphore| semaphore.strong_count() > 0);
|
||||
}
|
||||
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
|
||||
return Some(Arc::clone(registry.overflow.get_or_insert_with(|| {
|
||||
Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT))
|
||||
})));
|
||||
}
|
||||
let semaphore = Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT));
|
||||
registry.peers.insert(addr.to_owned(), Arc::downgrade(&semaphore));
|
||||
Some(semaphore)
|
||||
}
|
||||
|
||||
struct RenameAdmissionGrant {
|
||||
permit: Option<OwnedSemaphorePermit>,
|
||||
batch: bool,
|
||||
}
|
||||
|
||||
impl RenameAdmissionGrant {
|
||||
fn release_one(&mut self) -> bool {
|
||||
self.permit.as_mut().is_some_and(|permit| permit.split(1).is_some())
|
||||
}
|
||||
|
||||
fn release_all(mut self) {
|
||||
drop(self.permit.take());
|
||||
}
|
||||
|
||||
fn quarantine_until(mut self, deadline: tokio::time::Instant) {
|
||||
let Some(permit) = self.permit.take() else {
|
||||
return;
|
||||
};
|
||||
let items = permit.num_permits();
|
||||
if items == 0 {
|
||||
return;
|
||||
}
|
||||
let hold = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine(self.batch, items, hold);
|
||||
let batch = self.batch;
|
||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||
handle.spawn(async move {
|
||||
tokio::time::sleep_until(deadline).await;
|
||||
drop(permit);
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine_released(batch, items);
|
||||
});
|
||||
} else {
|
||||
// Without a runtime there is no safe deadline task. Leaking until
|
||||
// process exit is safer than admitting work over a still-running mutation.
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine_no_runtime(batch, items);
|
||||
std::mem::forget(permit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct RenameBatchCallGuard {
|
||||
admission: Option<RenameAdmissionGrant>,
|
||||
deadline: tokio::time::Instant,
|
||||
dispatch_state: RenameAdmissionDispatchState,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum RenameAdmissionDispatchState {
|
||||
NotDispatched,
|
||||
MayHavePublished,
|
||||
}
|
||||
|
||||
impl RenameBatchCallGuard {
|
||||
fn mark_dispatched(&mut self) {
|
||||
self.dispatch_state = RenameAdmissionDispatchState::MayHavePublished;
|
||||
}
|
||||
|
||||
fn complete(mut self) {
|
||||
debug_assert_eq!(self.dispatch_state, RenameAdmissionDispatchState::MayHavePublished);
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.release_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn release_without_quarantine(mut self) {
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.release_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn into_stream(mut self, inner: Streaming<RenameDataBatchResult>, expected_mask: u16) -> RemoteRenameBatchStream {
|
||||
debug_assert_eq!(self.dispatch_state, RenameAdmissionDispatchState::MayHavePublished);
|
||||
RemoteRenameBatchStream {
|
||||
inner,
|
||||
expected_mask,
|
||||
seen_mask: 0,
|
||||
admission: self.admission.take(),
|
||||
deadline: self.deadline,
|
||||
terminal: false,
|
||||
stream_started_at: rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now),
|
||||
first_item_at: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RenameBatchCallGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.dispatch_state == RenameAdmissionDispatchState::MayHavePublished
|
||||
&& let Some(admission) = self.admission.take()
|
||||
{
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenameBatchClientItem<'a> {
|
||||
pub slot: usize,
|
||||
pub disk: String,
|
||||
pub file_info: &'a FileInfo,
|
||||
}
|
||||
|
||||
pub(crate) enum RenameBatchStartError {
|
||||
NotDispatched(Error),
|
||||
UnsupportedBeforeStream,
|
||||
MayHavePublished(Error),
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteRenameBatchOutcome {
|
||||
pub slot: usize,
|
||||
pub result: Result<RenameDataResp>,
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteRenameBatchStream {
|
||||
inner: Streaming<RenameDataBatchResult>,
|
||||
expected_mask: u16,
|
||||
seen_mask: u16,
|
||||
admission: Option<RenameAdmissionGrant>,
|
||||
deadline: tokio::time::Instant,
|
||||
terminal: bool,
|
||||
stream_started_at: Option<Instant>,
|
||||
first_item_at: Option<Instant>,
|
||||
}
|
||||
|
||||
impl RemoteRenameBatchStream {
|
||||
fn terminate_ambiguous(&mut self, error: Error) -> Error {
|
||||
if !self.terminal {
|
||||
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
self.terminal = true;
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
pub(crate) async fn next_outcome(&mut self) -> Result<Option<RemoteRenameBatchOutcome>> {
|
||||
if self.terminal {
|
||||
return Ok(None);
|
||||
}
|
||||
let response = match tokio::time::timeout_at(self.deadline, self.inner.message()).await {
|
||||
Err(_) => return Err(self.terminate_ambiguous(DiskError::Timeout)),
|
||||
Ok(Err(status)) => return Err(self.terminate_ambiguous(status.into())),
|
||||
Ok(Ok(None)) if self.seen_mask == self.expected_mask => {
|
||||
self.terminal = true;
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Ok(None)) => return Err(self.terminate_ambiguous(DiskError::Unexpected)),
|
||||
Ok(Ok(Some(response))) => response,
|
||||
};
|
||||
let Ok(slot) = usize::try_from(response.item_index) else {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
};
|
||||
if slot >= 16 {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
let bit = 1_u16 << slot;
|
||||
if self.expected_mask & bit == 0 || self.seen_mask & bit != 0 {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
self.seen_mask |= bit;
|
||||
if self.admission.as_mut().is_some_and(|admission| !admission.release_one()) {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
if let Some(stream_started_at) = self.stream_started_at {
|
||||
let completed_at = Instant::now();
|
||||
match self.first_item_at {
|
||||
Some(first_item_at) if self.seen_mask == self.expected_mask => {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_first_to_last(
|
||||
completed_at.saturating_duration_since(first_item_at),
|
||||
);
|
||||
}
|
||||
Some(_) => {}
|
||||
None => {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_stream_to_first(
|
||||
completed_at.saturating_duration_since(stream_started_at),
|
||||
);
|
||||
self.first_item_at = Some(completed_at);
|
||||
if self.seen_mask == self.expected_mask {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_first_to_last(Duration::ZERO);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_item_completed();
|
||||
let result = if response.success {
|
||||
if response.error.is_some() {
|
||||
Err(DiskError::FileCorrupt)
|
||||
} else {
|
||||
decode_msgpack_strict(&response.rename_data_resp_msgpack)
|
||||
}
|
||||
} else {
|
||||
Err(response.error.unwrap_or_default().into())
|
||||
};
|
||||
Ok(Some(RemoteRenameBatchOutcome { slot, result }))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RemoteRenameBatchStream {
|
||||
fn drop(&mut self) {
|
||||
if !self.terminal {
|
||||
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BulkBodyPermit {
|
||||
_metrics: rustfs_io_metrics::internode_metrics::InternodeBulkBodyAdmissionGuard,
|
||||
_permit: OwnedSemaphorePermit,
|
||||
@@ -1473,10 +1226,6 @@ fn spawn_control_channel_prewarm(addr: String) {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
pub(crate) fn rename_stream_batch_enabled(&self) -> bool {
|
||||
self.rename_admission.is_some()
|
||||
}
|
||||
|
||||
async fn guarded_delete_versions_rpc(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1802,8 +1551,6 @@ impl RemoteDisk {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
|
||||
let bulk_body_admission = peer_bulk_body_admission(&addr, bulk_body_concurrency_limit()?);
|
||||
let bulk_body_wait_timeout = bulk_body_wait_timeout();
|
||||
let rename_admission =
|
||||
peer_rename_admission(&addr, rustfs_utils::get_env_bool(ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH, false));
|
||||
|
||||
let disk = Self {
|
||||
handle_id: Uuid::new_v4(),
|
||||
@@ -1826,7 +1573,6 @@ impl RemoteDisk {
|
||||
network_error_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
|
||||
bulk_body_admission,
|
||||
bulk_body_wait_timeout,
|
||||
rename_admission,
|
||||
};
|
||||
record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);
|
||||
|
||||
@@ -2562,54 +2308,6 @@ impl RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_peer_batch_with_timeout<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
operation: F,
|
||||
timeout_duration: Duration,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
if timeout_duration == Duration::ZERO {
|
||||
self.record_timeout(op, timeout_duration);
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match time::timeout(timeout_duration, operation()).await {
|
||||
Ok(operation_result) => {
|
||||
if operation_result.is_ok() {
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
let evict = operation_result
|
||||
.as_ref()
|
||||
.err()
|
||||
.is_some_and(|error| !error.is_metacache_output_stream_closed() && is_network_like_disk_error(error));
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, FailureHealthAction::IgnoreFailure)
|
||||
.await;
|
||||
if evict {
|
||||
evict_failed_connection_with_reason(
|
||||
&self.addr,
|
||||
ChannelRebuildReason::RpcFailure,
|
||||
ConnectionEvictionLogLevel::Debug,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
operation_result
|
||||
}
|
||||
Err(_) => {
|
||||
self.record_timeout(op, timeout_duration);
|
||||
evict_failed_connection_with_reason(
|
||||
&self.addr,
|
||||
ChannelRebuildReason::OperationTimeout,
|
||||
ConnectionEvictionLogLevel::Debug,
|
||||
)
|
||||
.await;
|
||||
Err(DiskError::Timeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_timeout(&self, op: &'static str, timeout_duration: Duration) {
|
||||
counter!(
|
||||
"rustfs_drive_op_timeout_total",
|
||||
@@ -2796,38 +2494,6 @@ impl RemoteDisk {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn acquire_rename_admission_until(
|
||||
&self,
|
||||
item_count: usize,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<Option<RenameAdmissionGrant>> {
|
||||
let Some(semaphore) = &self.rename_admission else {
|
||||
return Ok(None);
|
||||
};
|
||||
if item_count == 0 || item_count > rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT {
|
||||
return Err(Error::other("invalid internode rename admission weight"));
|
||||
}
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
}
|
||||
let weight = u32::try_from(item_count).map_err(Error::other)?;
|
||||
let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
let (result, timed_out) = match tokio::time::timeout_at(deadline, Arc::clone(semaphore).acquire_many_owned(weight)).await
|
||||
{
|
||||
Ok(Ok(permit)) => (Ok(permit), false),
|
||||
Ok(Err(_)) => (Err(Error::RemoteClientUnavailable("internode rename admission closed".to_owned())), false),
|
||||
Err(_) => (Err(DiskError::DiskOngoingReq), true),
|
||||
};
|
||||
if let Some(started) = started {
|
||||
rustfs_io_metrics::record_ecstore_rename_admission(item_count > 1, item_count, started.elapsed(), timed_out);
|
||||
}
|
||||
let permit = result?;
|
||||
Ok(Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: item_count > 1,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get_bulk_client(&self) -> Result<NodeRpcClient> {
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
@@ -2968,18 +2634,6 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_msgpack_strict<T: DeserializeOwned>(binary: &[u8]) -> Result<T> {
|
||||
if binary.is_empty() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let mut cursor = Cursor::new(binary);
|
||||
let value = rmp_serde::from_read(&mut cursor).map_err(|_| DiskError::FileCorrupt)?;
|
||||
if cursor.position() != u64::try_from(binary.len()).map_err(Error::other)? {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn decode_multipart_publication_response(binary: &[u8]) -> Result<RenameDataResp> {
|
||||
let result = if binary.len() > MultipartSourceOperation::max_wire_payload_bytes() {
|
||||
Err(DiskError::FileCorrupt)
|
||||
@@ -3177,97 +2831,6 @@ impl RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn start_rename_data_batch(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
items: Vec<RenameBatchClientItem<'_>>,
|
||||
) -> std::result::Result<RemoteRenameBatchStream, RenameBatchStartError> {
|
||||
if self.rename_admission.is_none() || items.is_empty() || items.len() > 16 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::UnsupportedDisk));
|
||||
}
|
||||
let deadline = tokio::time::Instant::now() + get_max_timeout_duration();
|
||||
let mut expected_mask = 0_u16;
|
||||
let mut request_items = Vec::with_capacity(items.len());
|
||||
for item in items {
|
||||
if item.slot >= 16 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
|
||||
}
|
||||
let bit = 1_u16 << item.slot;
|
||||
if expected_mask & bit != 0 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
|
||||
}
|
||||
expected_mask |= bit;
|
||||
let item_index = u32::try_from(item.slot)
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let file_info_msgpack = encode_file_info_msgpack(item.file_info).map_err(RenameBatchStartError::NotDispatched)?;
|
||||
request_items.push(RenameDataBatchItem {
|
||||
item_index,
|
||||
disk: item.disk,
|
||||
file_info_msgpack: file_info_msgpack.into(),
|
||||
});
|
||||
}
|
||||
let mut request = Request::new(RenameDataBatchRequest {
|
||||
src_volume: src_volume.to_owned(),
|
||||
src_path: src_path.to_owned(),
|
||||
dst_volume: dst_volume.to_owned(),
|
||||
dst_path: dst_path.to_owned(),
|
||||
items: request_items,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_batch_request_body(request.get_ref())
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body)
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let mut client = match tokio::time::timeout_at(deadline, self.get_client()).await {
|
||||
Ok(result) => result.map_err(RenameBatchStartError::NotDispatched)?,
|
||||
Err(_) => return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout)),
|
||||
};
|
||||
let admission = self
|
||||
.acquire_rename_admission_until(request.get_ref().items.len(), deadline)
|
||||
.await
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let guard = RenameBatchCallGuard {
|
||||
admission,
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
let call_timeout = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
if call_timeout == Duration::ZERO {
|
||||
guard.release_without_quarantine();
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout));
|
||||
}
|
||||
let mut guard = guard;
|
||||
guard.mark_dispatched();
|
||||
let response = match self
|
||||
.execute_peer_batch_with_timeout(
|
||||
"rename_data_batch",
|
||||
|| async {
|
||||
match client.rename_data_batch(request).await {
|
||||
Ok(response) => Ok(response),
|
||||
Err(status) if status.code() == Code::Unimplemented => Err(DiskError::UnsupportedDisk),
|
||||
Err(status) => Err(status.into()),
|
||||
}
|
||||
},
|
||||
call_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(DiskError::UnsupportedDisk) => {
|
||||
guard.release_without_quarantine();
|
||||
return Err(RenameBatchStartError::UnsupportedBeforeStream);
|
||||
}
|
||||
Err(error) => return Err(RenameBatchStartError::MayHavePublished(error)),
|
||||
Ok(response) => response,
|
||||
};
|
||||
Ok(guard.into_stream(response.into_inner(), expected_mask))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
@@ -3339,27 +2902,6 @@ impl RemoteDisk {
|
||||
let scanner_publication_lease = scanner_publication_lease_token
|
||||
.map(|token| Bytes::copy_from_slice(token.as_bytes()))
|
||||
.unwrap_or_default();
|
||||
let rename_deadline = tokio::time::Instant::now() + get_max_timeout_duration();
|
||||
let rename_admission = self.acquire_rename_admission_until(1, rename_deadline).await?;
|
||||
let mut rename_admission_guard = RenameBatchCallGuard {
|
||||
admission: rename_admission,
|
||||
deadline: rename_deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
let rpc_kind = if multipart_rename {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_MULTIPART
|
||||
} else if pool_meta_cas {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_POOL_META_CAS
|
||||
} else if quota_fenced {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE
|
||||
} else if incarnation_bound {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_INCARNATION
|
||||
} else if scanner_publication_lease_token.is_some() {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE
|
||||
} else {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_ORDINARY
|
||||
};
|
||||
rustfs_io_metrics::record_ecstore_rename_remote_rpc(rpc_kind);
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let mut retried_dual_write = false;
|
||||
let response = loop {
|
||||
@@ -3416,7 +2958,6 @@ impl RemoteDisk {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
|
||||
attempt_prepare_started,
|
||||
);
|
||||
rename_admission_guard.mark_dispatched();
|
||||
let result = rustfs_io_metrics::observe_put_stage_future(
|
||||
async {
|
||||
if multipart_rename {
|
||||
@@ -3447,7 +2988,6 @@ impl RemoteDisk {
|
||||
Err(status) => return Err(status.into()),
|
||||
}
|
||||
};
|
||||
rename_admission_guard.complete();
|
||||
let response_process_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response_headers_started = rustfs_io_metrics::put_stage_timer();
|
||||
let _ = response.metadata().len();
|
||||
@@ -13423,127 +12963,6 @@ mod tests {
|
||||
.expect("shared gate remains open");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_admission_is_weighted_shared_and_released_per_item() {
|
||||
use std::future::Future;
|
||||
use std::task::{Context, Poll, Waker};
|
||||
|
||||
let first = peer_rename_admission("http://rename-peer-a", true).expect("enabled rename gate");
|
||||
let same = peer_rename_admission("http://rename-peer-a", true).expect("same peer rename gate");
|
||||
let other = peer_rename_admission("http://rename-peer-b", true).expect("other peer rename gate");
|
||||
assert!(Arc::ptr_eq(&first, &same));
|
||||
assert!(!Arc::ptr_eq(&first, &other));
|
||||
assert!(peer_rename_admission("http://rename-disabled", false).is_none());
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&first)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let mut grant = RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
};
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
assert!(grant.release_one());
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3);
|
||||
drop(grant);
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let held = Arc::clone(&first)
|
||||
.acquire_many_owned(u32::try_from(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3).expect("small limit"))
|
||||
.await
|
||||
.expect("reserve all but three permits");
|
||||
let mut waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
|
||||
assert!(matches!(waiting.as_mut().poll(&mut Context::from_waker(Waker::noop())), Poll::Pending));
|
||||
assert_eq!(first.available_permits(), 0, "the FIFO waiter reserves currently available permits");
|
||||
drop(waiting);
|
||||
assert_eq!(
|
||||
first.available_permits(),
|
||||
3,
|
||||
"cancelling a queued weighted acquire must return its reservation"
|
||||
);
|
||||
let waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
|
||||
drop(held);
|
||||
let acquired = tokio::time::timeout(Duration::from_secs(1), waiting)
|
||||
.await
|
||||
.expect("weighted waiter wakes")
|
||||
.expect("rename gate remains open");
|
||||
drop(acquired);
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_admission_quarantine_holds_until_absolute_deadline() {
|
||||
let gate = peer_rename_admission("http://rename-quarantine-peer", true).expect("enabled rename gate");
|
||||
let permit = Arc::clone(&gate)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
|
||||
RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}
|
||||
.quarantine_until(deadline);
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
|
||||
tokio::time::advance(Duration::from_millis(999)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_millis(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_call_guard_quarantines_only_after_dispatch() {
|
||||
let gate = peer_rename_admission("http://rename-call-guard-peer", true).expect("enabled rename gate");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("pre-dispatch permits");
|
||||
drop(RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
});
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("dispatched permits");
|
||||
let mut guard = RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
guard.mark_dispatched();
|
||||
drop(guard);
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&gate)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("proven pre-mutation permits");
|
||||
let mut guard = RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
guard.mark_dispatched();
|
||||
guard.release_without_quarantine();
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_disk_builds_optional_complete_body_gate_from_configuration() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, Some("3"))], async {
|
||||
|
||||
@@ -12378,7 +12378,6 @@ struct DecommissionCapacityLockOrderBarrierState {
|
||||
external_heal_target_lock_attempted: tokio::sync::Notify,
|
||||
target_gate_retry_entered: tokio::sync::Notify,
|
||||
target_gate_retry_entries: AtomicUsize,
|
||||
target_gate_retry_pause: std::sync::OnceLock<tokio::sync::Semaphore>,
|
||||
target_gate_inline_retry_entered: tokio::sync::Notify,
|
||||
target_gate_inline_retry_entries: AtomicUsize,
|
||||
target_gate_exact_reloads: AtomicUsize,
|
||||
@@ -12426,7 +12425,6 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
external_heal_target_lock_attempted: tokio::sync::Notify::new(),
|
||||
target_gate_retry_entered: tokio::sync::Notify::new(),
|
||||
target_gate_retry_entries: AtomicUsize::new(0),
|
||||
target_gate_retry_pause: std::sync::OnceLock::new(),
|
||||
target_gate_inline_retry_entered: tokio::sync::Notify::new(),
|
||||
target_gate_inline_retry_entries: AtomicUsize::new(0),
|
||||
target_gate_exact_reloads: AtomicUsize::new(0),
|
||||
@@ -12522,13 +12520,6 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
.expect("decommission entries should observe target gate contention");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn pause_target_gate_retries(&self) -> &tokio::sync::Semaphore {
|
||||
self.state
|
||||
.target_gate_retry_pause
|
||||
.get_or_init(|| tokio::sync::Semaphore::new(0))
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn target_gate_inline_retries(&self) -> usize {
|
||||
self.state.target_gate_inline_retry_entries.load(Ordering::Acquire)
|
||||
@@ -12639,9 +12630,6 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
#[cfg(test)]
|
||||
impl Drop for DecommissionCapacityLockOrderBarrier {
|
||||
fn drop(&mut self) {
|
||||
if let Some(pause) = self.state.target_gate_retry_pause.get() {
|
||||
pause.close();
|
||||
}
|
||||
self.state.owner_release.notify_one();
|
||||
self.state.external_object_capacity_probe_release.notify_one();
|
||||
self.state.external_object_commit_phase_release.notify_one();
|
||||
@@ -12770,7 +12758,7 @@ async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn observe_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
@@ -12781,11 +12769,6 @@ async fn observe_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.target_gate_retry_entries.fetch_add(1, Ordering::AcqRel);
|
||||
barrier.target_gate_retry_entered.notify_one();
|
||||
if let Some(pause) = barrier.target_gate_retry_pause.get()
|
||||
&& let Ok(permit) = pause.acquire().await
|
||||
{
|
||||
permit.forget();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17655,7 +17638,7 @@ impl ECStore {
|
||||
target_busy_attempt: usize,
|
||||
) -> Result<Option<DecommissionCapacityTargetPermit>> {
|
||||
#[cfg(test)]
|
||||
observe_decommission_target_gate_retry(self.id).await;
|
||||
notify_decommission_target_gate_retry(self.id);
|
||||
let mut wait_attempt = target_busy_attempt;
|
||||
let target_guard = loop {
|
||||
let retry_delay =
|
||||
|
||||
@@ -8360,16 +8360,13 @@ mod decommission_lock_order_tests {
|
||||
.await
|
||||
.expect("hold the target gate before starting both waiters");
|
||||
let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id);
|
||||
// Both workers must observe the original gate as busy, but slow disk I/O
|
||||
// in the holder or first waiter must not consume the second waiter's retry budget.
|
||||
let retry_pause = retry_observer.pause_target_gate_retries();
|
||||
|
||||
let mut workers = tokio::task::JoinSet::new();
|
||||
let mut workers = Vec::new();
|
||||
for object in objects {
|
||||
let worker_store = Arc::clone(&other_store);
|
||||
let worker_bucket = bucket.clone();
|
||||
let source_set = other_store.pools[0].get_disks_by_key(object);
|
||||
workers.spawn(async move {
|
||||
workers.push(tokio::spawn(async move {
|
||||
worker_store
|
||||
.decommission_entry_for_test(
|
||||
0,
|
||||
@@ -8381,7 +8378,7 @@ mod decommission_lock_order_tests {
|
||||
source_set,
|
||||
)
|
||||
.await
|
||||
});
|
||||
}));
|
||||
}
|
||||
retry_observer.wait_until_target_gate_retries(2).await;
|
||||
assert_eq!(
|
||||
@@ -8428,7 +8425,6 @@ mod decommission_lock_order_tests {
|
||||
);
|
||||
|
||||
drop(target_guard);
|
||||
retry_pause.add_permits(1);
|
||||
retry_observer.wait_until_owner_paused().await;
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
@@ -8447,13 +8443,6 @@ mod decommission_lock_order_tests {
|
||||
.expect("a pinned waiter must not hold the alternate target gate");
|
||||
drop(alternate_probe);
|
||||
retry_observer.release_owner();
|
||||
tokio::time::timeout(Duration::from_secs(30), workers.join_next())
|
||||
.await
|
||||
.expect("the pinned waiter should finish before the other retries")
|
||||
.expect("the pinned waiter must exist")
|
||||
.expect("the pinned waiter task should not panic")
|
||||
.expect("the pinned waiter migration should succeed");
|
||||
retry_pause.add_permits(1);
|
||||
retry_observer.wait_until_owner_paused().await;
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
@@ -8473,12 +8462,13 @@ mod decommission_lock_order_tests {
|
||||
drop(original_probe);
|
||||
retry_observer.release_owner();
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), workers.join_next())
|
||||
.await
|
||||
.expect("the reselected waiter should finish after permit transfer")
|
||||
.expect("the reselected waiter must exist")
|
||||
.expect("the reselected waiter task should not panic")
|
||||
.expect("the reselected waiter migration should succeed");
|
||||
for worker in workers {
|
||||
tokio::time::timeout(Duration::from_secs(30), worker)
|
||||
.await
|
||||
.expect("same-target waiter should finish after permit transfer")
|
||||
.expect("same-target waiter task should not panic")
|
||||
.expect("same-target waiter migration should succeed");
|
||||
}
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
2,
|
||||
|
||||
@@ -604,10 +604,10 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
|
||||
let disk = match new_disk(ep, opt).await {
|
||||
Ok(disk) => disk,
|
||||
Err(crate::disk::error::DiskError::FileAccessDenied | crate::disk::error::DiskError::VolumeNotFound) => {
|
||||
// Preserve the configured slot when a local volume is unavailable.
|
||||
// LocalDisk logs the cause; format loading retains authority
|
||||
// over fatal errors and quorum. Never create a missing root.
|
||||
Err(crate::disk::error::DiskError::FileAccessDenied) => {
|
||||
// LocalDisk already logged the filesystem cause and repair
|
||||
// hint. Keep this slot unavailable so ECStore initialization
|
||||
// can apply its existing fatal-error and quorum rules.
|
||||
global_local_disk_map.insert(ep.to_string(), None);
|
||||
continue;
|
||||
}
|
||||
@@ -888,95 +888,6 @@ mod tests {
|
||||
disk_map.keys().collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn missing_local_volume_preserves_offline_slot_without_creating_root() {
|
||||
use crate::disk::{DiskOption, error::DiskError};
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
|
||||
for drive_count in [2, 3, 4] {
|
||||
let directory = tempfile::tempdir().expect("create volume parent");
|
||||
let mut endpoints = Vec::new();
|
||||
for index in 0..drive_count {
|
||||
let path = directory.path().join(format!("disk{index}"));
|
||||
if index != 0 {
|
||||
std::fs::create_dir(&path).expect("create surviving volume");
|
||||
}
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
let missing = endpoints[0].to_string();
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: drive_count,
|
||||
endpoints: endpoints.into(),
|
||||
cmd_line: "missing-volume".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let options = DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
};
|
||||
super::initialize_local_disk_maps(&ctx, pools.clone(), &options)
|
||||
.await
|
||||
.expect("missing volume should remain offline for quorum evaluation");
|
||||
let map = ctx.local_disk_map();
|
||||
let map = map.read().await;
|
||||
assert_eq!(map.len(), drive_count);
|
||||
assert!(matches!(map.get(&missing), Some(None)));
|
||||
assert_eq!(map.values().flatten().count(), drive_count - 1);
|
||||
drop(map);
|
||||
let drives = ctx.local_disk_set_drives();
|
||||
let drives = drives.read().await;
|
||||
assert_eq!(drives[0][0].len(), drive_count);
|
||||
assert!(drives[0][0][0].is_none());
|
||||
assert!(drives[0][0][1..].iter().all(Option::is_some));
|
||||
drop(drives);
|
||||
|
||||
let (disks, errors) = crate::store::init_format::init_disks(&ctx, &pools.0[0].endpoints, &options).await;
|
||||
assert!(disks[0].is_none());
|
||||
assert_eq!(errors[0], Some(DiskError::VolumeNotFound));
|
||||
assert!(disks[1..].iter().all(Option::is_some));
|
||||
assert!(errors[1..].iter().all(Option::is_none));
|
||||
assert!(!directory.path().join("disk0").exists(), "startup must not create a missing mount root");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_local_volume_still_fails_registry_initialization() {
|
||||
use crate::disk::DiskOption;
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
|
||||
let directory = tempfile::tempdir().expect("create volume parent");
|
||||
let path = directory.path().join("regular-file");
|
||||
std::fs::write(&path, b"preserve existing file").expect("create invalid volume");
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 1,
|
||||
endpoints: vec![endpoint].into(),
|
||||
cmd_line: "invalid-volume".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let options = DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
};
|
||||
super::initialize_local_disk_maps(&ctx, pools, &options)
|
||||
.await
|
||||
.expect_err("a non-directory volume must retain its initialization error");
|
||||
assert_eq!(std::fs::read(&path).expect("read existing file"), b"preserve existing file");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_invalid_registry_pool_index_has_no_authority_or_registry_effect() {
|
||||
use crate::disk::DiskOption;
|
||||
|
||||
@@ -82,6 +82,8 @@ impl EventNotifier {
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct EventArgs {
|
||||
/// Bucket instance verified by storage when producing this event.
|
||||
pub bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
pub event_name: String,
|
||||
pub bucket_name: String,
|
||||
pub object: ObjectInfo,
|
||||
@@ -135,6 +137,7 @@ pub(crate) mod test_recorder {
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct RecordedEvent {
|
||||
pub(crate) event_name: String,
|
||||
pub(crate) bucket_incarnation_id: Option<Uuid>,
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) object: String,
|
||||
pub(crate) version_id: Option<Uuid>,
|
||||
@@ -154,6 +157,7 @@ pub(crate) mod test_recorder {
|
||||
register_event_dispatch_hook(|args| {
|
||||
recorded().lock().unwrap_or_else(|err| err.into_inner()).push(RecordedEvent {
|
||||
event_name: args.event_name,
|
||||
bucket_incarnation_id: args.bucket_incarnation_id,
|
||||
bucket: args.bucket_name,
|
||||
object: args.object.name,
|
||||
version_id: args.object.version_id,
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
use crate::diagnostics::admin_server_info::get_local_server_property;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::admin::StorageAdminApi;
|
||||
#[cfg(test)]
|
||||
use chrono::Utc;
|
||||
use jiff::Timestamp;
|
||||
use rustfs_heal_contracts::heal_channel::DriveState;
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
@@ -598,6 +600,10 @@ mod test {
|
||||
use serial_test::serial;
|
||||
use std::time::Duration;
|
||||
|
||||
fn chrono_to_jiff_timestamp(timestamp: chrono::DateTime<Utc>) -> jiff::Timestamp {
|
||||
jiff::Timestamp::try_from(std::time::SystemTime::from(timestamp)).expect("test timestamp should fit in jiff")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tes_types() {
|
||||
let t = MetricType::ALL;
|
||||
@@ -679,10 +685,10 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_mapping_preserves_partial_source_status() {
|
||||
let current_started = Timestamp::now() - jiff::SignedDuration::from_secs(5);
|
||||
let current_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
|
||||
current_cycle_active: true,
|
||||
current_started,
|
||||
current_started: chrono_to_jiff_timestamp(current_started),
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
cycle_timeout_total: 3,
|
||||
@@ -697,7 +703,7 @@ mod test {
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, current_started);
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
assert_eq!(scanner.cycle_timeout_total, 3);
|
||||
@@ -717,8 +723,8 @@ mod test {
|
||||
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
|
||||
let previous_init_time = *rustfs_scanner_metrics::GLOBAL_INIT_TIME.read().await;
|
||||
let previous_cycle = global_metrics().get_cycle().await;
|
||||
let init_time = Timestamp::now() - jiff::SignedDuration::from_hours(1);
|
||||
let cycle_started = Timestamp::now() - jiff::SignedDuration::from_secs(5);
|
||||
let init_time = Utc::now() - chrono::Duration::hours(1);
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
*rustfs_scanner_metrics::GLOBAL_INIT_TIME.write().await = Some(init_time);
|
||||
let cycle = CurrentCycle {
|
||||
current: 0,
|
||||
@@ -742,7 +748,7 @@ mod test {
|
||||
aggregated.merge(decoded);
|
||||
let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, cycle_started);
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(cycle_started));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17333,12 +17333,8 @@ mod tests {
|
||||
source.metadata_only = true;
|
||||
source.etag = Some("metadata-copy-etag".to_string());
|
||||
|
||||
let dst_opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..opts.clone()
|
||||
};
|
||||
let copied = set_disks
|
||||
.copy_object(bucket, object, bucket, object, &mut source, &opts, &dst_opts)
|
||||
.copy_object(bucket, object, bucket, object, &mut source, &opts, &opts)
|
||||
.await
|
||||
.expect("metadata self-copy should succeed");
|
||||
assert_eq!(copied.user_defined.get("x-amz-meta-copy-check").map(String::as_str), Some("present"));
|
||||
|
||||
@@ -9294,7 +9294,12 @@ impl SetDisks {
|
||||
)
|
||||
.await?;
|
||||
if let Some((_, deleted_object)) = replication_delete {
|
||||
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await;
|
||||
ReplicationLifecycleBridge::schedule_delete(
|
||||
bucket.to_string(),
|
||||
deleted_object,
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
deleted += 1;
|
||||
}
|
||||
@@ -11563,7 +11568,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
)
|
||||
.await?;
|
||||
if let Some((_, deleted_object)) = replication_delete {
|
||||
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await;
|
||||
ReplicationLifecycleBridge::schedule_delete(
|
||||
bucket.to_string(),
|
||||
deleted_object,
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -12732,6 +12742,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let obj_info = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
send_event(EventArgs {
|
||||
event_name: event_name.to_string(),
|
||||
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
|
||||
bucket_name: bucket.to_string(),
|
||||
object: obj_info,
|
||||
user_agent: "Internal: [ILM-Transition]".to_string(),
|
||||
@@ -12878,6 +12889,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, &opts).await?;
|
||||
send_event(EventArgs {
|
||||
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
|
||||
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
|
||||
bucket_name: bucket.to_string(),
|
||||
object: restored_info,
|
||||
user_agent: "Internal: [Restore-Completed]".to_string(),
|
||||
@@ -13014,6 +13026,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, opts).await?;
|
||||
send_event(EventArgs {
|
||||
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
|
||||
bucket_incarnation_id: opts.expected_bucket_incarnation_id,
|
||||
bucket_name: bucket.to_string(),
|
||||
object: restored_info,
|
||||
user_agent: "Internal: [Restore-Completed]".to_string(),
|
||||
@@ -17182,12 +17195,7 @@ mod metadata_mutation_generation_tests {
|
||||
copy_object,
|
||||
&mut copy_info,
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("metadata COPY should succeed");
|
||||
@@ -17365,11 +17373,7 @@ mod metadata_mutation_generation_tests {
|
||||
let metadata = Arc::make_mut(&mut source.user_defined);
|
||||
metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "new=value".to_string());
|
||||
metadata.insert("expires".to_string(), expires.format(&Rfc3339).expect("expiry"));
|
||||
let dst_opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..Default::default()
|
||||
};
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut source, &opts, &dst_opts)
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut source, &opts, &opts)
|
||||
.await
|
||||
.expect("explicit metadata wins over externalized fields");
|
||||
let mut copied = set.get_object_info(bucket, "object", &opts).await.expect("copied metadata");
|
||||
@@ -17380,7 +17384,7 @@ mod metadata_mutation_generation_tests {
|
||||
copied.metadata_only = true;
|
||||
copied.user_tags = Arc::new(String::new());
|
||||
copied.expires = None;
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut copied, &opts, &dst_opts)
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut copied, &opts, &opts)
|
||||
.await
|
||||
.expect("explicit empty replacement");
|
||||
let cleared = set.get_object_info(bucket, "object", &opts).await.expect("cleared metadata");
|
||||
@@ -17397,10 +17401,6 @@ mod metadata_mutation_generation_tests {
|
||||
}
|
||||
|
||||
let valid_object = "valid-sidecar";
|
||||
let dst_opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..Default::default()
|
||||
};
|
||||
let (mut valid_source, _) = put_and_prime(&set_disks, bucket, valid_object, b"valid sidecar body").await;
|
||||
persist_part_checksum_sidecar(&set_disks, bucket, valid_object, r#"[[1,[["CRC32C","AAAAAA=="]]]]"#).await;
|
||||
valid_source.metadata_only = true;
|
||||
@@ -17413,7 +17413,7 @@ mod metadata_mutation_generation_tests {
|
||||
valid_object,
|
||||
&mut valid_source,
|
||||
&ObjectOptions::default(),
|
||||
&dst_opts,
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("metadata-only copy should preserve a valid checksum sidecar");
|
||||
@@ -17486,7 +17486,7 @@ mod metadata_mutation_generation_tests {
|
||||
conflicting_object,
|
||||
&mut conflicting_source,
|
||||
&ObjectOptions::default(),
|
||||
&dst_opts,
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("metadata-only copy must reject conflicting checksum aliases");
|
||||
@@ -25532,9 +25532,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()),
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
|
||||
@@ -11043,12 +11043,7 @@ mod dangling_read_tests {
|
||||
object,
|
||||
&mut source,
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("a CopyObject source without a version id must resolve like a latest GET");
|
||||
|
||||
@@ -205,12 +205,6 @@ fn heal_format_fence_lost_error() -> Error {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Whether this exact storage instance requires an incarnation-bound write
|
||||
/// for the object currently owned by a bucket-heal scope.
|
||||
pub fn object_requires_incarnation(&self, bucket: &str, object: &str) -> bool {
|
||||
bucket_heal_scope_for_object(bucket, object).is_some_and(|scope| std::ptr::eq(Arc::as_ptr(&scope.store), self))
|
||||
}
|
||||
|
||||
pub async fn purge_delete_marker_with_proof(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
|
||||
@@ -752,60 +752,18 @@ impl ECStore {
|
||||
Ok(ec)
|
||||
}
|
||||
|
||||
async fn init_pool_metadata_for_startup(self: &Arc<Self>) -> Result<(PoolMeta, PoolMetaReplicaState)> {
|
||||
#[instrument(level = "debug", skip(self, rx))]
|
||||
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
runtime_sources::ensure_boot_time().await;
|
||||
|
||||
let should_persist_pool_meta = self
|
||||
.pools
|
||||
.first()
|
||||
.is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
|
||||
// Lock order is save gate -> distributed metadata fence, matching
|
||||
// Lock order is save gate -> distributed metadata writer, matching
|
||||
// runtime mutations. RPC locks are available before ECStore publication
|
||||
// and do not depend on the later fleet proof.
|
||||
let mut startup_write_state = self.pool_meta_save_gate.write().await;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let startup_attempt = uuid::Uuid::new_v4();
|
||||
// A committed snapshot needs only a shared namespace fence. A writer
|
||||
// transition releases this reader and re-reads every observation below
|
||||
// under the existing exclusive fence before applying side effects.
|
||||
if !startup_write_state.bootstrap_identity_proven() && startup_write_state.attested_pool_indices().is_empty() {
|
||||
let pool = self
|
||||
.pools
|
||||
.first()
|
||||
.ok_or_else(|| Error::other("startup has no storage pools"))?;
|
||||
let lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, "pool.bin").await?;
|
||||
let reader = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let load = load_pool_meta_for_startup(self.pools.clone(), &mut startup_write_state);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load);
|
||||
match load.await {
|
||||
Ok((meta, replicas)) => {
|
||||
let topology_update = meta.validate(self.pools.clone())?;
|
||||
let requires_writer = should_persist_pool_meta
|
||||
&& (topology_update
|
||||
|| (replicas.needs_repair && replicas.repair_write_safe)
|
||||
|| startup_write_state.identity_requires_repair());
|
||||
if !topology_update && !requires_writer {
|
||||
startup_write_state.ensure_startup_identity_initialized()?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
crate::core::pools::startup_cas_test_observe(serde_json::json!({
|
||||
"kind": "startup-classifier", "attempt": startup_attempt,
|
||||
"elected_writer": should_persist_pool_meta,
|
||||
"needs_repair": replicas.needs_repair,
|
||||
"repair_write_safe": replicas.repair_write_safe,
|
||||
"topology_update": topology_update,
|
||||
}));
|
||||
self.install_startup_pool_metadata(&meta, Some(&reader)).await?;
|
||||
return Ok((meta, replicas));
|
||||
}
|
||||
}
|
||||
Err(err)
|
||||
if should_persist_pool_meta
|
||||
&& err
|
||||
.pool_metadata_failure()
|
||||
.is_some_and(|failure| failure.kind == crate::error::PoolMetadataFailure::ReadUnavailable) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
drop(reader);
|
||||
}
|
||||
let startup_pool_meta_guard = if should_persist_pool_meta {
|
||||
let pool = self
|
||||
.pools
|
||||
@@ -866,6 +824,7 @@ impl ECStore {
|
||||
"repair_write_safe": pool_meta_replica_state.repair_write_safe,
|
||||
"topology_update": update,
|
||||
}));
|
||||
let endpoints = runtime_sources::endpoint_pools_or_default();
|
||||
|
||||
let mut installed_pool_meta = if update {
|
||||
PoolMeta::new(&self.pools, &meta)?
|
||||
@@ -922,34 +881,19 @@ impl ECStore {
|
||||
installed_pool_meta = persist.await?;
|
||||
}
|
||||
|
||||
self.install_startup_pool_metadata(&installed_pool_meta, startup_pool_meta_guard.as_deref())
|
||||
.await?;
|
||||
if startup_pool_meta_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
|
||||
{
|
||||
return Err(Error::other("startup pool metadata writer fence was lost before publication"));
|
||||
}
|
||||
{
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
*pool_meta = installed_pool_meta.clone();
|
||||
}
|
||||
drop(startup_pool_meta_guard);
|
||||
drop(startup_write_state);
|
||||
|
||||
Ok((installed_pool_meta, pool_meta_replica_state))
|
||||
}
|
||||
|
||||
async fn install_startup_pool_metadata(
|
||||
&self,
|
||||
metadata: &PoolMeta,
|
||||
fence: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<()> {
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
if fence.is_some_and(|guard| guard.is_lock_lost() || guard.is_released()) {
|
||||
return Err(Error::other("startup pool metadata fence was lost before publication"));
|
||||
}
|
||||
*pool_meta = metadata.clone();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, rx))]
|
||||
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
runtime_sources::ensure_boot_time().await;
|
||||
|
||||
let (installed_pool_meta, pool_meta_replica_state) = self.init_pool_metadata_for_startup().await?;
|
||||
let endpoints = runtime_sources::endpoint_pools_or_default();
|
||||
|
||||
resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?;
|
||||
let rebalance_resume_required = {
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
@@ -3787,89 +3731,6 @@ mod tests {
|
||||
(instance_ctx, store, shutdown)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn committed_startup_pool_metadata_uses_shared_fence_without_disk_writes() {
|
||||
use crate::disk::{FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking;
|
||||
|
||||
for reopen in [false, true] {
|
||||
let directory = tempfile::tempdir().expect("create committed metadata deployment");
|
||||
let (_, seeded, seed_shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(directory.path(), "startup-read-fence", &[4])).await;
|
||||
let (store, shutdown) = if reopen {
|
||||
seed_shutdown.cancel();
|
||||
drop(seeded);
|
||||
let (_, reopened, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(directory.path(), "startup-read-fence", &[4])).await;
|
||||
(reopened, shutdown)
|
||||
} else {
|
||||
(seeded, seed_shutdown)
|
||||
};
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
let snapshot = || {
|
||||
let mut files = std::collections::BTreeMap::new();
|
||||
for (index, slot) in disks.iter().enumerate() {
|
||||
let disk = slot.as_ref().expect("committed drive handle");
|
||||
for object in [POOL_META_NAME, POOL_META_IDENTITY_NAME, FORMAT_CONFIG_FILE] {
|
||||
let root = disk.path().join(RUSTFS_META_BUCKET).join(object);
|
||||
assert!(root.exists(), "committed authority must be present: {root:?}");
|
||||
let mut pending = vec![root];
|
||||
while let Some(path) = pending.pop() {
|
||||
let metadata = std::fs::symlink_metadata(&path).expect("inspect authority file type");
|
||||
assert!(!metadata.file_type().is_symlink(), "authority fixture must not follow symlinks");
|
||||
if metadata.is_dir() {
|
||||
for entry in std::fs::read_dir(&path).expect("inspect authority object directory") {
|
||||
pending.push(entry.expect("authority directory entry").path());
|
||||
}
|
||||
} else {
|
||||
assert!(metadata.is_file(), "authority fixture must contain regular files");
|
||||
let contents = std::fs::read(&path).expect("read physical authority bytes");
|
||||
let relative = path
|
||||
.strip_prefix(disk.path())
|
||||
.expect("authority stays under fixture drive")
|
||||
.to_path_buf();
|
||||
files.insert((index, relative), contents);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
files
|
||||
};
|
||||
let before = snapshot();
|
||||
let lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("metadata namespace lock");
|
||||
let mut reader = lock
|
||||
.get_read_lock(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("hold a competing metadata reader");
|
||||
let (metadata, replicas) = tokio::time::timeout(Duration::from_secs(2), store.init_pool_metadata_for_startup())
|
||||
.await
|
||||
.expect("committed startup must not wait for an exclusive lock")
|
||||
.expect("committed startup metadata is readable under a shared fence");
|
||||
assert_eq!(metadata.pools[0].cmd_line, "startup-read-fence-pool-0");
|
||||
assert!(!replicas.needs_repair);
|
||||
assert!(!reader.is_lock_lost());
|
||||
let after = snapshot();
|
||||
assert_eq!(
|
||||
after, before,
|
||||
"shared startup observation cannot mutate metadata, identity, or format authority"
|
||||
);
|
||||
assert!(reader.release(), "release the snapshot's shared fence");
|
||||
let mut invalid = metadata.clone();
|
||||
invalid.pools[0].cmd_line = "must-not-be-published".to_string();
|
||||
store
|
||||
.install_startup_pool_metadata(&invalid, Some(&reader))
|
||||
.await
|
||||
.expect_err("a released fence cannot publish startup metadata");
|
||||
assert_eq!(store.pool_meta.read().await.pools[0].cmd_line, metadata.pools[0].cmd_line);
|
||||
drop(reader);
|
||||
shutdown.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn interrupted_format_bootstrap_recovers_two_pools_through_store_init() {
|
||||
|
||||
@@ -169,34 +169,36 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
{
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
let native_read_candidate = if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum))) {
|
||||
select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let partial_native_with_blanks = formats_present > 0
|
||||
if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum)))
|
||||
&& let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
{
|
||||
retain_format_quorum_members(
|
||||
instance_ctx,
|
||||
disks,
|
||||
&format,
|
||||
&members,
|
||||
Some(&unformatted_bootstrap_routes),
|
||||
set_drive_count,
|
||||
)
|
||||
.await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
});
|
||||
}
|
||||
let resumable_partial_migration = formats_present > 0
|
||||
&& errs.iter().any(|error| matches!(error, Some(DiskError::UnformattedDisk)))
|
||||
&& format_quorum.as_ref().is_some_and(Result::is_err);
|
||||
let resumable_partial_migration = partial_native_with_blanks
|
||||
&& format_quorum.as_ref().is_some_and(Result::is_err)
|
||||
&& formats.iter().zip(&errs).all(|(format, error)| {
|
||||
format.is_some() || matches!(error, Some(DiskError::UnformattedDisk | DiskError::DiskNotFound))
|
||||
});
|
||||
|
||||
let allow_migration = first_disk && (all_unformatted || resumable_partial_migration);
|
||||
if partial_native_with_blanks && !allow_migration && native_read_candidate.is_none() {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
if allow_migration || (partial_native_with_blanks && native_read_candidate.is_some()) {
|
||||
if first_disk && (all_unformatted || resumable_partial_migration) {
|
||||
info!(all_unformatted, resumable_partial_migration, "checking for a legacy storage format");
|
||||
let mode = match native_read_candidate.as_ref() {
|
||||
Some((reference, _)) => LegacyFormatMode::NativeRead {
|
||||
reference,
|
||||
native_errors: &errs,
|
||||
allow_migration,
|
||||
},
|
||||
None => LegacyFormatMode::Migration,
|
||||
};
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count, mode).await {
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => {
|
||||
info!("Migrated format from MinIO config");
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, None, set_drive_count).await?;
|
||||
@@ -215,10 +217,10 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
);
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::ReadCompatible) => {}
|
||||
Ok(LegacyFormatOutcome::None) => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
if first_disk && fresh_bootstrap_proven {
|
||||
if fresh_bootstrap_proven {
|
||||
let fm = init_format_erasure(instance_ctx, disks, set_count, set_drive_count, deployment_id).await?;
|
||||
return Ok(LoadedFormat {
|
||||
format: fm,
|
||||
@@ -227,26 +229,6 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve legacy migration before admitting native reads. Blank replacement
|
||||
// slots contribute no votes and receive no formatting/bootstrap authority.
|
||||
if let Some((format, members)) = native_read_candidate {
|
||||
retain_format_quorum_members(
|
||||
instance_ctx,
|
||||
disks,
|
||||
&format,
|
||||
&members,
|
||||
Some(&unformatted_bootstrap_routes),
|
||||
set_drive_count,
|
||||
)
|
||||
.await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
});
|
||||
}
|
||||
info!(
|
||||
"first_disk: {}, should_init_erasure_disks: {}",
|
||||
first_disk,
|
||||
@@ -289,11 +271,9 @@ async fn verified_legacy_adoption_source(
|
||||
set_count: usize,
|
||||
set_drive_count: usize,
|
||||
) -> Result<PoolMetaBootstrapAuthority> {
|
||||
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count, LegacyFormatMode::Migration).await {
|
||||
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { .. }) => Ok(PoolMetaBootstrapAuthority::LegacyAdoption),
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::ReadCompatible | LegacyFormatOutcome::Incompatible) => {
|
||||
Ok(PoolMetaBootstrapAuthority::None)
|
||||
}
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::Incompatible) => Ok(PoolMetaBootstrapAuthority::None),
|
||||
Err(err) => {
|
||||
debug!(
|
||||
error = %err,
|
||||
@@ -463,49 +443,25 @@ enum LegacyFormatOutcome {
|
||||
/// version mismatch, or a parse failure). The caller must decide how to
|
||||
/// proceed; creating a fresh format would leave the legacy objects unreadable.
|
||||
Incompatible,
|
||||
/// Observed legacy formats agree with the native read quorum, without granting writer authority.
|
||||
ReadCompatible,
|
||||
/// No MinIO `format.json` was present on any disk (a normal fresh install).
|
||||
None,
|
||||
}
|
||||
|
||||
/// Native reads validate observed legacy identities without electing a migration writer.
|
||||
enum LegacyFormatMode<'a> {
|
||||
Migration,
|
||||
NativeRead {
|
||||
reference: &'a FormatV3,
|
||||
native_errors: &'a [Option<DiskError>],
|
||||
allow_migration: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// Tries to migrate an on-disk MinIO `format.json` into RustFS format files.
|
||||
///
|
||||
/// Returns [`LegacyFormatOutcome`] describing whether a legacy format was present
|
||||
/// and, if so, whether it was compatible. `Err` is returned for read or persist
|
||||
/// failures that prevent a conclusive migration decision. Native read admission
|
||||
/// checks every observed legacy format against its existing layout and tolerates
|
||||
/// unavailable nonvoters. Migration requires complete observations and a strict majority.
|
||||
/// failures that prevent a conclusive migration decision.
|
||||
async fn try_migrate_format(
|
||||
disks: &[Option<DiskStore>],
|
||||
rustfs_formats: &[Option<FormatV3>],
|
||||
set_count: usize,
|
||||
set_drive_count: usize,
|
||||
mode: LegacyFormatMode<'_>,
|
||||
) -> Result<LegacyFormatOutcome> {
|
||||
let (native_reference, native_errors, allow_migration) = match mode {
|
||||
LegacyFormatMode::Migration => (None, None, true),
|
||||
LegacyFormatMode::NativeRead {
|
||||
reference,
|
||||
native_errors,
|
||||
allow_migration,
|
||||
} => (Some(reference), Some(native_errors), allow_migration),
|
||||
};
|
||||
let legacy_format_max_bytes = legacy_format_max_bytes(disks.len())?;
|
||||
let mut legacy_seen = false;
|
||||
let mut invalid_legacy_format = false;
|
||||
let mut read_error: Option<Error> = None;
|
||||
let mut unavailable_error = None;
|
||||
let mut legacy_formats = vec![None; disks.len()];
|
||||
|
||||
let reads = stream::iter(
|
||||
@@ -517,13 +473,6 @@ async fn try_migrate_format(
|
||||
let Some(disk) = disk else {
|
||||
return (index, None);
|
||||
};
|
||||
// Native format loading already proved this peer unavailable.
|
||||
// Reuse that observation and keep it ineligible for migration.
|
||||
if let Some(error) = native_errors.and_then(|errors| errors.get(index)).and_then(Option::as_ref)
|
||||
&& is_network_like_disk_error(error)
|
||||
{
|
||||
return (index, Some(Err(error.clone())));
|
||||
}
|
||||
let reader = match disk.read_file(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE).await {
|
||||
Ok(reader) => reader,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return (index, None),
|
||||
@@ -563,9 +512,6 @@ async fn try_migrate_format(
|
||||
match read {
|
||||
Ok(Some(format)) => legacy_formats[index] = Some(format),
|
||||
Ok(None) => invalid_legacy_format = true,
|
||||
Err(err) if native_reference.is_some() && is_network_like_disk_error(&err) => {
|
||||
unavailable_error.get_or_insert(err);
|
||||
}
|
||||
Err(err) => {
|
||||
read_error.get_or_insert_with(|| err.into());
|
||||
}
|
||||
@@ -582,15 +528,6 @@ async fn try_migrate_format(
|
||||
return Ok(LegacyFormatOutcome::Incompatible);
|
||||
}
|
||||
|
||||
if let Some(reference) = native_reference {
|
||||
if !formats_match_reference_slots(&legacy_formats, reference, 0) {
|
||||
return Ok(LegacyFormatOutcome::Incompatible);
|
||||
}
|
||||
if !allow_migration || unavailable_error.is_some() || legacy_formats.iter().flatten().count() <= disks.len() / 2 {
|
||||
return Ok(LegacyFormatOutcome::ReadCompatible);
|
||||
}
|
||||
}
|
||||
|
||||
let format = match get_format_erasure_in_quorum(&legacy_formats, 0) {
|
||||
Ok(format) => format,
|
||||
Err(_) => return Ok(LegacyFormatOutcome::Incompatible),
|
||||
@@ -740,7 +677,7 @@ fn select_existing_format_read_quorum(
|
||||
|| check_format_erasure_value_for_topology(reference, formats.len(), set_drive_count).is_err()
|
||||
|| formats.iter().zip(errors).any(|(format, error)| match (format, error) {
|
||||
(Some(_), None) => false,
|
||||
(None, Some(DiskError::DiskNotFound | DiskError::UnformattedDisk)) => false,
|
||||
(None, Some(DiskError::DiskNotFound)) => false,
|
||||
(None, Some(error)) => !is_network_like_disk_error(error),
|
||||
_ => true,
|
||||
})
|
||||
@@ -1508,23 +1445,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn existing_format_read_quorum_excludes_blank_and_network_unavailable_slots() {
|
||||
let (formats, mut errors) = half_pool_formats();
|
||||
errors[2] = Some(DiskError::UnformattedDisk);
|
||||
errors[3] = Some(DiskError::Timeout);
|
||||
let (_, members) = select_existing_format_read_quorum(&formats, &errors, 4)
|
||||
.expect("blank and unreachable slots cannot block an existing native read quorum");
|
||||
assert!(!members[2] && !members[3]);
|
||||
assert_eq!(members.iter().filter(|member| **member).count(), 12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn existing_format_read_quorum_rejects_conflicts_and_invalid_slots() {
|
||||
fn existing_format_read_quorum_rejects_conflicts_invalid_slots_and_blank_disks() {
|
||||
for failure in [
|
||||
"conflict",
|
||||
"wrong_slot",
|
||||
"nil",
|
||||
"duplicate",
|
||||
"blank",
|
||||
"corrupt",
|
||||
"access_denied",
|
||||
"tie",
|
||||
@@ -1546,6 +1473,7 @@ mod tests {
|
||||
format.erasure.sets[5][3] = format.erasure.sets[0][0];
|
||||
}
|
||||
}
|
||||
"blank" => errors[2] = Some(DiskError::UnformattedDisk),
|
||||
"corrupt" => errors[2] = Some(DiskError::CorruptedFormat),
|
||||
"access_denied" => errors[2] = Some(DiskError::FileAccessDenied),
|
||||
"tie" => {
|
||||
@@ -1604,186 +1532,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_format_read_quorum_quarantines_blank_replacement_without_writes() {
|
||||
for first_disk in [false, true] {
|
||||
for surviving_slot in 0..2 {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
|
||||
let blank = 1 - surviving_slot;
|
||||
let mut format = FormatV3::new(1, 2);
|
||||
format.erasure.this = format.erasure.sets[0][surviving_slot];
|
||||
save_format_file(&disks[surviving_slot], &Some(format.clone()))
|
||||
.await
|
||||
.expect("persist surviving format");
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&original).await;
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(
|
||||
&Arc::new(InstanceContext::new()),
|
||||
first_disk,
|
||||
&mut disks,
|
||||
1,
|
||||
2,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("existing native format must load at read quorum with a blank replacement");
|
||||
assert_eq!(loaded.format.shared_identity(), format.shared_identity());
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[surviving_slot].is_some());
|
||||
assert!(disks[blank].is_none(), "blank replacement cannot vote or serve data");
|
||||
assert_eq!(format_bytes(&original).await, before, "read admission cannot write either format");
|
||||
assert!(matches!(
|
||||
load_format_erasure(original[blank].as_ref().expect("blank handle"), false).await,
|
||||
Err(DiskError::UnformattedDisk)
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_read_quorum_with_blank_and_unavailable_slots_validates_legacy_without_writes() {
|
||||
for first_disk in [false, true] {
|
||||
for scenario in [
|
||||
"native-remote",
|
||||
"legacy-missing",
|
||||
"foreign-remote",
|
||||
"corrupt-remote",
|
||||
"wrong-slot-missing",
|
||||
] {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(4, 4).await;
|
||||
let reference = FormatV3::new(1, 4);
|
||||
for slot in 0..2 {
|
||||
let mut format = reference.clone();
|
||||
format.erasure.this = format.erasure.sets[0][slot];
|
||||
save_format_file(&disks[slot], &Some(format.clone()))
|
||||
.await
|
||||
.expect("persist native read quorum");
|
||||
if scenario != "native-remote" {
|
||||
write_legacy_format(&disks[slot], &format).await;
|
||||
}
|
||||
}
|
||||
match scenario {
|
||||
"foreign-remote" => {
|
||||
let mut foreign = FormatV3::new(1, 4);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][2];
|
||||
write_legacy_format(&disks[2], &foreign).await;
|
||||
}
|
||||
"corrupt-remote" => write_legacy_bytes(&disks[2], bytes::Bytes::from_static(b"corrupt legacy")).await,
|
||||
"wrong-slot-missing" => {
|
||||
let mut wrong_slot = reference.clone();
|
||||
wrong_slot.erasure.this = reference.erasure.sets[0][0];
|
||||
write_legacy_format(&disks[2], &wrong_slot).await;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&original).await;
|
||||
if scenario.ends_with("remote") {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve unavailable peer address");
|
||||
let address = listener.local_addr().expect("peer address");
|
||||
drop(listener);
|
||||
let mut endpoint = Endpoint::try_from(format!("http://{address}/disk").as_str()).expect("remote endpoint");
|
||||
endpoint.is_local = false;
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(3);
|
||||
let remote = crate::cluster::rpc::RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
Arc::new(crate::cluster::rpc::TcpHttpInternodeDataTransport),
|
||||
)
|
||||
.await
|
||||
.expect("construct unavailable remote handle");
|
||||
let remote = Arc::new(crate::disk::Disk::Remote(Box::new(remote)));
|
||||
let error = remote
|
||||
.read_all(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
|
||||
.await
|
||||
.expect_err("remote peer must be unavailable");
|
||||
assert!(is_network_like_disk_error(&error), "expected unavailable peer, got {error:?}");
|
||||
disks[3] = Some(remote);
|
||||
} else {
|
||||
disks[3] = None;
|
||||
}
|
||||
let result = connect_load_init_formats_with_instance_ctx(
|
||||
&Arc::new(InstanceContext::new()),
|
||||
first_disk,
|
||||
&mut disks,
|
||||
1,
|
||||
4,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
if matches!(scenario, "native-remote" | "legacy-missing") {
|
||||
let loaded = result.unwrap_or_else(|error| panic!("{scenario} must admit native reads: {error:?}"));
|
||||
assert_eq!(loaded.format.shared_identity(), reference.shared_identity());
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[..2].iter().all(Option::is_some));
|
||||
assert!(disks[2..].iter().all(Option::is_none), "unavailable and blank slots cannot vote");
|
||||
} else {
|
||||
assert!(
|
||||
matches!(result, Err(Error::CorruptedFormat)),
|
||||
"{scenario} observed legacy evidence must reject reads"
|
||||
);
|
||||
}
|
||||
assert_eq!(format_bytes(&original).await, before, "{scenario} cannot write native formats");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_native_format_with_legacy_quorum_reads_without_migration_writes() {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
|
||||
let legacy = FormatV3::new(1, 2);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut native = legacy.clone();
|
||||
native.erasure.this = native.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(native))
|
||||
.await
|
||||
.expect("persist partial native format");
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&disks).await;
|
||||
let loaded =
|
||||
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), false, &mut disks, 1, 2, None)
|
||||
.await
|
||||
.expect("follower can read the existing native layout without migrating legacy formats");
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[0].is_some() && disks[1].is_none(), "legacy-only slots cannot vote for native reads");
|
||||
assert_eq!(format_bytes(&original).await, before, "follower legacy inspection cannot persist formats");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn follower_below_native_read_quorum_skips_legacy_reads_without_writes() {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(3, 3).await;
|
||||
let mut format = FormatV3::new(1, 3);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(format))
|
||||
.await
|
||||
.expect("persist one native format below the two-drive read quorum");
|
||||
for disk in &disks {
|
||||
write_legacy_bytes(disk, bytes::Bytes::from_static(b"invalid legacy format")).await;
|
||||
}
|
||||
let before = format_bytes(&disks).await;
|
||||
|
||||
let result = connect_load_init_formats(false, &mut disks, 1, 3, None).await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(Error::ErasureReadQuorum)),
|
||||
"a non-reader must reject quorum before probing corrupt legacy files"
|
||||
);
|
||||
assert_eq!(format_bytes(&disks).await, before);
|
||||
for disk in disks.iter().flatten() {
|
||||
assert_eq!(
|
||||
disk.read_all(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE)
|
||||
.await
|
||||
.expect("legacy bytes must remain untouched"),
|
||||
bytes::Bytes::from_static(b"invalid legacy format")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_format_load_succeeds_with_a_strict_majority() {
|
||||
let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await;
|
||||
@@ -2627,7 +2375,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_format_migration_resumes_partial_rustfs_writes() {
|
||||
for drive_count in [2, 3, 4] {
|
||||
for drive_count in [3, 4] {
|
||||
let (_temp_dir, mut disks) = local_disks(drive_count).await;
|
||||
let legacy = FormatV3::new(1, drive_count);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
@@ -2686,32 +2434,23 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_format_migration_rejects_a_foreign_partial_rustfs_format() {
|
||||
for first_disk in [false, true] {
|
||||
for drive_count in [2, 3] {
|
||||
let (_temp_dir, mut disks) = local_disks(drive_count).await;
|
||||
let legacy = FormatV3::new(1, drive_count);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut foreign = FormatV3::new(1, drive_count);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][0];
|
||||
let foreign_id = foreign.id;
|
||||
save_format_file(&disks[0], &Some(foreign))
|
||||
.await
|
||||
.expect("foreign partial RustFS format should be written");
|
||||
let (_temp_dir, mut disks) = local_disks(3).await;
|
||||
let legacy = FormatV3::new(1, 3);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut foreign = FormatV3::new(1, 3);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][0];
|
||||
let foreign_id = foreign.id;
|
||||
save_format_file(&disks[0], &Some(foreign))
|
||||
.await
|
||||
.expect("foreign partial RustFS format should be written");
|
||||
|
||||
let result = connect_load_init_formats(first_disk, &mut disks, 1, drive_count, None).await;
|
||||
if !first_disk && drive_count == 3 {
|
||||
assert!(matches!(result, Err(Error::ErasureReadQuorum)), "a follower below read quorum must wait");
|
||||
} else {
|
||||
assert!(
|
||||
matches!(result, Err(Error::CorruptedFormat)),
|
||||
"an eligible reader or migration writer must reject the foreign legacy layout"
|
||||
);
|
||||
}
|
||||
let (formats, _) = load_format_erasure_all(&disks, false).await;
|
||||
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
|
||||
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
|
||||
}
|
||||
}
|
||||
assert!(matches!(
|
||||
connect_load_init_formats(true, &mut disks, 1, 3, None).await,
|
||||
Err(Error::CorruptedFormat)
|
||||
));
|
||||
let (formats, _) = load_format_erasure_all(&disks, false).await;
|
||||
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
|
||||
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -83,7 +83,6 @@ rustfs-common = { workspace = true }
|
||||
rustfs-heal-contracts = { workspace = true }
|
||||
rustfs-madmin = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync", "io-util", "time", "macros", "fs", "rt-multi-thread"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
|
||||
tonic = { workspace = true }
|
||||
@@ -111,7 +110,7 @@ walkdir = { workspace = true }
|
||||
http = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
jiff = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -2871,7 +2871,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn w13_timestamp() -> String {
|
||||
jiff::Timestamp::now().strftime("%Y-%m-%dT%H:%M:%SZ").to_string()
|
||||
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
|
||||
}
|
||||
|
||||
fn w13_selection_contains(selection: &str, lane: &str) -> bool {
|
||||
|
||||
+126
-92
@@ -18,7 +18,7 @@ use crate::sys::{get_claims_from_token_with_secret, get_claims_from_token_with_s
|
||||
use crate::{
|
||||
cache::{Cache, CacheEntity, LockedCache},
|
||||
error::{Error as IamError, is_err_no_such_group, is_err_no_such_policy, is_err_no_such_user},
|
||||
store::{GroupEdit, GroupInfo, MappedPolicy, Store, UserType, object::IAM_CONFIG_PREFIX},
|
||||
store::{GroupInfo, MappedPolicy, Store, UserType, object::IAM_CONFIG_PREFIX},
|
||||
sys::{
|
||||
MAX_SVCSESSION_POLICY_SIZE, SESSION_POLICY_NAME, SESSION_POLICY_NAME_EXTRACTED, SITE_REPLICATOR_CLAIM,
|
||||
SITE_REPLICATOR_SERVICE_ACCOUNT, STATUS_DISABLED, STATUS_ENABLED, UpdateServiceAccountOpts,
|
||||
@@ -1122,9 +1122,7 @@ where
|
||||
|
||||
let direct_policies = mapped_policy.to_slice();
|
||||
let has_identity_mapping = !direct_policies.is_empty();
|
||||
// An uncached group mapping can contain a Deny even when a direct policy exists.
|
||||
let mut policies = direct_policies;
|
||||
policies.extend(self.policy_db_get_with_groups(Vec::new(), groups).await?);
|
||||
let policies = self.policy_db_get_with_groups(direct_policies, groups).await?;
|
||||
Ok((policies, has_identity_mapping))
|
||||
}
|
||||
|
||||
@@ -1580,7 +1578,7 @@ where
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
self.add_user_at_with_secret_change(access_key, args, updated_at, true)
|
||||
self.add_user_at_with_secret_change(access_key, args, updated_at)
|
||||
.await
|
||||
.map(|(updated_at, _, _guard)| updated_at)
|
||||
}
|
||||
@@ -1591,7 +1589,6 @@ where
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
allow_status_change: bool,
|
||||
) -> Result<(OffsetDateTime, bool, tokio::sync::OwnedMutexGuard<crate::cache::MutationLockState>)> {
|
||||
// A virtual OIDC parent is a canonical base64url-encoded SHA-256 digest.
|
||||
// It has no stored user record to collide with in the checks below.
|
||||
@@ -1649,14 +1646,9 @@ where
|
||||
});
|
||||
user_entry.update_at = Some(updated_at);
|
||||
|
||||
let secret_changed = if allow_status_change {
|
||||
self.api
|
||||
.save_user_identity(access_key, UserType::Reg, user_entry.clone(), None)
|
||||
.await?;
|
||||
secret_changed
|
||||
} else {
|
||||
self.api.save_regular_user_preserving_status(access_key, &user_entry).await?
|
||||
};
|
||||
self.api
|
||||
.save_user_identity(access_key, UserType::Reg, user_entry.clone(), None)
|
||||
.await?;
|
||||
|
||||
self.update_user_with_claims(access_key, user_entry)?;
|
||||
|
||||
@@ -2001,25 +1993,38 @@ where
|
||||
// whose time predates the entity's load time — a replicated edit
|
||||
// whose source time is older than this node's startup would
|
||||
// otherwise never reach the cache.
|
||||
drop(cache);
|
||||
let (edit, current) = self.api.begin_group_edit(group).await?;
|
||||
let gi = match current {
|
||||
Some(mut gi) => {
|
||||
let gi = match cache.groups.get(group) {
|
||||
Some(res) => {
|
||||
let mut gi = res.clone();
|
||||
let mut uniq_set: HashSet<String, std::collections::hash_map::RandomState> =
|
||||
HashSet::from_iter(gi.members.iter().cloned());
|
||||
uniq_set.extend(members);
|
||||
uniq_set.extend(members.iter().cloned());
|
||||
|
||||
gi.members = uniq_set.into_iter().collect();
|
||||
gi.update_at = Some(updated_at);
|
||||
gi
|
||||
}
|
||||
None => {
|
||||
let mut gi = GroupInfo::new(members);
|
||||
let mut gi = GroupInfo::new(members.clone());
|
||||
gi.update_at = Some(updated_at);
|
||||
gi
|
||||
}
|
||||
};
|
||||
self.save_group_and_publish(group, &*edit, gi).await?;
|
||||
drop(cache);
|
||||
|
||||
self.api.save_group_info(group, gi.clone()).await?;
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(group, &gi, now);
|
||||
|
||||
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
|
||||
members.iter().for_each(|member| {
|
||||
let mut m = user_group_memberships.get(member).cloned().unwrap_or_default();
|
||||
m.insert(group.to_string());
|
||||
cache.add_or_update_user_group_membership(member, &m, now);
|
||||
});
|
||||
});
|
||||
|
||||
Ok(updated_at)
|
||||
}
|
||||
@@ -2035,8 +2040,13 @@ where
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
|
||||
let (edit, current) = self.api.begin_group_edit(name).await?;
|
||||
let mut gi = current.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
|
||||
let cache = self.cache.snapshot();
|
||||
let groups = Arc::clone(&cache.groups);
|
||||
let mut gi = match groups.get(name) {
|
||||
Some(gi) => gi.clone(),
|
||||
None => return Err(Error::NoSuchGroup(name.to_string())),
|
||||
};
|
||||
drop(cache);
|
||||
|
||||
if enable {
|
||||
gi.status = STATUS_ENABLED.to_owned();
|
||||
@@ -2045,7 +2055,11 @@ where
|
||||
}
|
||||
gi.update_at = Some(updated_at);
|
||||
|
||||
self.save_group_and_publish(name, &*edit, gi).await?;
|
||||
self.api.save_group_info(name, gi.clone()).await?;
|
||||
|
||||
// Cache publication time is the local clock, not the record stamp
|
||||
// (see `add_users_to_group_at`).
|
||||
self.cache.add_or_update_group(name, &gi, OffsetDateTime::now_utc());
|
||||
|
||||
Ok(updated_at)
|
||||
}
|
||||
@@ -2132,27 +2146,37 @@ where
|
||||
update_cache_only: bool,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let removed: HashSet<_> = members.into_iter().collect();
|
||||
let remove = |mut gi: GroupInfo| {
|
||||
gi.members.retain(|m| !removed.contains(m));
|
||||
gi.update_at = Some(updated_at);
|
||||
gi
|
||||
};
|
||||
if update_cache_only {
|
||||
return self.cache.with_write_lock(|cache| {
|
||||
let gi = cache
|
||||
.state()
|
||||
.groups
|
||||
.get(name)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
|
||||
self.publish_group_unlocked(cache, name, &remove(gi));
|
||||
Ok(updated_at)
|
||||
});
|
||||
let cache = self.cache.snapshot();
|
||||
let mut gi = cache
|
||||
.groups
|
||||
.get(name)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
|
||||
drop(cache);
|
||||
|
||||
let s: HashSet<&String> = HashSet::from_iter(gi.members.iter());
|
||||
let d: HashSet<&String> = HashSet::from_iter(members.iter());
|
||||
gi.members = s.difference(&d).map(|v| v.to_string()).collect::<Vec<String>>();
|
||||
gi.update_at = Some(updated_at);
|
||||
if !update_cache_only {
|
||||
self.api.save_group_info(name, gi.clone()).await?;
|
||||
}
|
||||
let (edit, current) = self.api.begin_group_edit(name).await?;
|
||||
let gi = current.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
|
||||
self.save_group_and_publish(name, &*edit, remove(gi)).await?;
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
// Sample after storage completes so a concurrent reload cannot
|
||||
// make this publication older than the cache it must update.
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(name, &gi, now);
|
||||
|
||||
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
|
||||
members.iter().for_each(|member| {
|
||||
if let Some(m) = user_group_memberships.get(member) {
|
||||
let mut m = m.clone();
|
||||
m.remove(name);
|
||||
cache.add_or_update_user_group_membership(member, &m, now);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Ok(updated_at)
|
||||
}
|
||||
@@ -2177,6 +2201,7 @@ where
|
||||
|
||||
let cache = self.cache.snapshot();
|
||||
let users_cache = Arc::clone(&cache.users);
|
||||
let groups_cache = Arc::clone(&cache.groups);
|
||||
drop(cache);
|
||||
|
||||
for member in members.iter() {
|
||||
@@ -2189,60 +2214,65 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
if !members.is_empty() {
|
||||
return self.remove_members_from_group_at(group, members, false, updated_at).await;
|
||||
}
|
||||
let (edit, current) = self.api.begin_group_edit(group).await?;
|
||||
let gi = current.ok_or_else(|| Error::NoSuchGroup(group.to_string()))?;
|
||||
if !gi.members.is_empty() {
|
||||
let gi = if members.is_empty() {
|
||||
// Reload from backend so we see latest members (e.g. after user was deleted elsewhere)
|
||||
let mut m = HashMap::new();
|
||||
self.api.load_group(group, &mut m).await?;
|
||||
m.get(group).cloned().ok_or_else(|| Error::NoSuchGroup(group.to_string()))?
|
||||
} else {
|
||||
groups_cache
|
||||
.get(group)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::NoSuchGroup(group.to_string()))?
|
||||
};
|
||||
|
||||
if members.is_empty() && !gi.members.is_empty() {
|
||||
return Err(Error::GroupNotEmpty);
|
||||
}
|
||||
|
||||
edit.delete_empty().await?;
|
||||
if members.is_empty() {
|
||||
if let Err(err) = self.api.delete_mapped_policy(group, UserType::Reg, true).await
|
||||
&& !is_err_no_such_policy(&err)
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
Self::replace_cached_group_members(cache, group, &[], now);
|
||||
cache.delete_group(group, now);
|
||||
cache.delete_group_policy(group, now);
|
||||
});
|
||||
if let Err(err) = self.api.delete_group_info(group).await
|
||||
&& !is_err_no_such_group(&err)
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(updated_at)
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
self.remove_group_from_memberships_map_unlocked(cache, group, now);
|
||||
cache.delete_group(group, now);
|
||||
cache.delete_group_policy(group, now);
|
||||
});
|
||||
|
||||
return Ok(updated_at);
|
||||
}
|
||||
|
||||
self.remove_members_from_group_at(group, members, false, updated_at).await
|
||||
}
|
||||
|
||||
async fn save_group_and_publish(&self, group: &str, edit: &dyn GroupEdit, gi: GroupInfo) -> Result<()> {
|
||||
edit.save(&gi).await?;
|
||||
self.cache
|
||||
.with_write_lock(|cache| self.publish_group_unlocked(cache, group, &gi));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn publish_group_unlocked(&self, cache: &mut LockedCache, group: &str, gi: &GroupInfo) {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(group, gi, now);
|
||||
Self::replace_cached_group_members(cache, group, &gi.members, now);
|
||||
}
|
||||
|
||||
fn replace_cached_group_members(cache: &mut LockedCache, group: &str, members: &[String], now: OffsetDateTime) {
|
||||
// This node may have missed peer edits, so reconcile against the complete record,
|
||||
// including memberships that were already stale before this edit began.
|
||||
let mut missing: HashSet<_> = members.iter().collect();
|
||||
let reverse = Arc::clone(&cache.state().user_group_memberships);
|
||||
for (user, groups) in reverse.iter() {
|
||||
if groups.contains(group) && !missing.remove(user) {
|
||||
let mut groups = groups.clone();
|
||||
groups.remove(group);
|
||||
if groups.is_empty() {
|
||||
cache.delete_user_group_membership(user, now);
|
||||
} else {
|
||||
cache.add_or_update_user_group_membership(user, &groups, now);
|
||||
}
|
||||
fn remove_group_from_memberships_map_unlocked(&self, cache: &mut LockedCache, group: &str, now: OffsetDateTime) {
|
||||
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
|
||||
for (k, v) in user_group_memberships.iter() {
|
||||
if v.contains(group) {
|
||||
let mut m = v.clone();
|
||||
m.remove(group);
|
||||
cache.add_or_update_user_group_membership(k, &m, now);
|
||||
}
|
||||
}
|
||||
for user in missing {
|
||||
let mut groups = reverse.get(user).cloned().unwrap_or_default();
|
||||
groups.insert(group.to_owned());
|
||||
cache.add_or_update_user_group_membership(user, &groups, now);
|
||||
}
|
||||
|
||||
fn update_group_memberships_map_unlocked(&self, cache: &mut LockedCache, group: &str, gi: &GroupInfo, now: OffsetDateTime) {
|
||||
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
|
||||
for member in gi.members.iter() {
|
||||
let mut m = user_group_memberships.get(member).cloned().unwrap_or_default();
|
||||
m.insert(group.to_string());
|
||||
cache.add_or_update_user_group_membership(member, &m, now);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2267,7 +2297,7 @@ where
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
Self::replace_cached_group_members(cache, group, &[], now);
|
||||
self.remove_group_from_memberships_map_unlocked(cache, group, now);
|
||||
cache.delete_group(group, now);
|
||||
cache.delete_group_policy(group, now);
|
||||
});
|
||||
@@ -2277,8 +2307,12 @@ where
|
||||
|
||||
let gi = m[group].clone();
|
||||
|
||||
self.cache
|
||||
.with_write_lock(|cache| self.publish_group_unlocked(cache, group, &gi));
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(group, &gi, now);
|
||||
self.remove_group_from_memberships_map_unlocked(cache, group, now);
|
||||
self.update_group_memberships_map_unlocked(cache, group, &gi, now);
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -104,12 +104,8 @@ pub(crate) mod crate_boundary {
|
||||
}
|
||||
|
||||
pub(crate) mod object_store {
|
||||
pub(crate) use super::storage_contracts::{
|
||||
HTTPPreconditions, ListOperations, NamespaceLocking, ObjectInfoOrErr, ObjectOperations,
|
||||
};
|
||||
pub(crate) use super::storage_contracts::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
|
||||
pub(crate) use rustfs_ecstore::api::config::com::read_config_preserve_empty_with_metadata_opts;
|
||||
pub(crate) use rustfs_ecstore::api::object::WriteCommitGuard;
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::get_lock_acquire_timeout;
|
||||
}
|
||||
|
||||
pub(crate) mod runtime {
|
||||
|
||||
@@ -21,14 +21,6 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// A backend-owned transaction for one named group. Persistence must fail if its lock is lost.
|
||||
#[async_trait::async_trait]
|
||||
pub trait GroupEdit: Send + Sync {
|
||||
async fn save(&self, group: &GroupInfo) -> Result<()>;
|
||||
/// Remove the policy mapping, then the group record, after the caller verified emptiness.
|
||||
async fn delete_empty(&self) -> Result<()>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait Store: Clone + Send + Sync + 'static {
|
||||
fn has_watcher(&self) -> bool;
|
||||
@@ -42,11 +34,6 @@ pub trait Store: Clone + Send + Sync + 'static {
|
||||
async fn delete_iam_config(&self, path: impl AsRef<str> + Send) -> Result<()>;
|
||||
|
||||
async fn save_user_identity(&self, name: &str, user_type: UserType, item: UserIdentity, ttl: Option<usize>) -> Result<()>;
|
||||
/// Create a regular identity or replace one without changing its persisted status.
|
||||
/// Return whether a previous secret was replaced. Backends without conditional writes fail closed.
|
||||
async fn save_regular_user_preserving_status(&self, _name: &str, _item: &UserIdentity) -> Result<bool> {
|
||||
Err(crate::error::Error::IAMActionNotAllowed)
|
||||
}
|
||||
async fn delete_user_identity(&self, name: &str, user_type: UserType) -> Result<()>;
|
||||
async fn load_user_identity(&self, name: &str, user_type: UserType) -> Result<UserIdentity>;
|
||||
|
||||
@@ -54,12 +41,6 @@ pub trait Store: Clone + Send + Sync + 'static {
|
||||
async fn load_users(&self, user_type: UserType, m: &mut HashMap<String, UserIdentity>) -> Result<()>;
|
||||
async fn load_secret_key(&self, name: &str, user_type: UserType) -> Result<String>;
|
||||
|
||||
/// Serialize edits to this group across all writers sharing the backend, then read its
|
||||
/// durable state. Only confirmed absence yields `None`. Keep the returned guard through
|
||||
/// save/delete and local cache publication; dropping it releases the transaction lock.
|
||||
async fn begin_group_edit(&self, _name: &str) -> Result<(Box<dyn GroupEdit>, Option<GroupInfo>)> {
|
||||
Err(crate::error::Error::other("backend does not support serialized group edits"))
|
||||
}
|
||||
async fn save_group_info(&self, name: &str, item: GroupInfo) -> Result<()>;
|
||||
async fn delete_group_info(&self, name: &str) -> Result<()>;
|
||||
async fn load_group(&self, name: &str, m: &mut HashMap<String, GroupInfo>) -> Result<()>;
|
||||
|
||||
@@ -12,9 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{GroupEdit, GroupInfo, MappedPolicy, Store, UserType};
|
||||
use super::{GroupInfo, MappedPolicy, Store, UserType};
|
||||
use crate::error::{Error, Result, is_err_config_not_found, is_err_no_such_group};
|
||||
use crate::storage_api::object_store::WriteCommitGuard;
|
||||
use crate::{
|
||||
IAM_CONFIG_ROOT_PREFIX, IamStorageError, IamStore, classify_iam_system_path_failure_reason, delete_iam_config,
|
||||
read_iam_config_no_lock, read_iam_config_with_metadata, save_iam_config, save_iam_config_with_opts,
|
||||
@@ -42,8 +41,8 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use crate::storage_api::object_store::{
|
||||
HTTPPreconditions, ListOperations, NamespaceLocking, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
|
||||
get_lock_acquire_timeout, read_config_preserve_empty_with_metadata_opts,
|
||||
HTTPPreconditions, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
|
||||
read_config_preserve_empty_with_metadata_opts,
|
||||
};
|
||||
|
||||
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{IAM_CONFIG_ROOT_PREFIX}/iam"));
|
||||
@@ -86,29 +85,6 @@ fn get_group_info_path(group: &str) -> String {
|
||||
path_join_buf(&[&IAM_CONFIG_GROUPS_PREFIX, group, IAM_GROUP_MEMBERS_FILE])
|
||||
}
|
||||
|
||||
fn unchanged_status_precondition(
|
||||
previous: &UserIdentity,
|
||||
item: &UserIdentity,
|
||||
etag: Option<String>,
|
||||
) -> Result<(HTTPPreconditions, bool)> {
|
||||
if previous.credentials.status != item.credentials.status
|
||||
|| previous.credentials.is_temp()
|
||||
|| previous.credentials.is_service_account()
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
let tag = etag
|
||||
.filter(|tag| !tag.is_empty() && tag != "*")
|
||||
.ok_or_else(|| Error::other("missing IAM identity revision"))?;
|
||||
Ok((
|
||||
HTTPPreconditions {
|
||||
if_match: Some(tag),
|
||||
..Default::default()
|
||||
},
|
||||
previous.credentials.secret_key != item.credentials.secret_key,
|
||||
))
|
||||
}
|
||||
|
||||
fn get_policy_doc_path(name: &str) -> String {
|
||||
path_join_buf(&[&IAM_CONFIG_POLICIES_PREFIX, name, IAM_POLICY_FILE])
|
||||
}
|
||||
@@ -232,53 +208,6 @@ pub struct ObjectStore {
|
||||
object_api: Arc<IamStore>,
|
||||
}
|
||||
|
||||
struct ObjectGroupEdit {
|
||||
api: Arc<IamStore>,
|
||||
path: String,
|
||||
policy_path: String,
|
||||
guard: WriteCommitGuard,
|
||||
}
|
||||
|
||||
impl ObjectGroupEdit {
|
||||
fn fenced(&self, mut opts: IamObjectOptions) -> Result<IamObjectOptions> {
|
||||
if self.guard.is_lock_lost() {
|
||||
return Err(Error::other("group transaction lock lost"));
|
||||
}
|
||||
if let Some(signal) = self.guard.lock_lost_signal() {
|
||||
opts.add_namespace_lock_lost_signal(signal);
|
||||
}
|
||||
Ok(opts)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl GroupEdit for ObjectGroupEdit {
|
||||
async fn save(&self, group: &GroupInfo) -> Result<()> {
|
||||
let data = ObjectStore::prepare_data_for_storage(&serde_json::to_vec(group)?)?;
|
||||
let opts = self.fenced(IamObjectOptions {
|
||||
max_parity: true,
|
||||
..Default::default()
|
||||
})?;
|
||||
save_iam_config_with_opts(self.api.clone(), &self.path, data, &opts).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_empty(&self) -> Result<()> {
|
||||
for path in [&self.policy_path, &self.path] {
|
||||
let opts = self.fenced(IamObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
..Default::default()
|
||||
})?;
|
||||
match self.api.delete_object(ObjectStore::BUCKET_NAME, path, opts).await {
|
||||
Ok(_) | Err(IamStorageError::FileNotFound | IamStorageError::ObjectNotFound(_, _)) => (),
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectStore {
|
||||
const BUCKET_NAME: &'static str = ".rustfs.sys";
|
||||
|
||||
@@ -968,34 +897,6 @@ impl Store for ObjectStore {
|
||||
e
|
||||
})
|
||||
}
|
||||
async fn save_regular_user_preserving_status(&self, name: &str, item: &UserIdentity) -> Result<bool> {
|
||||
let path = get_user_identity_path(name, UserType::Reg);
|
||||
let existing =
|
||||
read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), &path, &LoadMode::Locked.read_opts()).await;
|
||||
let (conditions, rotated) = match existing.map_err(Error::from) {
|
||||
Ok((data, metadata)) => {
|
||||
let previous: UserIdentity = Self::decode_iam_config_strict(&data)?;
|
||||
unchanged_status_precondition(&previous, item, metadata.etag)?
|
||||
}
|
||||
Err(err) if is_err_config_not_found(&err) => (
|
||||
HTTPPreconditions {
|
||||
if_none_match: Some("*".into()),
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let opts = IamObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(conditions),
|
||||
..Default::default()
|
||||
};
|
||||
let data = Self::prepare_data_for_storage(&serde_json::to_vec(item)?)?;
|
||||
save_iam_config_with_opts(self.object_api.clone(), &path, data, &opts).await?;
|
||||
Ok(rotated)
|
||||
}
|
||||
|
||||
async fn delete_user_identity(&self, name: &str, user_type: UserType) -> Result<()> {
|
||||
self.delete_iam_config(get_user_identity_path(name, user_type))
|
||||
.await
|
||||
@@ -1052,28 +953,6 @@ impl Store for ObjectStore {
|
||||
Ok(u.credentials.secret_key)
|
||||
}
|
||||
|
||||
async fn begin_group_edit(&self, name: &str) -> Result<(Box<dyn GroupEdit>, Option<GroupInfo>)> {
|
||||
let path = get_group_info_path(name);
|
||||
// Take the transaction lock before any object locks. A distinct key lets ordinary
|
||||
// config reads, writes and deletes keep taking their own namespace locks.
|
||||
let lock = self.object_api.new_ns_lock(Self::BUCKET_NAME, &format!("{path}.txn")).await?;
|
||||
let guard = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
let group = match self.load_iam_config_strict(&path).await {
|
||||
Ok(gi) => Some(gi),
|
||||
Err(Error::ConfigNotFound) => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let edit = ObjectGroupEdit {
|
||||
api: self.object_api.clone(),
|
||||
path,
|
||||
policy_path: get_mapped_policy_path(name, UserType::Reg, true),
|
||||
guard,
|
||||
};
|
||||
Ok((Box::new(edit), group))
|
||||
}
|
||||
|
||||
async fn save_group_info(&self, name: &str, item: GroupInfo) -> Result<()> {
|
||||
self.save_iam_config(item, get_group_info_path(name)).await
|
||||
}
|
||||
@@ -1624,78 +1503,6 @@ mod tests {
|
||||
assert!(matches!(err, Error::NoSuchUser(name) if name == "missing-sts"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restricted_user_upsert_requires_unchanged_status_and_exact_revision() {
|
||||
let old = super::UserIdentity::from(Credentials {
|
||||
status: "off".into(),
|
||||
secret_key: "old-fixture".into(),
|
||||
..Default::default()
|
||||
});
|
||||
let mut new = old.clone();
|
||||
new.credentials.secret_key = "new-fixture".into();
|
||||
let (predicate, rotated) =
|
||||
super::unchanged_status_precondition(&old, &new, Some("rev1".into())).expect("status remains disabled");
|
||||
assert!(rotated);
|
||||
assert_eq!(predicate.if_match.as_deref(), Some("rev1"));
|
||||
assert!(super::unchanged_status_precondition(&old, &new, None).is_err());
|
||||
new.credentials.status = "on".into();
|
||||
assert!(matches!(
|
||||
super::unchanged_status_precondition(&old, &new, Some("rev1".into())),
|
||||
Err(crate::error::Error::IAMActionNotAllowed)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn restricted_user_upsert_checks_durable_status_before_replacing_credentials() {
|
||||
use crate::store::{Store, UserType};
|
||||
let _credentials = test_cred();
|
||||
let temp = tempfile::tempdir().expect("IAM fixture directory");
|
||||
let env = Box::pin(
|
||||
rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(temp.path())
|
||||
.init_bucket_metadata(false)
|
||||
.build(),
|
||||
)
|
||||
.await;
|
||||
let store = ObjectStore::new(env.ecstore);
|
||||
let name = "status-fixture";
|
||||
let mut entry = super::UserIdentity::from(Credentials {
|
||||
access_key: name.into(),
|
||||
status: "off".into(),
|
||||
secret_key: "first-fixture".into(),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(
|
||||
!store
|
||||
.save_regular_user_preserving_status(name, &entry)
|
||||
.await
|
||||
.expect("create absent account")
|
||||
);
|
||||
entry.credentials.status = "on".into();
|
||||
entry.credentials.secret_key = "second-fixture".into();
|
||||
assert!(matches!(
|
||||
store.save_regular_user_preserving_status(name, &entry).await,
|
||||
Err(Error::IAMActionNotAllowed)
|
||||
));
|
||||
let persisted = store
|
||||
.load_user_identity(name, UserType::Reg)
|
||||
.await
|
||||
.expect("reload denied write");
|
||||
assert_eq!(persisted.credentials.status, "off");
|
||||
assert_eq!(persisted.credentials.secret_key, "first-fixture");
|
||||
entry.credentials.status = "off".into();
|
||||
assert!(
|
||||
store
|
||||
.save_regular_user_preserving_status(name, &entry)
|
||||
.await
|
||||
.expect("rotate while disabled")
|
||||
);
|
||||
let persisted = store.load_user_identity(name, UserType::Reg).await.expect("reload rotation");
|
||||
assert_eq!(persisted.credentials.status, "off");
|
||||
assert_eq!(persisted.credentials.secret_key, "second-fixture");
|
||||
}
|
||||
|
||||
fn test_cred() -> Credentials {
|
||||
if let Some(cred) = crate::root_credentials::credentials() {
|
||||
return cred;
|
||||
|
||||
+65
-425
@@ -76,8 +76,6 @@ pub const STATUS_ENABLED: &str = "enabled";
|
||||
pub const STATUS_DISABLED: &str = "disabled";
|
||||
|
||||
pub const POLICYNAME: &str = "policy";
|
||||
/// Signed subset of the OIDC policy selection granted independently of IAM mappings.
|
||||
pub const OIDC_EXPLICIT_POLICIES_CLAIM: &str = "x-rustfs-internal-oidc-explicit-policies";
|
||||
pub const PLUGIN_ATTEMPT_CREDENTIAL_CLAIM: &str = "rustfs:plugin-attempt-v1";
|
||||
pub const SESSION_POLICY_NAME: &str = "sessionPolicy";
|
||||
pub const SESSION_POLICY_NAME_EXTRACTED: &str = "sessionPolicy-extracted";
|
||||
@@ -1297,22 +1295,6 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
self.upsert_regular_user(access_key, args, updated_at, true).await
|
||||
}
|
||||
|
||||
/// Create a regular user, or update its credentials without changing the stored account status.
|
||||
pub async fn create_user_preserving_status(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result<OffsetDateTime> {
|
||||
self.upsert_regular_user(access_key, args, OffsetDateTime::now_utc(), false)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn upsert_regular_user(
|
||||
&self,
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
allow_status_change: bool,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
|
||||
let iam = self.clone();
|
||||
@@ -1320,10 +1302,7 @@ impl<T: Store> IamSys<T> {
|
||||
let args = args.clone();
|
||||
#[cfg(test)]
|
||||
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
|
||||
let operation = async move {
|
||||
iam.create_user_at_inner(&access_key, &args, updated_at, allow_status_change)
|
||||
.await
|
||||
};
|
||||
let operation = async move { iam.create_user_at_inner(&access_key, &args, updated_at).await };
|
||||
#[cfg(test)]
|
||||
let task = runtime.spawn(async move {
|
||||
if let Some(probe) = notification_probe {
|
||||
@@ -1341,7 +1320,6 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
allow_status_change: bool,
|
||||
) -> Result<OffsetDateTime> {
|
||||
if !is_access_key_valid(access_key) {
|
||||
return Err(IamError::InvalidAccessKeyLength);
|
||||
@@ -1357,7 +1335,7 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
let (updated_at, secret_changed, parent_guard) = self
|
||||
.store
|
||||
.add_user_at_with_secret_change(access_key, args, updated_at, allow_status_change)
|
||||
.add_user_at_with_secret_change(access_key, args, updated_at)
|
||||
.await?;
|
||||
if secret_changed {
|
||||
self.revoke_sts_sessions_for_parent(access_key).await?;
|
||||
@@ -1692,10 +1670,16 @@ impl<T: Store> IamSys<T> {
|
||||
parent_user: &str,
|
||||
groups: &Option<Vec<String>>,
|
||||
) -> Result<Option<HashMap<String, Value>>> {
|
||||
let Some((resolved, _)) = self.verified_federated_policy(claims, parent_user, groups).await? else {
|
||||
let Some(policy_names) = self.verified_federated_policy_names(claims, parent_user, groups).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let (resolved, _) = self.store.merge_policies(&policy_names.join(",")).await;
|
||||
let resolved_names = MappedPolicy::new(&resolved).to_slice();
|
||||
if policy_names.iter().any(|policy_name| !resolved_names.contains(policy_name)) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
|
||||
let boundary = match claims.get(SESSION_POLICY_NAME) {
|
||||
Some(Value::String(encoded)) => {
|
||||
decode_session_policy(encoded)?;
|
||||
@@ -1712,9 +1696,6 @@ impl<T: Store> IamSys<T> {
|
||||
if claims.get(OIDC_VIRTUAL_PARENT_CLAIM).and_then(Value::as_str) == Some(parent_user) {
|
||||
verified_claims.insert(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), Value::String(parent_user.to_string()));
|
||||
}
|
||||
if let Some(explicit) = claims.get(OIDC_EXPLICIT_POLICIES_CLAIM) {
|
||||
verified_claims.insert(OIDC_EXPLICIT_POLICIES_CLAIM.to_owned(), explicit.clone());
|
||||
}
|
||||
if let Some(boundary) = boundary {
|
||||
verified_claims.insert(FEDERATED_POLICY_BOUNDARY_CLAIM.to_string(), Value::String(boundary));
|
||||
}
|
||||
@@ -1896,24 +1877,20 @@ impl<T: Store> IamSys<T> {
|
||||
Ok(policy_names)
|
||||
}
|
||||
|
||||
async fn resolve_oidc_policy(&self, names: &[String]) -> Result<(String, Policy)> {
|
||||
if names.is_empty() || names.iter().any(|name| !is_safe_claim_policy_name(name)) {
|
||||
fn signed_oidc_policy_names(claims: &HashMap<String, Value>, parent_user: &str) -> Result<Vec<String>> {
|
||||
let policy_names = Self::safe_claim_policy_names(claims, parent_user);
|
||||
if policy_names.is_empty() {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
let (resolved, policy) = self.store.merge_policies(&names.join(",")).await;
|
||||
let resolved_names = MappedPolicy::new(&resolved).to_slice();
|
||||
if names.iter().any(|name| !resolved_names.contains(name)) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
Ok((resolved, policy))
|
||||
Ok(policy_names)
|
||||
}
|
||||
|
||||
async fn verified_federated_policy(
|
||||
async fn verified_federated_policy_names(
|
||||
&self,
|
||||
claims: &HashMap<String, Value>,
|
||||
parent_user: &str,
|
||||
groups: &Option<Vec<String>>,
|
||||
) -> Result<Option<(String, Policy)>> {
|
||||
) -> Result<Option<Vec<String>>> {
|
||||
if !is_rustfs_oidc_claims(claims) {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -1921,72 +1898,12 @@ impl<T: Store> IamSys<T> {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
|
||||
let explicit = claims.get(OIDC_EXPLICIT_POLICIES_CLAIM);
|
||||
if explicit.is_none() {
|
||||
// Earlier grouped credentials cannot distinguish an explicit Deny from a detached grant.
|
||||
if claims.contains_key("groups") || groups.as_ref().is_some_and(|groups| !groups.is_empty()) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
if !is_verified_federated_session(claims, parent_user) && !has_verified_federated_policy(claims) {
|
||||
let names = self.oidc_policy_names(claims, parent_user, groups).await?;
|
||||
return self.resolve_oidc_policy(&names).await.map(Some);
|
||||
}
|
||||
}
|
||||
let issued: Vec<String> = claims
|
||||
.get(POLICYNAME)
|
||||
.and_then(Value::as_str)
|
||||
.ok_or(IamError::InvalidArgument)?
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.map(str::to_owned)
|
||||
.collect();
|
||||
if issued.iter().any(|name| !is_safe_claim_policy_name(name)) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
let Some(explicit) = explicit else {
|
||||
return self.resolve_oidc_policy(&issued).await.map(Some);
|
||||
let policy_names = if is_verified_federated_session(claims, parent_user) {
|
||||
Self::signed_oidc_policy_names(claims, parent_user)?
|
||||
} else {
|
||||
self.oidc_policy_names(claims, parent_user, groups).await?
|
||||
};
|
||||
if !is_verified_federated_session(claims, parent_user) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
let explicit: HashSet<&str> = explicit
|
||||
.as_array()
|
||||
.ok_or(IamError::InvalidArgument)?
|
||||
.iter()
|
||||
.map(|v| {
|
||||
v.as_str()
|
||||
.filter(|name| issued.iter().any(|n| n == name))
|
||||
.ok_or(IamError::InvalidArgument)
|
||||
})
|
||||
.collect::<Result<_>>()?;
|
||||
// Use the issuer's signed group list, including on replicated and derived credentials.
|
||||
let signed_groups: Option<Vec<String>> = claims
|
||||
.get("groups")
|
||||
.map(serde::Deserialize::deserialize)
|
||||
.transpose()
|
||||
.map_err(|_| IamError::InvalidArgument)?;
|
||||
if signed_groups.is_none() && groups.as_ref().is_some_and(|groups| !groups.is_empty()) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
let current = self.store.sts_policy_db_get(parent_user, &signed_groups).await?;
|
||||
let active: Vec<_> = issued
|
||||
.iter()
|
||||
.filter(|name| explicit.contains(name.as_str()) || current.contains(name))
|
||||
.cloned()
|
||||
.collect();
|
||||
let (resolved, mut policy) = self.resolve_oidc_policy(&active).await?;
|
||||
// Later attachments can restrict the credential, but cannot widen its issuance ceiling.
|
||||
let new: Vec<_> = current.into_iter().filter(|name| !issued.contains(name)).collect();
|
||||
if !new.is_empty() {
|
||||
let (_, restrictions) = self.resolve_oidc_policy(&new).await?;
|
||||
policy.statements.extend(
|
||||
restrictions
|
||||
.statements
|
||||
.into_iter()
|
||||
.filter(|s| s.effect == rustfs_policy::policy::Effect::Deny),
|
||||
);
|
||||
}
|
||||
Ok(Some((resolved, policy)))
|
||||
Ok(Some(policy_names))
|
||||
}
|
||||
|
||||
/// Compatibility wrapper for service-account authorization entry points.
|
||||
@@ -2016,9 +1933,6 @@ impl<T: Store> IamSys<T> {
|
||||
let state = Self::policy_plugin_state().await;
|
||||
if marked {
|
||||
if args.is_owner
|
||||
// Plugin-attempt sessions are confined to their allowlisted work;
|
||||
// they cannot use a default permission to manage the parent identity.
|
||||
|| self_service_deny_check(args)
|
||||
|| args.claims.get(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM) != Some(&Value::Bool(true))
|
||||
|| !matches!(state, PolicyPluginState::Disabled)
|
||||
{
|
||||
@@ -2167,10 +2081,10 @@ impl<T: Store> IamSys<T> {
|
||||
session_policy,
|
||||
} => {
|
||||
let session_ok = evaluate_prepared_session_policy(session_policy, args).await;
|
||||
if session_ok == Some(false) && is_rustfs_oidc_claims(args.claims) {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
if let Some(ok) = session_ok {
|
||||
return Ok(ok && (*is_owner || combined_policy.is_allowed(args).await));
|
||||
}
|
||||
session_ok != Some(false) && (*is_owner || derived_policy_allowed(combined_policy, args).await?)
|
||||
*is_owner || combined_policy.is_allowed(args).await
|
||||
}
|
||||
PreparedIamMode::ServiceAccount {
|
||||
is_owner,
|
||||
@@ -2191,18 +2105,12 @@ impl<T: Store> IamSys<T> {
|
||||
true
|
||||
};
|
||||
|
||||
if !boundary_allowed {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
let parent_allowed =
|
||||
*bypass_parent_policy || *is_owner || derived_policy_allowed(combined_policy, &parent_args).await?;
|
||||
(*bypass_parent_policy || *is_owner || combined_policy.is_allowed(&parent_args).await) && boundary_allowed;
|
||||
match mode {
|
||||
PreparedServicePolicyMode::Inherited => parent_allowed,
|
||||
PreparedServicePolicyMode::SessionBound => {
|
||||
let session_ok = evaluate_prepared_session_policy(session_policy, args).await;
|
||||
if session_ok == Some(false) && is_rustfs_oidc_claims(args.claims) {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
if let Some(ok) = session_ok {
|
||||
return Ok(ok && parent_allowed);
|
||||
}
|
||||
@@ -2222,24 +2130,9 @@ impl<T: Store> IamSys<T> {
|
||||
};
|
||||
|
||||
if policies.is_empty() {
|
||||
// The policy lookup also returns an empty set for an inactive user.
|
||||
// Only a live identity gets the default for its own credential changes.
|
||||
let mode = if self_service_deny_check(args)
|
||||
&& self
|
||||
.store
|
||||
.get_user(args.account)
|
||||
.await
|
||||
.is_some_and(|u| u.credentials.is_valid())
|
||||
{
|
||||
PreparedIamMode::Regular {
|
||||
combined_policy: Policy::default(),
|
||||
}
|
||||
} else {
|
||||
PreparedIamMode::Deny
|
||||
};
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2266,19 +2159,24 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_sts_auth(&self, args: &Args<'_>, parent_user: &str) -> PreparedIamAuth {
|
||||
let federated_policy = match self.verified_federated_policy(args.claims, parent_user, args.groups).await {
|
||||
Ok(policy) => policy,
|
||||
let federated_policy_names = match self
|
||||
.verified_federated_policy_names(args.claims, parent_user, args.groups)
|
||||
.await
|
||||
{
|
||||
Ok(policy_names) => policy_names,
|
||||
Err(_) => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
};
|
||||
let federated_session = federated_policy_names.is_some();
|
||||
let verified_federated_session = is_verified_federated_session(args.claims, parent_user);
|
||||
let is_owner = !is_rustfs_oidc_claims(args.claims) && crate::is_root_access_key(parent_user);
|
||||
let role_arn = args.get_role_arn();
|
||||
|
||||
let (effective_groups, groups_source, mut policy_names) = if federated_policy.is_some() {
|
||||
let (effective_groups, groups_source, policies) = if federated_session {
|
||||
(args.groups.clone(), "oidc_claims", Vec::new())
|
||||
} else if is_owner {
|
||||
(None, "owner", Vec::new())
|
||||
@@ -2310,27 +2208,21 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
},
|
||||
};
|
||||
let p = match self.policy_db_get(parent_user, &effective_groups).await {
|
||||
Ok(policies) => policies,
|
||||
Err(_) if self_service_deny_check(args) => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
Err(_) => Vec::new(),
|
||||
};
|
||||
let p = self.policy_db_get(parent_user, &effective_groups).await.unwrap_or_default();
|
||||
(effective_groups, groups_source, p)
|
||||
};
|
||||
|
||||
let mut policy_names = if let Some(policy_names) = federated_policy_names {
|
||||
policy_names
|
||||
} else {
|
||||
policies
|
||||
};
|
||||
if !is_owner && policy_names.is_empty() && !is_rustfs_oidc_claims(args.claims) {
|
||||
policy_names = Self::safe_claim_policy_names(args.claims, parent_user);
|
||||
}
|
||||
|
||||
let mut combined_policy = Policy::default();
|
||||
if let Some((_, policy)) = federated_policy {
|
||||
combined_policy = policy;
|
||||
} else if !is_owner {
|
||||
if !is_owner {
|
||||
if policy_names.is_empty() {
|
||||
if args.deny_only {
|
||||
combined_policy = Policy::default();
|
||||
@@ -2349,7 +2241,7 @@ impl<T: Store> IamSys<T> {
|
||||
requested_policies = %requested_policies,
|
||||
"prepare_sts_auth: no STS policy names resolved"
|
||||
);
|
||||
if args.deny_only && !self_service_deny_check(args) {
|
||||
if args.deny_only && !verified_federated_session {
|
||||
combined_policy = Policy::default();
|
||||
} else {
|
||||
return PreparedIamAuth {
|
||||
@@ -2363,7 +2255,7 @@ impl<T: Store> IamSys<T> {
|
||||
.iter()
|
||||
.any(|policy_name| !resolved_policy_names.iter().any(|resolved| resolved == policy_name));
|
||||
if has_unresolved_policy_names {
|
||||
if self_service_deny_check(args) {
|
||||
if verified_federated_session {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
@@ -2440,14 +2332,14 @@ impl<T: Store> IamSys<T> {
|
||||
if is_rustfs_oidc_claims(args.claims) && !has_verified_federated_policy {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
let federated_boundary = if has_verified_federated_policy {
|
||||
if !is_valid_verified_federated_policy(args.claims, parent_user) {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
match federated_policy_boundary(args.claims) {
|
||||
@@ -2455,7 +2347,7 @@ impl<T: Store> IamSys<T> {
|
||||
Err(_) => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -2474,8 +2366,10 @@ impl<T: Store> IamSys<T> {
|
||||
let is_owner = !is_rustfs_oidc_claims(args.claims) && crate::is_root_access_key(parent_user);
|
||||
let role_arn = args.get_role_arn();
|
||||
|
||||
let svc_policies = if is_owner || bypass_parent_policy || has_verified_federated_policy {
|
||||
let svc_policies = if is_owner || bypass_parent_policy {
|
||||
Vec::new()
|
||||
} else if has_verified_federated_policy {
|
||||
Self::safe_claim_policy_names(args.claims, parent_user)
|
||||
} else if role_arn.is_some() {
|
||||
let Ok(arn) = ARN::parse(role_arn.unwrap_or_default()) else {
|
||||
tracing::warn!(
|
||||
@@ -2499,7 +2393,7 @@ impl<T: Store> IamSys<T> {
|
||||
policies
|
||||
};
|
||||
|
||||
if !is_owner && !bypass_parent_policy && !has_verified_federated_policy && svc_policies.is_empty() {
|
||||
if !is_owner && !bypass_parent_policy && svc_policies.is_empty() {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
@@ -2508,17 +2402,12 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
let combined_policy = if is_owner || bypass_parent_policy {
|
||||
Policy::default()
|
||||
} else if has_verified_federated_policy {
|
||||
let Ok(Some((_, policy))) = self.verified_federated_policy(args.claims, parent_user, args.groups).await else {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
};
|
||||
};
|
||||
policy
|
||||
} else {
|
||||
let (a, c) = self.store.merge_policies(&svc_policies.join(",")).await;
|
||||
if a.is_empty() {
|
||||
let resolved = MappedPolicy::new(&a).to_slice();
|
||||
if a.is_empty()
|
||||
|| has_verified_federated_policy && svc_policies.iter().any(|policy_name| !resolved.contains(policy_name))
|
||||
{
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
@@ -2589,26 +2478,8 @@ async fn prepared_session_policy_needs_existing_object_tag_for_args(policy: &Pre
|
||||
}
|
||||
}
|
||||
|
||||
fn self_service_deny_check(args: &Args<'_>) -> bool {
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
args.deny_only
|
||||
&& matches!(
|
||||
args.action,
|
||||
Action::AdminAction(
|
||||
AdminAction::ChangeOwnPasswordAdminAction
|
||||
| AdminAction::CreateUserAdminAction
|
||||
| AdminAction::ManageOwnMFAAdminAction
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
fn prepare_session_policy(args: &Args<'_>, empty_is_none: bool) -> PreparedSessionPolicy {
|
||||
let Some(policy_str) = extract_session_policy_text(args.claims) else {
|
||||
if self_service_deny_check(args)
|
||||
&& (args.claims.contains_key(SESSION_POLICY_NAME) || args.claims.contains_key(SESSION_POLICY_NAME_EXTRACTED))
|
||||
{
|
||||
return PreparedSessionPolicy::DenyAll;
|
||||
}
|
||||
return PreparedSessionPolicy::None;
|
||||
};
|
||||
|
||||
@@ -2681,7 +2552,6 @@ pub fn remove_verified_federated_policy(claims: &mut HashMap<String, Value>) ->
|
||||
let removed = claims.remove(VERIFIED_FEDERATED_POLICY_CLAIM).is_some();
|
||||
claims.remove(FEDERATED_POLICY_BOUNDARY_CLAIM);
|
||||
claims.remove(OIDC_VIRTUAL_PARENT_CLAIM);
|
||||
claims.remove(OIDC_EXPLICIT_POLICIES_CLAIM);
|
||||
removed
|
||||
}
|
||||
|
||||
@@ -2690,10 +2560,6 @@ fn merge_derived_service_account_claims(
|
||||
source_claims: &HashMap<String, Value>,
|
||||
verified_federated_policy: Option<HashMap<String, Value>>,
|
||||
) -> Result<()> {
|
||||
if verified_federated_policy.is_some() {
|
||||
// Only the parent's signed groups may participate in federated policy resolution.
|
||||
target_claims.remove("groups");
|
||||
}
|
||||
for (key, value) in source_claims {
|
||||
if key != "exp" {
|
||||
target_claims.insert(key.clone(), value.clone());
|
||||
@@ -2760,24 +2626,6 @@ fn extract_session_policy_text(claims: &HashMap<String, Value>) -> Option<String
|
||||
String::from_utf8(bytes).ok()
|
||||
}
|
||||
|
||||
async fn derived_policy_allowed(policy: &Policy, args: &Args<'_>) -> Result<bool> {
|
||||
let allowed = policy.is_allowed(args).await;
|
||||
// A bucket policy may supplement a missing Allow, but not override a live OIDC Deny.
|
||||
if !allowed
|
||||
&& is_rustfs_oidc_claims(args.claims)
|
||||
&& (args.deny_only
|
||||
|| !policy
|
||||
.is_allowed(&Args {
|
||||
deny_only: true,
|
||||
..args.clone()
|
||||
})
|
||||
.await)
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
Ok(allowed)
|
||||
}
|
||||
|
||||
async fn evaluate_prepared_session_policy(policy: &PreparedSessionPolicy, args: &Args<'_>) -> Option<bool> {
|
||||
match policy {
|
||||
PreparedSessionPolicy::None => None,
|
||||
@@ -2864,7 +2712,7 @@ mod tests {
|
||||
use crate::cache::{Cache, CacheEntity};
|
||||
use crate::error::Error;
|
||||
use crate::manager::get_default_policies;
|
||||
use crate::store::{GroupEdit, GroupInfo, MappedPolicy, Store, UserType};
|
||||
use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
|
||||
use rustfs_credentials::{Credentials, init_global_action_credentials};
|
||||
use rustfs_policy::auth::{UserIdentity, get_new_credentials_with_metadata};
|
||||
use rustfs_policy::policy::Args;
|
||||
@@ -3600,30 +3448,10 @@ mod tests {
|
||||
derived_save_started: Arc<tokio::sync::Notify>,
|
||||
release_derived_save: Arc<tokio::sync::Notify>,
|
||||
block_group_save: Arc<std::sync::atomic::AtomicBool>,
|
||||
group_locks: Arc<Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>>,
|
||||
saved_groups: Arc<Mutex<HashMap<String, GroupInfo>>>,
|
||||
group_save_started: Arc<tokio::sync::Notify>,
|
||||
group_save_release: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
struct MockGroupEdit {
|
||||
store: StsTestMockStore,
|
||||
name: String,
|
||||
_guard: tokio::sync::OwnedMutexGuard<()>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl GroupEdit for MockGroupEdit {
|
||||
async fn save(&self, group: &GroupInfo) -> Result<()> {
|
||||
self.store.save_group_info(&self.name, group.clone()).await
|
||||
}
|
||||
|
||||
async fn delete_empty(&self) -> Result<()> {
|
||||
self.store.delete_mapped_policy(&self.name, UserType::Reg, true).await?;
|
||||
self.store.delete_group_info(&self.name).await
|
||||
}
|
||||
}
|
||||
|
||||
impl StsTestMockStore {
|
||||
fn new(empty_policies: bool) -> Self {
|
||||
Self {
|
||||
@@ -3644,12 +3472,6 @@ mod tests {
|
||||
derived_save_started: Arc::new(tokio::sync::Notify::new()),
|
||||
release_derived_save: Arc::new(tokio::sync::Notify::new()),
|
||||
block_group_save: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
group_locks: Default::default(),
|
||||
saved_groups: Arc::new(Mutex::new(if empty_policies {
|
||||
HashMap::new()
|
||||
} else {
|
||||
HashMap::from([("testgroup".to_owned(), GroupInfo::new(vec!["sts-fallback-test-parent".to_owned()]))])
|
||||
})),
|
||||
group_save_started: Arc::new(tokio::sync::Notify::new()),
|
||||
group_save_release: Arc::new(tokio::sync::Notify::new()),
|
||||
}
|
||||
@@ -3792,36 +3614,15 @@ mod tests {
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
async fn begin_group_edit(&self, name: &str) -> Result<(Box<dyn GroupEdit>, Option<GroupInfo>)> {
|
||||
let lock = self
|
||||
.group_locks
|
||||
.lock()
|
||||
.expect("group locks")
|
||||
.entry(name.to_owned())
|
||||
.or_default()
|
||||
.clone();
|
||||
let guard = lock.lock_owned().await;
|
||||
Ok((
|
||||
Box::new(MockGroupEdit {
|
||||
store: self.clone(),
|
||||
name: name.to_owned(),
|
||||
_guard: guard,
|
||||
}),
|
||||
self.saved_groups.lock().expect("saved groups").get(name).cloned(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn save_group_info(&self, name: &str, item: GroupInfo) -> Result<()> {
|
||||
async fn save_group_info(&self, _name: &str, _item: GroupInfo) -> Result<()> {
|
||||
if self.block_group_save.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
self.group_save_started.notify_one();
|
||||
self.group_save_release.notified().await;
|
||||
}
|
||||
self.saved_groups.lock().expect("saved groups").insert(name.to_owned(), item);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_group_info(&self, name: &str) -> Result<()> {
|
||||
self.saved_groups.lock().expect("saved groups").remove(name);
|
||||
async fn delete_group_info(&self, _name: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -3874,14 +3675,11 @@ mod tests {
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
async fn delete_mapped_policy(&self, name: &str, user_type: UserType, is_group: bool) -> Result<()> {
|
||||
async fn delete_mapped_policy(&self, name: &str, user_type: UserType, _is_group: bool) -> Result<()> {
|
||||
self.deleted_mapped_policies
|
||||
.lock()
|
||||
.expect("deleted_mapped_policies mutex poisoned")
|
||||
.push((name.to_string(), user_type));
|
||||
if is_group {
|
||||
return Ok(());
|
||||
}
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
@@ -3990,7 +3788,9 @@ mod tests {
|
||||
users.insert(PARENT_USER.to_string(), parent_identity);
|
||||
users.extend(saved_users);
|
||||
|
||||
let groups = self.saved_groups.lock().expect("saved groups").clone();
|
||||
let group = GroupInfo::new(vec![PARENT_USER.to_string()]);
|
||||
let mut groups = HashMap::new();
|
||||
groups.insert(GROUP_NAME.to_string(), group);
|
||||
|
||||
let group_policy = MappedPolicy::new("readwrite");
|
||||
let mut group_policies = HashMap::new();
|
||||
@@ -4023,82 +3823,6 @@ mod tests {
|
||||
IamSys::new(cache)
|
||||
}
|
||||
|
||||
async fn shared_group_caches() -> (StsTestMockStore, Arc<IamCache<StsTestMockStore>>, Arc<IamCache<StsTestMockStore>>) {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let (first, second) = temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], async {
|
||||
(
|
||||
IamCache::new(store.clone()).await.expect("first cache"),
|
||||
IamCache::new(store.clone()).await.expect("second cache"),
|
||||
)
|
||||
})
|
||||
.await;
|
||||
(store, first, second)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn group_edit_status_and_removal_survive_shared_backend_reload() {
|
||||
let (store, first, second) = shared_group_caches().await;
|
||||
let group = "testgroup";
|
||||
let member = "sts-fallback-test-parent";
|
||||
second
|
||||
.add_users_to_group("unrelated", vec![member.to_owned()])
|
||||
.await
|
||||
.expect("seed unrelated membership");
|
||||
store.block_group_save.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let mut removal = Box::pin(first.remove_users_from_group(group, vec![member.to_owned()]));
|
||||
assert!(futures::poll!(&mut removal).is_pending());
|
||||
store.block_group_save.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||
let mut status = Box::pin(second.set_group_status(group, false));
|
||||
assert!(futures::poll!(&mut status).is_pending(), "status must wait for the other writer");
|
||||
store.group_save_release.notify_one();
|
||||
removal.await.expect("commit removal");
|
||||
status.await.expect("commit status");
|
||||
|
||||
store.load_all(&first.cache).await.expect("reload durable groups");
|
||||
for node in [&first, &second] {
|
||||
let state = node.cache.snapshot();
|
||||
assert!(state.groups[group].members.is_empty());
|
||||
assert_eq!(state.groups[group].status, "disabled");
|
||||
assert_eq!(state.user_group_memberships[member], HashSet::from(["unrelated".to_owned()]));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn group_edit_delete_orders_emptiness_check_with_add() {
|
||||
let (store, first, second) = shared_group_caches().await;
|
||||
let group = "testgroup";
|
||||
let member = "sts-fallback-test-parent";
|
||||
second
|
||||
.remove_users_from_group(group, vec![member.to_owned()])
|
||||
.await
|
||||
.expect("empty group");
|
||||
store.block_group_save.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let mut add = Box::pin(first.add_users_to_group(group, vec![member.to_owned()]));
|
||||
assert!(futures::poll!(&mut add).is_pending());
|
||||
let mut delete = Box::pin(second.remove_users_from_group(group, vec![]));
|
||||
assert!(futures::poll!(&mut delete).is_pending(), "delete must wait before checking emptiness");
|
||||
store.group_save_release.notify_one();
|
||||
add.await.expect("restore membership");
|
||||
assert!(matches!(delete.await, Err(Error::GroupNotEmpty)));
|
||||
assert!(store.deleted_mapped_policies.lock().expect("policy deletions").is_empty());
|
||||
store.block_group_save.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||
second.set_group_status(group, false).await.expect("refresh from peer add");
|
||||
assert!(second.cache.snapshot().user_group_memberships[member].contains(group));
|
||||
first
|
||||
.remove_users_from_group(group, vec![member.to_owned()])
|
||||
.await
|
||||
.expect("empty group");
|
||||
second
|
||||
.remove_users_from_group(group, vec![])
|
||||
.await
|
||||
.expect("delete empty group");
|
||||
assert!(matches!(first.set_group_status(group, true).await, Err(Error::NoSuchGroup(_))));
|
||||
assert!(!second.cache.snapshot().user_group_memberships.contains_key(member));
|
||||
assert!(!store.saved_groups.lock().expect("durable groups").contains_key(group));
|
||||
}
|
||||
|
||||
async fn assert_group_write_during_reload_is_published(remove: bool) {
|
||||
let iam_sys = Arc::new(temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], test_iam_sys()).await);
|
||||
let member = "sts-fallback-test-parent";
|
||||
@@ -6058,15 +5782,13 @@ mod tests {
|
||||
("sub".to_string(), Value::String("subject-123".to_string())),
|
||||
("parent".to_string(), Value::String(parent_user.to_string())),
|
||||
(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), Value::String(parent_user.to_string())),
|
||||
(POLICYNAME.to_string(), Value::String("readwrite,readonly".to_string())),
|
||||
(OIDC_EXPLICIT_POLICIES_CLAIM.to_string(), serde_json::json!(["readonly"])),
|
||||
("groups".to_string(), serde_json::json!(["testgroup"])),
|
||||
(POLICYNAME.to_string(), Value::String("readwrite".to_string())),
|
||||
]);
|
||||
let conditions = HashMap::new();
|
||||
let sts_args = Args {
|
||||
account: "OIDCGROUPCALLER01",
|
||||
groups: &groups,
|
||||
action: Action::S3Action(S3Action::PutObjectAction),
|
||||
action: Action::S3Action(S3Action::GetObjectAction),
|
||||
bucket: "bucket",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
@@ -6101,10 +5823,7 @@ mod tests {
|
||||
.await
|
||||
.expect("group-authorized OIDC caller should create a service account");
|
||||
assert_eq!(replication_claims.get(VERIFIED_FEDERATED_POLICY_CLAIM), Some(&Value::Bool(true)));
|
||||
assert_eq!(
|
||||
replication_claims.get(OIDC_EXPLICIT_POLICIES_CLAIM),
|
||||
claims.get(OIDC_EXPLICIT_POLICIES_CLAIM)
|
||||
);
|
||||
assert_eq!(replication_claims.get(POLICYNAME), Some(&Value::String("readwrite".to_string())));
|
||||
|
||||
let service_claims = iam_sys
|
||||
.get_claims_for_svc_acc(&service_account.access_key)
|
||||
@@ -6120,83 +5839,6 @@ mod tests {
|
||||
iam_sys.is_allowed(&service_args).await,
|
||||
"OIDC service account must resolve the policy names selected at STS issuance"
|
||||
);
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let deny_read = Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::bucket/*"}]}"#).expect("parse Deny");
|
||||
iam_sys
|
||||
.store
|
||||
.cache
|
||||
.add_or_update_policy_doc("deny-read", &PolicyDoc::new(deny_read), now);
|
||||
for (mapping, enabled, read, write) in [
|
||||
("", true, true, false),
|
||||
("readwrite", false, true, false),
|
||||
("readwrite,deny-read", true, false, true),
|
||||
("readwrite,missing-deny", true, false, false),
|
||||
] {
|
||||
if mapping.is_empty() {
|
||||
iam_sys.store.cache.delete_group_policy("testgroup", now);
|
||||
} else {
|
||||
iam_sys
|
||||
.store
|
||||
.cache
|
||||
.add_or_update_group_policy("testgroup", &MappedPolicy::new(mapping), now);
|
||||
}
|
||||
let mut group = GroupInfo::new(Vec::new());
|
||||
if !enabled {
|
||||
group.status = STATUS_DISABLED.to_owned();
|
||||
}
|
||||
iam_sys.store.cache.add_or_update_group("testgroup", &group, now);
|
||||
for (action, expected) in [(S3Action::GetObjectAction, read), (S3Action::PutObjectAction, write)] {
|
||||
let sts = Args {
|
||||
action: Action::S3Action(action),
|
||||
..sts_args.clone()
|
||||
};
|
||||
let service = Args {
|
||||
action: sts.action,
|
||||
..service_args.clone()
|
||||
};
|
||||
let prepared = iam_sys.prepare_sts_auth(&sts, parent_user).await;
|
||||
assert_eq!(
|
||||
iam_sys.eval_prepared(&prepared, &sts).await,
|
||||
expected,
|
||||
"STS: {mapping}, enabled={enabled}"
|
||||
);
|
||||
assert_eq!(
|
||||
iam_sys.is_allowed(&service).await,
|
||||
expected,
|
||||
"service account: {mapping}, enabled={enabled}"
|
||||
);
|
||||
if mapping == "readwrite,deny-read" && action == S3Action::GetObjectAction {
|
||||
let child = iam_sys.prepare_service_account_auth(&service, parent_user).await;
|
||||
for (prepared, args) in [(&prepared, &sts), (&child, &service)] {
|
||||
assert!(
|
||||
matches!(iam_sys.try_eval_prepared(prepared, args).await, Err(Error::IAMActionNotAllowed)),
|
||||
"a newly mapped Deny must stop bucket policy fallback"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
iam_sys.store.cache.delete_group_policy("testgroup", now);
|
||||
for (source, account) in [(&claims, sts_args.account), (&service_claims, service_args.account)] {
|
||||
let mut legacy = source.clone();
|
||||
legacy.remove(OIDC_EXPLICIT_POLICIES_CLAIM);
|
||||
let args = Args {
|
||||
claims: &legacy,
|
||||
account,
|
||||
deny_only: true,
|
||||
..sts_args.clone()
|
||||
};
|
||||
let prepared = if account == sts_args.account {
|
||||
iam_sys.prepare_sts_auth(&args, parent_user).await
|
||||
} else {
|
||||
iam_sys.prepare_service_account_auth(&args, parent_user).await
|
||||
};
|
||||
assert!(
|
||||
!iam_sys.eval_prepared(&prepared, &args).await,
|
||||
"ambiguous legacy grouped credentials must be reissued"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -6224,8 +5866,6 @@ mod tests {
|
||||
("parent".to_string(), Value::String(parent_user.to_string())),
|
||||
(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), Value::String(parent_user.to_string())),
|
||||
(POLICYNAME.to_string(), Value::String("readonly,deny-delete".to_string())),
|
||||
(OIDC_EXPLICIT_POLICIES_CLAIM.to_string(), serde_json::json!(["readonly", "deny-delete"])),
|
||||
("groups".to_string(), serde_json::json!(["testgroup"])),
|
||||
]);
|
||||
let groups = Some(vec!["testgroup".to_string()]);
|
||||
let conditions = HashMap::new();
|
||||
@@ -6430,7 +6070,7 @@ mod tests {
|
||||
|
||||
let prepared = iam_sys.prepare_sts_auth(&args, parent_user).await;
|
||||
|
||||
assert!(matches!(prepared.mode, PreparedIamMode::TerminalDeny));
|
||||
assert!(matches!(prepared.mode, PreparedIamMode::Deny));
|
||||
assert!(!iam_sys.eval_prepared(&prepared, &args).await);
|
||||
}
|
||||
|
||||
|
||||
@@ -33,18 +33,23 @@ default = []
|
||||
diagnostic-internode-http2-ping = []
|
||||
diagnostic-read-version-caller = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"rustfs-s3-ops/hotpath",
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-s3-ops/hotpath-alloc",
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-s3-ops/hotpath-cpu",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
metrics = { workspace = true }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
num_cpus = { workspace = true }
|
||||
|
||||
@@ -59,7 +59,6 @@ use std::sync::{
|
||||
LazyLock, Mutex,
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Global switch for detailed per-stage PUT metrics (path label, stage durations).
|
||||
/// When `false`, `record_put_object_path` and `record_put_object_stage_duration`
|
||||
@@ -450,125 +449,6 @@ pub const PUT_RENAME_QUORUM_FANOUT_STATE_PANIC: &str = "panic";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST: &str = "quorum_first";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL: &str = "quorum_tail";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR: &str = "error";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_ORDINARY: &str = "ordinary";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE: &str = "quota_fence";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_INCARNATION: &str = "incarnation";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_MULTIPART: &str = "multipart";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE: &str = "scanner_lease";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_POOL_META_CAS: &str = "pool_meta_cas";
|
||||
pub const ECSTORE_RENAME_RPC_KINDS: [&str; 6] = [
|
||||
ECSTORE_RENAME_RPC_KIND_ORDINARY,
|
||||
ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE,
|
||||
ECSTORE_RENAME_RPC_KIND_INCARNATION,
|
||||
ECSTORE_RENAME_RPC_KIND_MULTIPART,
|
||||
ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE,
|
||||
ECSTORE_RENAME_RPC_KIND_POOL_META_CAS,
|
||||
];
|
||||
pub const ECSTORE_RENAME_PEER_WIDTHS: [&str; 18] = [
|
||||
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "gt16",
|
||||
];
|
||||
pub const ECSTORE_RENAME_COVERAGE: [&str; 3] = ["none", "partial", "full"];
|
||||
pub const ECSTORE_RENAME_BATCH_EXECUTION: [&str; 10] = [
|
||||
"planned_groups",
|
||||
"planned_items",
|
||||
"stream_groups",
|
||||
"stream_items",
|
||||
"completed_items",
|
||||
"fallback_groups",
|
||||
"fallback_items",
|
||||
"not_dispatched_items",
|
||||
"may_have_published_items",
|
||||
"terminal_missing_items",
|
||||
];
|
||||
pub const ECSTORE_RENAME_ADMISSION: [&str; 10] = [
|
||||
"unary_attempts",
|
||||
"unary_items",
|
||||
"unary_wait_ns_total",
|
||||
"unary_wait_ns_max",
|
||||
"unary_timeouts",
|
||||
"batch_attempts",
|
||||
"batch_items",
|
||||
"batch_wait_ns_total",
|
||||
"batch_wait_ns_max",
|
||||
"batch_timeouts",
|
||||
];
|
||||
pub const ECSTORE_RENAME_QUARANTINE: [&str; 12] = [
|
||||
"unary_events",
|
||||
"unary_items",
|
||||
"unary_hold_ns_total",
|
||||
"unary_hold_ns_max",
|
||||
"unary_released_items",
|
||||
"unary_no_runtime_items",
|
||||
"batch_events",
|
||||
"batch_items",
|
||||
"batch_hold_ns_total",
|
||||
"batch_hold_ns_max",
|
||||
"batch_released_items",
|
||||
"batch_no_runtime_items",
|
||||
];
|
||||
pub const ECSTORE_RENAME_BATCH_TIMING: [&str; 4] = [
|
||||
"client_stream_to_first",
|
||||
"client_first_to_last",
|
||||
"server_dispatch_to_first",
|
||||
"server_first_to_last",
|
||||
];
|
||||
|
||||
static ECSTORE_RENAME_RPC_COUNTS: [AtomicU64; 6] = [const { AtomicU64::new(0) }; 6];
|
||||
static ECSTORE_RENAME_REMOTE_WIDTH_COUNTS: [AtomicU64; 18] = [const { AtomicU64::new(0) }; 18];
|
||||
static ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS: [AtomicU64; 18] = [const { AtomicU64::new(0) }; 18];
|
||||
static ECSTORE_RENAME_COVERAGE_COUNTS: [AtomicU64; 3] = [const { AtomicU64::new(0) }; 3];
|
||||
static ECSTORE_RENAME_BATCH_EXECUTION_COUNTS: [AtomicU64; 10] = [const { AtomicU64::new(0) }; 10];
|
||||
static ECSTORE_RENAME_ADMISSION_COUNTS: [AtomicU64; 10] = [const { AtomicU64::new(0) }; 10];
|
||||
static ECSTORE_RENAME_QUARANTINE_COUNTS: [AtomicU64; 12] = [const { AtomicU64::new(0) }; 12];
|
||||
static ECSTORE_RENAME_BATCH_TIMING_COUNTS: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4];
|
||||
static ECSTORE_RENAME_BATCH_TIMING_TOTAL_NS: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4];
|
||||
static ECSTORE_RENAME_BATCH_TIMING_MAX_NS: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4];
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct EcstoreRenameDiagnosticsSnapshot {
|
||||
pub rpc_counts: [u64; 6],
|
||||
pub remote_width_counts: [u64; 18],
|
||||
pub batchable_width_counts: [u64; 18],
|
||||
pub coverage_counts: [u64; 3],
|
||||
pub batch_execution_counts: [u64; 10],
|
||||
pub admission_counts: [u64; 10],
|
||||
pub quarantine_counts: [u64; 12],
|
||||
pub batch_timing_counts: [u64; 4],
|
||||
pub batch_timing_total_ns: [u64; 4],
|
||||
pub batch_timing_max_ns: [u64; 4],
|
||||
}
|
||||
|
||||
impl EcstoreRenameDiagnosticsSnapshot {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.rpc_counts
|
||||
.iter()
|
||||
.chain(self.remote_width_counts.iter())
|
||||
.chain(self.batchable_width_counts.iter())
|
||||
.chain(self.coverage_counts.iter())
|
||||
.chain(self.batch_execution_counts.iter())
|
||||
.chain(self.admission_counts.iter())
|
||||
.chain(self.quarantine_counts.iter())
|
||||
.chain(self.batch_timing_counts.iter())
|
||||
.chain(self.batch_timing_total_ns.iter())
|
||||
.chain(self.batch_timing_max_ns.iter())
|
||||
.all(|count| *count == 0)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot_ecstore_rename_diagnostics() -> EcstoreRenameDiagnosticsSnapshot {
|
||||
EcstoreRenameDiagnosticsSnapshot {
|
||||
rpc_counts: std::array::from_fn(|index| ECSTORE_RENAME_RPC_COUNTS[index].load(Ordering::Relaxed)),
|
||||
remote_width_counts: std::array::from_fn(|index| ECSTORE_RENAME_REMOTE_WIDTH_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batchable_width_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS[index].load(Ordering::Relaxed)),
|
||||
coverage_counts: std::array::from_fn(|index| ECSTORE_RENAME_COVERAGE_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batch_execution_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_EXECUTION_COUNTS[index].load(Ordering::Relaxed)),
|
||||
admission_counts: std::array::from_fn(|index| ECSTORE_RENAME_ADMISSION_COUNTS[index].load(Ordering::Relaxed)),
|
||||
quarantine_counts: std::array::from_fn(|index| ECSTORE_RENAME_QUARANTINE_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batch_timing_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_TIMING_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batch_timing_total_ns: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_TIMING_TOTAL_NS[index].load(Ordering::Relaxed)),
|
||||
batch_timing_max_ns: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_TIMING_MAX_NS[index].load(Ordering::Relaxed)),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn get_stage_metrics_enabled() -> bool {
|
||||
@@ -2783,169 +2663,6 @@ pub fn record_put_rename_quorum_wait_fanout(
|
||||
}
|
||||
}
|
||||
|
||||
/// Records a rename RPC after its canonical request digest has been attached.
|
||||
///
|
||||
/// This uses the PUT-stage diagnostics switch because it is temporary hot-path
|
||||
/// evidence, but the metric is ECStore-scoped: scanner and pool metadata
|
||||
/// renames are not S3 PutObject operations.
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_remote_rpc(kind: &'static str) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let Some(index) = ECSTORE_RENAME_RPC_KINDS.iter().position(|candidate| *candidate == kind) else {
|
||||
return;
|
||||
};
|
||||
ECSTORE_RENAME_RPC_COUNTS[index].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn rename_peer_width_index(width: usize) -> usize {
|
||||
width.min(17)
|
||||
}
|
||||
|
||||
/// Records the per-peer opportunity for a future body-bound rename batch.
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_candidate_peer(remote_disks: usize, batchable_disks: usize) {
|
||||
if !put_stage_metrics_enabled() || remote_disks == 0 {
|
||||
return;
|
||||
}
|
||||
ECSTORE_RENAME_REMOTE_WIDTH_COUNTS[rename_peer_width_index(remote_disks)].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS[rename_peer_width_index(batchable_disks)].fetch_add(1, Ordering::Relaxed);
|
||||
let coverage_index = if batchable_disks == 0 {
|
||||
0
|
||||
} else if batchable_disks == remote_disks {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
};
|
||||
ECSTORE_RENAME_COVERAGE_COUNTS[coverage_index].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn record_ecstore_rename_batch_execution(index: usize, count: usize) {
|
||||
if put_stage_metrics_enabled() {
|
||||
let count = u64::try_from(count).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_BATCH_EXECUTION_COUNTS[index].fetch_add(count, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_planned(items: usize) {
|
||||
record_ecstore_rename_batch_execution(0, 1);
|
||||
record_ecstore_rename_batch_execution(1, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_stream_started(items: usize) {
|
||||
record_ecstore_rename_batch_execution(2, 1);
|
||||
record_ecstore_rename_batch_execution(3, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_item_completed() {
|
||||
record_ecstore_rename_batch_execution(4, 1);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_fallback(items: usize) {
|
||||
record_ecstore_rename_batch_execution(5, 1);
|
||||
record_ecstore_rename_batch_execution(6, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_not_dispatched(items: usize) {
|
||||
record_ecstore_rename_batch_execution(7, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_may_have_published(items: usize) {
|
||||
record_ecstore_rename_batch_execution(8, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_terminal_missing(items: usize) {
|
||||
record_ecstore_rename_batch_execution(9, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_admission(batch: bool, items: usize, wait: Duration, timed_out: bool) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let base = usize::from(batch) * 5;
|
||||
let items = u64::try_from(items).unwrap_or(u64::MAX);
|
||||
let wait_ns = u64::try_from(wait.as_nanos()).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base + 1].fetch_add(items, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base + 2].fetch_add(wait_ns, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base + 3].fetch_max(wait_ns, Ordering::Relaxed);
|
||||
if timed_out {
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base + 4].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_quarantine(batch: bool, items: usize, hold: Duration) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let base = usize::from(batch) * 6;
|
||||
let items = u64::try_from(items).unwrap_or(u64::MAX);
|
||||
let hold_ns = u64::try_from(hold.as_nanos()).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 1].fetch_add(items, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 2].fetch_add(hold_ns, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 3].fetch_max(hold_ns, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_quarantine_released(batch: bool, items: usize) {
|
||||
if put_stage_metrics_enabled() {
|
||||
let base = usize::from(batch) * 6;
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 4].fetch_add(u64::try_from(items).unwrap_or(u64::MAX), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_quarantine_no_runtime(batch: bool, items: usize) {
|
||||
if put_stage_metrics_enabled() {
|
||||
let base = usize::from(batch) * 6;
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 5].fetch_add(u64::try_from(items).unwrap_or(u64::MAX), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn record_ecstore_rename_batch_timing(index: usize, duration: Duration) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let duration_ns = u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_BATCH_TIMING_COUNTS[index].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_BATCH_TIMING_TOTAL_NS[index].fetch_add(duration_ns, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_BATCH_TIMING_MAX_NS[index].fetch_max(duration_ns, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_client_stream_to_first(duration: Duration) {
|
||||
record_ecstore_rename_batch_timing(0, duration);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_client_first_to_last(duration: Duration) {
|
||||
record_ecstore_rename_batch_timing(1, duration);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_server_dispatch_to_first(duration: Duration) {
|
||||
record_ecstore_rename_batch_timing(2, duration);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_server_first_to_last(duration: Duration) {
|
||||
record_ecstore_rename_batch_timing(3, duration);
|
||||
}
|
||||
|
||||
/// Record generic internal operation stage duration (non-PUT paths).
|
||||
/// Use this for metacache walks, listing, lifecycle, and other background
|
||||
/// operations that are NOT part of the PUT object hot path.
|
||||
@@ -4868,85 +4585,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ecstore_rename_batch_candidate_metrics_are_gated_and_bounded() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
set_put_stage_metrics_enabled(false);
|
||||
let disabled_before = snapshot_ecstore_rename_diagnostics();
|
||||
record_ecstore_rename_remote_rpc(ECSTORE_RENAME_RPC_KIND_ORDINARY);
|
||||
record_ecstore_rename_batch_candidate_peer(4, 4);
|
||||
record_ecstore_rename_admission(false, 1, Duration::from_nanos(10), false);
|
||||
record_ecstore_rename_quarantine(true, 4, Duration::from_nanos(30));
|
||||
record_ecstore_rename_batch_client_stream_to_first(Duration::from_nanos(10));
|
||||
assert_eq!(snapshot_ecstore_rename_diagnostics(), disabled_before);
|
||||
|
||||
set_put_stage_metrics_enabled(true);
|
||||
let before = snapshot_ecstore_rename_diagnostics();
|
||||
for kind in ECSTORE_RENAME_RPC_KINDS {
|
||||
record_ecstore_rename_remote_rpc(kind);
|
||||
}
|
||||
record_ecstore_rename_batch_candidate_peer(4, 4);
|
||||
record_ecstore_rename_batch_candidate_peer(4, 3);
|
||||
record_ecstore_rename_batch_candidate_peer(4, 0);
|
||||
record_ecstore_rename_batch_candidate_peer(20, 20);
|
||||
record_ecstore_rename_batch_planned(4);
|
||||
record_ecstore_rename_batch_stream_started(4);
|
||||
record_ecstore_rename_batch_item_completed();
|
||||
record_ecstore_rename_batch_fallback(3);
|
||||
record_ecstore_rename_batch_not_dispatched(2);
|
||||
record_ecstore_rename_batch_may_have_published(4);
|
||||
record_ecstore_rename_batch_terminal_missing(1);
|
||||
record_ecstore_rename_admission(false, 1, Duration::from_nanos(10), false);
|
||||
record_ecstore_rename_admission(true, 4, Duration::from_nanos(20), true);
|
||||
record_ecstore_rename_quarantine(true, 4, Duration::from_nanos(30));
|
||||
record_ecstore_rename_quarantine_released(true, 3);
|
||||
record_ecstore_rename_quarantine_no_runtime(true, 1);
|
||||
record_ecstore_rename_batch_client_stream_to_first(Duration::from_nanos(10));
|
||||
record_ecstore_rename_batch_client_first_to_last(Duration::from_nanos(20));
|
||||
record_ecstore_rename_batch_server_dispatch_to_first(Duration::from_nanos(30));
|
||||
record_ecstore_rename_batch_server_first_to_last(Duration::from_nanos(40));
|
||||
set_put_stage_metrics_enabled(false);
|
||||
|
||||
let after = snapshot_ecstore_rename_diagnostics();
|
||||
for index in 0..ECSTORE_RENAME_RPC_KINDS.len() {
|
||||
assert_eq!(after.rpc_counts[index] - before.rpc_counts[index], 1);
|
||||
}
|
||||
assert_eq!(after.remote_width_counts[4] - before.remote_width_counts[4], 3);
|
||||
assert_eq!(after.remote_width_counts[17] - before.remote_width_counts[17], 1);
|
||||
assert_eq!(after.batchable_width_counts[0] - before.batchable_width_counts[0], 1);
|
||||
assert_eq!(after.batchable_width_counts[3] - before.batchable_width_counts[3], 1);
|
||||
assert_eq!(after.batchable_width_counts[4] - before.batchable_width_counts[4], 1);
|
||||
assert_eq!(after.batchable_width_counts[17] - before.batchable_width_counts[17], 1);
|
||||
assert_eq!(after.coverage_counts[0] - before.coverage_counts[0], 1);
|
||||
assert_eq!(after.coverage_counts[1] - before.coverage_counts[1], 1);
|
||||
assert_eq!(after.coverage_counts[2] - before.coverage_counts[2], 2);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 10, _>(|index| {
|
||||
after.batch_execution_counts[index] - before.batch_execution_counts[index]
|
||||
}),
|
||||
[1, 4, 1, 4, 1, 1, 3, 2, 4, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 10, _>(|index| after.admission_counts[index] - before.admission_counts[index]),
|
||||
[1, 1, 10, 10, 0, 1, 4, 20, 20, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 12, _>(|index| after.quarantine_counts[index] - before.quarantine_counts[index]),
|
||||
[0, 0, 0, 0, 0, 0, 1, 4, 30, 30, 3, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 4, _>(|index| after.batch_timing_counts[index] - before.batch_timing_counts[index]),
|
||||
[1, 1, 1, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 4, _>(|index| { after.batch_timing_total_ns[index] - before.batch_timing_total_ns[index] }),
|
||||
[10, 20, 30, 40]
|
||||
);
|
||||
for (actual, expected) in after.batch_timing_max_ns.into_iter().zip([10, 20, 30, 40]) {
|
||||
assert!(actual >= expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_object_diagnostic_buckets() {
|
||||
assert_eq!(put_object_size_bucket(0), "unknown");
|
||||
|
||||
@@ -58,6 +58,8 @@ zeroize = { workspace = true, features = ["derive"] }
|
||||
|
||||
# Configuration and storage
|
||||
url = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
|
||||
# Caching
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
|
||||
|
||||
@@ -65,7 +65,7 @@ use aws_sdk_kms::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_kms::types::{DataKeySpec, KeySpec as AwsKeySpec, KeyState as AwsKeyState, KeyUsageType, Tag};
|
||||
use aws_smithy_runtime_api::client::orchestrator::HttpResponse;
|
||||
use aws_smithy_types::Blob;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, empty_key_page, list_keys_page_size};
|
||||
@@ -234,12 +234,12 @@ fn map_key_usage(usage: Option<&KeyUsageType>) -> KeyUsage {
|
||||
}
|
||||
}
|
||||
|
||||
fn to_zoned(value: Option<&aws_smithy_types::DateTime>) -> Option<ZonedDateTime> {
|
||||
fn to_zoned(value: Option<&aws_smithy_types::DateTime>) -> Option<Zoned> {
|
||||
let value = value?;
|
||||
let nanos = i32::try_from(value.subsec_nanos()).ok()?;
|
||||
jiff::Timestamp::new(value.secs(), nanos)
|
||||
.ok()
|
||||
.map(|timestamp| timestamp.to_zoned_datetime(jiff::tz::TimeZone::UTC))
|
||||
.map(|timestamp| timestamp.to_zoned(jiff::tz::TimeZone::UTC))
|
||||
}
|
||||
|
||||
/// `None` for an empty context so the request omits the field entirely rather
|
||||
@@ -593,7 +593,7 @@ impl KmsBackend for AwsKmsBackend {
|
||||
encrypted_key: ciphertext.into_inner(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: request.encryption_context,
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
// AWS addresses backing keys internally and never exposes a version.
|
||||
master_key_version: None,
|
||||
// AWS binds the context into the blob itself; this field describes
|
||||
@@ -876,7 +876,7 @@ impl KmsBackend for AwsKmsBackend {
|
||||
/// report what AWS has already done: a key that is gone counts as removed,
|
||||
/// a key that left `PendingDeletion` counts as a state change, and a key
|
||||
/// still pending is never expired by RustFS regardless of the deadline.
|
||||
async fn remove_expired_key(&self, key_id: &str, _now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, key_id: &str, _now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
match self.describe(key_id).await {
|
||||
Ok(metadata) => Ok(match metadata.key_state {
|
||||
KeyState::PendingDeletion => ExpiredKeyRemoval::NotExpired,
|
||||
@@ -1492,7 +1492,7 @@ mod tests {
|
||||
let (http_client, backend) = scripted_backend(vec![ok_event(key_metadata_json("test-key", "PendingDeletion"))]);
|
||||
assert_eq!(
|
||||
backend
|
||||
.remove_expired_key("test-key", &ZonedDateTime::now())
|
||||
.remove_expired_key("test-key", &Zoned::now())
|
||||
.await
|
||||
.expect("observation should succeed"),
|
||||
ExpiredKeyRemoval::NotExpired
|
||||
@@ -1502,7 +1502,7 @@ mod tests {
|
||||
let (_http, backend) = scripted_backend(vec![error_event(400, "NotFoundException", "key does not exist")]);
|
||||
assert_eq!(
|
||||
backend
|
||||
.remove_expired_key("test-key", &ZonedDateTime::now())
|
||||
.remove_expired_key("test-key", &Zoned::now())
|
||||
.await
|
||||
.expect("a key AWS already removed is a completed removal"),
|
||||
ExpiredKeyRemoval::Removed
|
||||
@@ -1511,7 +1511,7 @@ mod tests {
|
||||
let (_http, backend) = scripted_backend(vec![ok_event(key_metadata_json("test-key", "Enabled"))]);
|
||||
assert_eq!(
|
||||
backend
|
||||
.remove_expired_key("test-key", &ZonedDateTime::now())
|
||||
.remove_expired_key("test-key", &Zoned::now())
|
||||
.await
|
||||
.expect("observation should succeed"),
|
||||
ExpiredKeyRemoval::StateChanged
|
||||
|
||||
@@ -35,7 +35,7 @@ use aes_gcm::{
|
||||
use argon2::{Algorithm, Argon2, Params, Version};
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use rand::RngExt;
|
||||
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -582,14 +582,14 @@ struct StoredMasterKey {
|
||||
description: Option<String>,
|
||||
metadata: HashMap<String, String>,
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
created_at: ZonedDateTime,
|
||||
created_at: Zoned,
|
||||
#[serde(with = "crate::time_serde::option_zoned")]
|
||||
rotated_at: Option<ZonedDateTime>,
|
||||
rotated_at: Option<Zoned>,
|
||||
created_by: Option<String>,
|
||||
/// Scheduled deletion deadline; absent on records written before deadline
|
||||
/// persistence landed, so it must stay optional for backward compatibility.
|
||||
#[serde(default, with = "crate::time_serde::option_zoned")]
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
deletion_date: Option<Zoned>,
|
||||
/// Encrypted key material (32 bytes encoded in base64 for AES-256)
|
||||
encrypted_key_material: String,
|
||||
/// Nonce used for encryption
|
||||
@@ -685,10 +685,10 @@ impl<'de> Deserialize<'de> for StoredMasterKey {
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] ZonedDateTime);
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] Zoned);
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct OptionalZonedValue(#[serde(with = "crate::time_serde::option_zoned")] Option<ZonedDateTime>);
|
||||
struct OptionalZonedValue(#[serde(with = "crate::time_serde::option_zoned")] Option<Zoned>);
|
||||
|
||||
struct StoredMasterKeyVisitor;
|
||||
|
||||
@@ -1556,7 +1556,7 @@ impl LocalKmsClient {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding,
|
||||
};
|
||||
@@ -1596,7 +1596,7 @@ impl LocalKmsClient {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
// Local rotation is rejected, so the key has a single material version.
|
||||
master_key_version: None,
|
||||
context_binding,
|
||||
@@ -1860,7 +1860,7 @@ impl LocalKmsClient {
|
||||
let mut master_key = self.load_master_key(key_id).await?;
|
||||
ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::ScheduleDeletion)?;
|
||||
master_key.status = KeyStatus::PendingDeletion;
|
||||
master_key.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(pending_window_days as u64 * 86400));
|
||||
master_key.deletion_date = Some(Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400));
|
||||
|
||||
// Preserve the existing key material (see enable_key): scheduling deletion must not
|
||||
// regenerate the master key, or cancelling the deletion later would recover a key that
|
||||
@@ -1985,7 +1985,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: request.key_usage,
|
||||
description: request.description,
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
@@ -2113,7 +2113,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
Ok((_stored, key_material)) => {
|
||||
let mut tombstone = master_key.clone();
|
||||
tombstone.status = KeyStatus::Deleted;
|
||||
tombstone.deletion_date = Some(ZonedDateTime::now());
|
||||
tombstone.deletion_date = Some(Zoned::now());
|
||||
self.client.save_master_key(&tombstone, &key_material).await?;
|
||||
}
|
||||
Err(error) => {
|
||||
@@ -2136,7 +2136,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
key_usage: master_key.usage,
|
||||
key_state: KeyState::PendingDeletion, // AWS KMS compatibility
|
||||
creation_date: master_key.created_at,
|
||||
deletion_date: Some(ZonedDateTime::now()),
|
||||
deletion_date: Some(Zoned::now()),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
origin: "AWS_KMS".to_string(),
|
||||
tags: master_key.metadata,
|
||||
@@ -2162,7 +2162,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
)));
|
||||
}
|
||||
|
||||
let deletion_date = ZonedDateTime::now() + Duration::from_secs(days as u64 * 86400);
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
master_key.status = KeyStatus::PendingDeletion;
|
||||
master_key.deletion_date = Some(deletion_date.clone());
|
||||
|
||||
@@ -2317,7 +2317,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.with_update_key_metadata(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
// The per-key write lock serializes this against a concurrent
|
||||
// cancellation, closing the check-then-remove race.
|
||||
let _write_guard = self.client.lock_key_for_write(key_id).await;
|
||||
@@ -3088,12 +3088,12 @@ mod tests {
|
||||
description: Option<String>,
|
||||
metadata: HashMap<String, String>,
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
created_at: ZonedDateTime,
|
||||
created_at: Zoned,
|
||||
#[serde(with = "crate::time_serde::option_zoned")]
|
||||
rotated_at: Option<ZonedDateTime>,
|
||||
rotated_at: Option<Zoned>,
|
||||
created_by: Option<String>,
|
||||
#[serde(default, with = "crate::time_serde::option_zoned")]
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
deletion_date: Option<Zoned>,
|
||||
encrypted_key_material: String,
|
||||
nonce: Vec<u8>,
|
||||
#[serde(default)]
|
||||
@@ -4284,7 +4284,7 @@ mod tests {
|
||||
let (_stored, key_material) = client.decode_stored_key(&key_id).await.expect("decode stored key");
|
||||
let mut tombstone = client.load_master_key(&key_id).await.expect("load key");
|
||||
tombstone.status = KeyStatus::Deleted;
|
||||
tombstone.deletion_date = Some(ZonedDateTime::now());
|
||||
tombstone.deletion_date = Some(Zoned::now());
|
||||
client
|
||||
.save_master_key(&tombstone, &key_material)
|
||||
.await
|
||||
@@ -4292,7 +4292,7 @@ mod tests {
|
||||
|
||||
// The sweep primitive completes the crashed removal...
|
||||
let outcome = backend
|
||||
.remove_expired_key(&key_id, &ZonedDateTime::now())
|
||||
.remove_expired_key(&key_id, &Zoned::now())
|
||||
.await
|
||||
.expect("tombstone completion");
|
||||
assert_eq!(outcome, crate::backends::ExpiredKeyRemoval::Removed);
|
||||
@@ -4300,7 +4300,7 @@ mod tests {
|
||||
|
||||
// ...and stays idempotent once the key is gone.
|
||||
let outcome = backend
|
||||
.remove_expired_key(&key_id, &ZonedDateTime::now())
|
||||
.remove_expired_key(&key_id, &Zoned::now())
|
||||
.await
|
||||
.expect("repeat removal");
|
||||
assert_eq!(outcome, crate::backends::ExpiredKeyRemoval::Removed);
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -608,7 +608,7 @@ pub trait KmsBackend: Send + Sync {
|
||||
///
|
||||
/// The default rejects the operation for backends without deletion
|
||||
/// support.
|
||||
async fn remove_expired_key(&self, _key_id: &str, _now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, _key_id: &str, _now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
Err(KmsError::unsupported_capability("backend without deletion support", "remove_expired_key"))
|
||||
}
|
||||
|
||||
@@ -934,7 +934,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn default_remove_expired_key_is_unsupported() {
|
||||
let error = MinimalBackend
|
||||
.remove_expired_key("any-key", &jiff::ZonedDateTime::now())
|
||||
.remove_expired_key("any-key", &jiff::Zoned::now())
|
||||
.await
|
||||
.expect_err("backends without deletion support must reject expired-key removal");
|
||||
assert!(
|
||||
|
||||
@@ -34,7 +34,7 @@ use aes_gcm::{
|
||||
aead::{Aead, KeyInit, Payload},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use rand::RngExt;
|
||||
use std::collections::HashMap;
|
||||
use tracing::debug;
|
||||
@@ -65,7 +65,7 @@ pub struct StaticKmsBackend {
|
||||
/// across a restart and differs between nodes of one cluster, because there
|
||||
/// is no creation event to anchor it to; callers must treat it as "when this
|
||||
/// node loaded the key", not as the key's birth date.
|
||||
created_at: ZonedDateTime,
|
||||
created_at: Zoned,
|
||||
}
|
||||
|
||||
impl StaticKmsBackend {
|
||||
@@ -89,7 +89,7 @@ impl StaticKmsBackend {
|
||||
Ok(Self {
|
||||
key_id: static_config.key_id.clone(),
|
||||
key: key.into(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -149,7 +149,7 @@ impl StaticKmsBackend {
|
||||
encrypted_key: encrypted,
|
||||
nonce: nonce_bytes.to_vec(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
// The static backend has a single fixed key with no rotation.
|
||||
master_key_version: None,
|
||||
// The context is already bound as AAD by the Static cipher path
|
||||
@@ -200,7 +200,7 @@ impl StaticKmsBackend {
|
||||
encrypted_key: encrypted,
|
||||
nonce: nonce_bytes.to_vec(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
// The static backend has a single fixed key with no rotation.
|
||||
master_key_version: None,
|
||||
// The context is already bound as AAD by the Static cipher path
|
||||
|
||||
@@ -35,7 +35,7 @@ use crate::policy::{self, AttemptError, OpClass, OwnWrite, OwnWrites, RetryPolic
|
||||
use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
@@ -90,7 +90,7 @@ struct VaultKeyData {
|
||||
/// Key usage type
|
||||
usage: KeyUsage,
|
||||
/// Key creation timestamp
|
||||
created_at: ZonedDateTime,
|
||||
created_at: Zoned,
|
||||
/// Key status
|
||||
status: KeyStatus,
|
||||
/// Key version
|
||||
@@ -104,7 +104,7 @@ struct VaultKeyData {
|
||||
/// Scheduled deletion deadline; absent on records written before deadline
|
||||
/// persistence landed, so it must stay optional for backward compatibility.
|
||||
#[serde(default)]
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
deletion_date: Option<Zoned>,
|
||||
/// When the key's material last became current through a rotation.
|
||||
///
|
||||
/// Written by [`VaultKmsClient::rotate_key`] as part of the same
|
||||
@@ -117,7 +117,7 @@ struct VaultKeyData {
|
||||
/// See [`crate::deletion_worker`] for why collapsing the two is safe for the
|
||||
/// rotation-age gauge.
|
||||
#[serde(default)]
|
||||
rotated_at: Option<ZonedDateTime>,
|
||||
rotated_at: Option<Zoned>,
|
||||
/// Encrypted key material (base64 encoded)
|
||||
encrypted_key_material: String,
|
||||
/// Version that pre-versioning envelopes (no `master_key_version`) resolve to.
|
||||
@@ -381,7 +381,7 @@ struct VaultKeyVersionRecord {
|
||||
/// Encrypted key material (base64 encoded)
|
||||
encrypted_key_material: String,
|
||||
/// When this version's material was created
|
||||
created_at: ZonedDateTime,
|
||||
created_at: Zoned,
|
||||
}
|
||||
|
||||
/// Sub-path (under each key path) reserved for immutable version records.
|
||||
@@ -1305,7 +1305,7 @@ impl VaultKmsClient {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: Some(key_data.version),
|
||||
context_binding,
|
||||
};
|
||||
@@ -1347,7 +1347,7 @@ impl VaultKmsClient {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: Some(key_data.version),
|
||||
context_binding,
|
||||
};
|
||||
@@ -1718,7 +1718,7 @@ impl VaultKmsClient {
|
||||
let key_data = VaultKeyData {
|
||||
algorithm: algorithm.to_string(),
|
||||
usage: attributes.map_or(KeyUsage::EncryptDecrypt, |request| request.key_usage.clone()),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
status: KeyStatus::Active,
|
||||
version: 1,
|
||||
description: attributes.and_then(|request| request.description.clone()),
|
||||
@@ -2003,7 +2003,7 @@ impl VaultKmsClient {
|
||||
let record = VaultKeyVersionRecord {
|
||||
version: new_version,
|
||||
encrypted_key_material: new_material.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
};
|
||||
if !self.try_create_key_version_record(key_id, &record).await? {
|
||||
// A record for the next version already exists: an interrupted rotation
|
||||
@@ -2027,7 +2027,7 @@ impl VaultKmsClient {
|
||||
// the recorded time never runs ahead of the current version.
|
||||
key_data.version = new_version;
|
||||
key_data.encrypted_key_material = new_material;
|
||||
key_data.rotated_at = Some(ZonedDateTime::now());
|
||||
key_data.rotated_at = Some(Zoned::now());
|
||||
// Fresh material, fresh AES-GCM nonce budget: the wrap ceiling is per
|
||||
// key material version, so the counter restarts with the same commit
|
||||
// that makes the new material current.
|
||||
@@ -2114,7 +2114,7 @@ impl VaultKmsBackend {
|
||||
/// record whose key material is missing: persisting it would cement the
|
||||
/// empty-material state under a fresh document version — a damaged key
|
||||
/// must go through an explicit repair operation, not a lifecycle update.
|
||||
async fn mark_key_pending_deletion(&self, key_id: &str, deletion_date: &ZonedDateTime) -> Result<()> {
|
||||
async fn mark_key_pending_deletion(&self, key_id: &str, deletion_date: &Zoned) -> Result<()> {
|
||||
self.client
|
||||
.update_key_data_with_cas(key_id, |key_data| {
|
||||
ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::ScheduleDeletion)?;
|
||||
@@ -2144,7 +2144,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: request.key_usage,
|
||||
description: request.description,
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "VAULT".to_string(),
|
||||
key_manager: "VAULT".to_string(),
|
||||
@@ -2287,7 +2287,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
.await?;
|
||||
self.client.delete_key(key_id).await?;
|
||||
key_metadata.key_state = KeyState::PendingDeletion;
|
||||
key_metadata.deletion_date = Some(ZonedDateTime::now());
|
||||
key_metadata.deletion_date = Some(Zoned::now());
|
||||
|
||||
// No deletion date: the key is permanently gone, not scheduled.
|
||||
None
|
||||
@@ -2306,7 +2306,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
)));
|
||||
}
|
||||
|
||||
let deletion_date = ZonedDateTime::now() + Duration::from_secs(days as u64 * 86400);
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
self.mark_key_pending_deletion(key_id, &deletion_date).await?;
|
||||
key_metadata.key_state = KeyState::PendingDeletion;
|
||||
key_metadata.deletion_date = Some(deletion_date.clone());
|
||||
@@ -2415,7 +2415,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
.with_production_supported(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
// Tombstone under check-and-set with per-attempt re-validation: a
|
||||
// cancellation landing between the read and the write makes the write
|
||||
// conflict, and the re-read then observes the cancelled state and
|
||||
@@ -2513,7 +2513,7 @@ mod tests {
|
||||
VaultKeyData {
|
||||
algorithm: "AES_256".to_string(),
|
||||
usage: KeyUsage::EncryptDecrypt,
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
status: KeyStatus::Active,
|
||||
version: 1,
|
||||
description: None,
|
||||
@@ -3197,7 +3197,7 @@ mod tests {
|
||||
let key_data = VaultKeyData {
|
||||
algorithm: "AES_256".to_string(),
|
||||
usage: KeyUsage::EncryptDecrypt,
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
status: KeyStatus::Active,
|
||||
version: 1,
|
||||
description: None,
|
||||
@@ -3235,8 +3235,8 @@ mod tests {
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.wrap_budget_reserved = 7_000_000;
|
||||
key_data.baseline_version = Some(2);
|
||||
key_data.rotated_at = Some(ZonedDateTime::now());
|
||||
key_data.deletion_date = Some(ZonedDateTime::now());
|
||||
key_data.rotated_at = Some(Zoned::now());
|
||||
key_data.deletion_date = Some(Zoned::now());
|
||||
key_data.description = Some("described".to_string());
|
||||
|
||||
let value = serde_json::to_value(&key_data).expect("serialize key data");
|
||||
@@ -3683,11 +3683,11 @@ mod tests {
|
||||
/// of deadline persistence turns this test red.
|
||||
#[test]
|
||||
fn vault_key_data_deletion_date_round_trips_and_stays_backward_compatible() {
|
||||
let deadline = ZonedDateTime::now() + Duration::from_secs(7 * 86400);
|
||||
let deadline = Zoned::now() + Duration::from_secs(7 * 86400);
|
||||
let key_data = VaultKeyData {
|
||||
algorithm: "AES_256".to_string(),
|
||||
usage: KeyUsage::EncryptDecrypt,
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
status: KeyStatus::PendingDeletion,
|
||||
version: 1,
|
||||
description: None,
|
||||
@@ -3703,7 +3703,7 @@ mod tests {
|
||||
let mut value = serde_json::to_value(&key_data).expect("serialize");
|
||||
let restored: VaultKeyData = serde_json::from_value(value.clone()).expect("round trip");
|
||||
assert_eq!(
|
||||
restored.deletion_date.as_ref().map(ZonedDateTime::timestamp),
|
||||
restored.deletion_date.as_ref().map(Zoned::timestamp),
|
||||
Some(deadline.timestamp()),
|
||||
"deletion deadline must survive the KV2 round trip"
|
||||
);
|
||||
@@ -4055,10 +4055,10 @@ mod tests {
|
||||
/// cancellation and destroyed the key.
|
||||
#[tokio::test]
|
||||
async fn wired_expired_key_sweep_yields_to_concurrent_cancellation() {
|
||||
let now = ZonedDateTime::now() + Duration::from_secs(3600);
|
||||
let now = Zoned::now() + Duration::from_secs(3600);
|
||||
let mut pending = healthy_key_data();
|
||||
pending.status = KeyStatus::PendingDeletion;
|
||||
pending.deletion_date = Some(ZonedDateTime::now());
|
||||
pending.deletion_date = Some(Zoned::now());
|
||||
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
ScriptedResponse::ok(kv2_metadata_read_data(1)),
|
||||
@@ -4103,7 +4103,7 @@ mod tests {
|
||||
async fn wired_cancel_deletion_after_sweep_tombstone_fails_closed() {
|
||||
let mut pending = healthy_key_data();
|
||||
pending.status = KeyStatus::PendingDeletion;
|
||||
pending.deletion_date = Some(ZonedDateTime::now());
|
||||
pending.deletion_date = Some(Zoned::now());
|
||||
let mut tombstoned = healthy_key_data();
|
||||
tombstoned.status = KeyStatus::Deleted;
|
||||
|
||||
@@ -4192,7 +4192,7 @@ mod tests {
|
||||
async fn wired_immediate_deletion_of_a_pending_key_destroys_it() {
|
||||
let mut pending = healthy_key_data();
|
||||
pending.status = KeyStatus::PendingDeletion;
|
||||
pending.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(7 * 86400));
|
||||
pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
|
||||
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// describe_key: key info plus stored metadata.
|
||||
@@ -4328,7 +4328,7 @@ mod tests {
|
||||
async fn wired_schedule_deletion_regates_after_conflict() {
|
||||
let mut already_pending = healthy_key_data();
|
||||
already_pending.status = KeyStatus::PendingDeletion;
|
||||
already_pending.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(7 * 86400));
|
||||
already_pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
|
||||
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
|
||||
@@ -4747,7 +4747,7 @@ mod tests {
|
||||
let record_v2 = VaultKeyVersionRecord {
|
||||
version: 2,
|
||||
encrypted_key_material: base64_simd::STANDARD.encode_to_string(material_v2),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
};
|
||||
// A well-formed envelope wrapped under version 2 — under a reverted
|
||||
// guard this decrypt would *succeed*, which is exactly the masked
|
||||
@@ -4763,7 +4763,7 @@ mod tests {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: Some(2),
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -4843,7 +4843,7 @@ mod tests {
|
||||
let record_v2 = VaultKeyVersionRecord {
|
||||
version: 2,
|
||||
encrypted_key_material: adopted_material.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
};
|
||||
|
||||
let (vault, client) = scripted_client(vec![
|
||||
@@ -5042,7 +5042,7 @@ mod tests {
|
||||
/// Reverting either half turns this test red.
|
||||
#[tokio::test]
|
||||
async fn wired_rotate_persists_rotation_time_and_describe_reports_it() {
|
||||
let created_at = ZonedDateTime::now() - Duration::from_secs(365 * 86400);
|
||||
let created_at = Zoned::now() - Duration::from_secs(365 * 86400);
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.created_at = created_at.clone();
|
||||
key_data.version = 2;
|
||||
@@ -5069,7 +5069,7 @@ mod tests {
|
||||
let persisted: VaultKeyData =
|
||||
serde_json::from_value(committed["data"].clone()).expect("the committed record must deserialize");
|
||||
assert_eq!(
|
||||
persisted.rotated_at.as_ref().map(ZonedDateTime::timestamp),
|
||||
persisted.rotated_at.as_ref().map(Zoned::timestamp),
|
||||
Some(reported.timestamp()),
|
||||
"the rotation must persist the time it reports: {committed}"
|
||||
);
|
||||
@@ -5091,12 +5091,12 @@ mod tests {
|
||||
.await
|
||||
.expect("describe the rotated key");
|
||||
assert_eq!(
|
||||
described.rotated_at.as_ref().map(ZonedDateTime::timestamp),
|
||||
described.rotated_at.as_ref().map(Zoned::timestamp),
|
||||
Some(reported.timestamp()),
|
||||
"describe must report the persisted rotation time"
|
||||
);
|
||||
assert_ne!(
|
||||
described.rotated_at.as_ref().map(ZonedDateTime::timestamp),
|
||||
described.rotated_at.as_ref().map(Zoned::timestamp),
|
||||
Some(created_at.timestamp()),
|
||||
"a rotated key must not be aged from its creation"
|
||||
);
|
||||
@@ -5109,7 +5109,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn wired_describe_key_invents_no_rotation_time_for_legacy_records() {
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.created_at = ZonedDateTime::now() - Duration::from_secs(365 * 86400);
|
||||
key_data.created_at = Zoned::now() - Duration::from_secs(365 * 86400);
|
||||
key_data.version = 4;
|
||||
key_data.baseline_version = Some(1);
|
||||
|
||||
@@ -5143,7 +5143,7 @@ mod tests {
|
||||
/// rest of their contents intact.
|
||||
#[test]
|
||||
fn vault_key_data_rotated_at_round_trips_and_stays_backward_compatible() {
|
||||
let rotated_at = ZonedDateTime::now();
|
||||
let rotated_at = Zoned::now();
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.rotated_at = Some(rotated_at.clone());
|
||||
key_data.baseline_version = Some(1);
|
||||
@@ -5153,7 +5153,7 @@ mod tests {
|
||||
let mut value = serde_json::to_value(&key_data).expect("serialize");
|
||||
let restored: VaultKeyData = serde_json::from_value(value.clone()).expect("round trip");
|
||||
assert_eq!(
|
||||
restored.rotated_at.as_ref().map(ZonedDateTime::timestamp),
|
||||
restored.rotated_at.as_ref().map(Zoned::timestamp),
|
||||
Some(rotated_at.timestamp()),
|
||||
"the rotation time must survive the KV2 round trip"
|
||||
);
|
||||
@@ -5190,7 +5190,7 @@ mod tests {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -5250,7 +5250,7 @@ mod tests {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -5302,7 +5302,7 @@ mod tests {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
|
||||
@@ -33,7 +33,7 @@ use crate::policy::{self, AttemptError, OpClass, OwnWrite, OwnWrites, RetryPolic
|
||||
use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use moka::future::Cache;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
@@ -141,8 +141,8 @@ struct TransitKeyMetadata {
|
||||
description: Option<String>,
|
||||
tags: HashMap<String, String>,
|
||||
key_state: KeyState,
|
||||
created_at: ZonedDateTime,
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
created_at: Zoned,
|
||||
deletion_date: Option<Zoned>,
|
||||
origin: String,
|
||||
created_by: Option<String>,
|
||||
current_version: u32,
|
||||
@@ -160,8 +160,8 @@ struct TransitKeyMetadataPersisted {
|
||||
description: Option<String>,
|
||||
tags: HashMap<String, String>,
|
||||
key_state: KeyState,
|
||||
created_at: ZonedDateTime,
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
created_at: Zoned,
|
||||
deletion_date: Option<Zoned>,
|
||||
origin: String,
|
||||
created_by: Option<String>,
|
||||
current_version: u32,
|
||||
@@ -336,7 +336,7 @@ impl TransitKeyMetadata {
|
||||
description: request.description.clone(),
|
||||
tags: request.tags.clone(),
|
||||
key_state: KeyState::Enabled,
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: request.origin.clone().unwrap_or_else(|| "VAULT_TRANSIT".to_string()),
|
||||
created_by: None,
|
||||
@@ -360,7 +360,7 @@ impl TransitKeyMetadata {
|
||||
description: None,
|
||||
tags: HashMap::new(),
|
||||
key_state: KeyState::Enabled,
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "VAULT_TRANSIT".to_string(),
|
||||
created_by: None,
|
||||
@@ -1055,7 +1055,7 @@ impl VaultTransitKmsClient {
|
||||
encrypted_key: encrypted_key.into_bytes(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
// Transit ciphertext already self-describes its key version
|
||||
// ("vault:vN:..."), so the envelope never carries one.
|
||||
master_key_version: None,
|
||||
@@ -1100,7 +1100,7 @@ impl VaultTransitKmsClient {
|
||||
encrypted_key: encrypted.into_bytes(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
// Transit ciphertext already self-describes its key version
|
||||
// ("vault:vN:..."), so the envelope never carries one.
|
||||
master_key_version: None,
|
||||
@@ -1543,7 +1543,7 @@ impl VaultTransitKmsClient {
|
||||
pending_window_days: u32,
|
||||
_context: Option<&OperationContext>,
|
||||
) -> Result<()> {
|
||||
let deletion_date = ZonedDateTime::now() + Duration::from_secs(pending_window_days as u64 * 86400);
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400);
|
||||
self.mutate_key_metadata(key_id, |metadata| {
|
||||
ensure_key_state_permits(key_id, &metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
|
||||
metadata.key_state = KeyState::PendingDeletion;
|
||||
@@ -1592,7 +1592,7 @@ impl VaultTransitKmsClient {
|
||||
description: metadata.description,
|
||||
metadata: metadata.tags,
|
||||
created_at: metadata.created_at,
|
||||
rotated_at: Some(ZonedDateTime::now()),
|
||||
rotated_at: Some(Zoned::now()),
|
||||
created_by: metadata.created_by,
|
||||
deletion_date: None,
|
||||
})
|
||||
@@ -1810,7 +1810,7 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
// Transit key is destroyed leaves a key that is already unusable.
|
||||
// Re-checked against every fresh snapshot: a concurrent
|
||||
// cancellation that commits first fails this removal.
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
self.client
|
||||
.mutate_key_metadata(&key_id, |metadata| match metadata.key_state {
|
||||
KeyState::PendingDeletion | KeyState::Unavailable => {
|
||||
@@ -1844,7 +1844,7 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
)));
|
||||
}
|
||||
|
||||
let scheduled = ZonedDateTime::now() + Duration::from_secs(days as u64 * 86400);
|
||||
let scheduled = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
self.client
|
||||
.mutate_key_metadata(&key_id, |metadata| {
|
||||
// Re-run the gate against every fresh snapshot: the check
|
||||
@@ -1934,7 +1934,7 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
.with_production_supported(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
// The transit key's existence anchors "already removed": once it is
|
||||
// gone only stale scheduling metadata can remain, so clean that up.
|
||||
match self.client.read_transit_key(key_id).await {
|
||||
@@ -3220,7 +3220,7 @@ mod tests {
|
||||
encrypted_key: b"vault:v1:AAAA".to_vec(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: Some(1),
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -3297,7 +3297,7 @@ mod tests {
|
||||
async fn wired_immediate_deletion_of_a_pending_key_destroys_it() {
|
||||
let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
pending.key_state = KeyState::PendingDeletion;
|
||||
pending.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(7 * 86400));
|
||||
pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// Current key metadata for the response.
|
||||
ScriptedResponse::ok(transit_key_read_data("wired-key")),
|
||||
@@ -3354,7 +3354,7 @@ mod tests {
|
||||
async fn wired_expired_sweep_backs_off_when_cancel_wins_the_cas_race() {
|
||||
let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
pending.key_state = KeyState::PendingDeletion;
|
||||
pending.deletion_date = Some(ZonedDateTime::now());
|
||||
pending.deletion_date = Some(Zoned::now());
|
||||
let cancelled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// The transit key still exists.
|
||||
@@ -3378,7 +3378,7 @@ mod tests {
|
||||
.await
|
||||
.expect("vault transit backend should build");
|
||||
|
||||
let now = ZonedDateTime::now() + Duration::from_secs(3600);
|
||||
let now = Zoned::now() + Duration::from_secs(3600);
|
||||
let outcome = backend
|
||||
.remove_expired_key("wired-key", &now)
|
||||
.await
|
||||
|
||||
@@ -75,7 +75,7 @@ use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::service::ObjectEncryptionService;
|
||||
use crate::types::{CreateKeyRequest, EncryptionAlgorithm, EncryptionMetadata, KeyUsage};
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
@@ -272,10 +272,10 @@ pub struct DrillRecoveryEvidence {
|
||||
pub struct DrillRpoEvidence {
|
||||
/// When the export sealed the bundle.
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub snapshot_completed_at: ZonedDateTime,
|
||||
pub snapshot_completed_at: Zoned,
|
||||
/// When the disaster struck.
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub disaster_at: ZonedDateTime,
|
||||
pub disaster_at: Zoned,
|
||||
/// The data-loss window: work committed inside it is not in the bundle.
|
||||
pub rpo_window_millis: u64,
|
||||
/// Keys created after the fence closed, which the bundle cannot contain.
|
||||
@@ -322,7 +322,7 @@ pub struct DrillEvidence {
|
||||
pub dataset: DrillDataset,
|
||||
/// When the run started.
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub started_at: ZonedDateTime,
|
||||
pub started_at: Zoned,
|
||||
/// Bundle evidence.
|
||||
pub bundle: DrillBundleEvidence,
|
||||
/// Recovery evidence.
|
||||
@@ -508,7 +508,7 @@ impl PhaseClock {
|
||||
/// still produces the record of *how* it failed.
|
||||
pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result<DrillEvidence> {
|
||||
request.validate()?;
|
||||
let started_at = ZonedDateTime::now();
|
||||
let started_at = Zoned::now();
|
||||
let mut clock = PhaseClock::new();
|
||||
let mut findings = Vec::new();
|
||||
|
||||
@@ -557,7 +557,7 @@ pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result<
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let snapshot_completed_at = ZonedDateTime::now();
|
||||
let snapshot_completed_at = Zoned::now();
|
||||
clock.finish(DrillPhase::Export);
|
||||
|
||||
// Past the fence: this key belongs to the window the bundle cannot
|
||||
@@ -576,7 +576,7 @@ pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result<
|
||||
let bundle_digest_before = tree_digest(&bundle_dir).await?;
|
||||
|
||||
let destroyed_files = apply_disaster(&live_dir, request.disaster).await?;
|
||||
let disaster_at = ZonedDateTime::now();
|
||||
let disaster_at = Zoned::now();
|
||||
clock.finish(DrillPhase::Disaster);
|
||||
|
||||
// Recovery step one is always the same: preserve what survived, then
|
||||
@@ -980,7 +980,7 @@ fn walk_files(root: &Path) -> std::io::Result<Vec<PathBuf>> {
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn elapsed_millis(from: &ZonedDateTime, to: &ZonedDateTime) -> u64 {
|
||||
fn elapsed_millis(from: &Zoned, to: &Zoned) -> u64 {
|
||||
let nanos = to.timestamp().as_nanosecond() - from.timestamp().as_nanosecond();
|
||||
u64::try_from(nanos / 1_000_000).unwrap_or(0)
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ use aes_gcm::{
|
||||
Aes256Gcm, Key, Nonce,
|
||||
aead::{Aead, KeyInit, Payload},
|
||||
};
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use rand::RngExt;
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -454,7 +454,7 @@ async fn build_and_write_bundle(
|
||||
// Normalized to UTC so the stored spelling is host-independent: the
|
||||
// local zone's name (or a POSIX TZ string in minimal containers) has
|
||||
// no business inside a portable bundle.
|
||||
created_at: ZonedDateTime::now().with_time_zone(jiff::tz::TimeZone::UTC),
|
||||
created_at: Zoned::now().with_time_zone(jiff::tz::TimeZone::UTC),
|
||||
rustfs_version: request.rustfs_version.clone(),
|
||||
deployment_identity: request.deployment_identity.clone(),
|
||||
backend: BackupBackendKind::Local,
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
use crate::backup::capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
|
||||
use crate::backup::error::BackupError;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeSet;
|
||||
@@ -503,7 +503,7 @@ pub struct BackupManifest {
|
||||
pub backup_id: String,
|
||||
/// Creation time of the bundle.
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub created_at: ZonedDateTime,
|
||||
pub created_at: Zoned,
|
||||
/// RustFS version that produced the bundle.
|
||||
pub rustfs_version: String,
|
||||
/// Opaque identity of the producing deployment, used by restore preflight
|
||||
|
||||
@@ -239,7 +239,7 @@ impl KmsCache {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{KeyState, KeyUsage};
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use std::time::Duration;
|
||||
@@ -264,7 +264,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
@@ -342,7 +342,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: Some("Test key".to_string()),
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
@@ -376,7 +376,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: Some("TTL test key".to_string()),
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
@@ -406,7 +406,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::backends::{ExpiredKeyRemoval, KmsBackend};
|
||||
use crate::error::Result;
|
||||
use crate::types::{KeyInfo, KeyStatus, ListKeysRequest, OperationContext};
|
||||
use async_trait::async_trait;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -117,7 +117,7 @@ struct KeyCensus {
|
||||
}
|
||||
|
||||
impl KeyCensus {
|
||||
fn observe(&mut self, key: &KeyInfo, now: &ZonedDateTime) {
|
||||
fn observe(&mut self, key: &KeyInfo, now: &Zoned) {
|
||||
match key.status {
|
||||
KeyStatus::PendingDeletion => self.pending_deletion += 1,
|
||||
KeyStatus::Deleted => self.tombstones += 1,
|
||||
@@ -145,7 +145,7 @@ impl KeyCensus {
|
||||
|
||||
/// Seconds from `earlier` to `now`, clamped at zero so clock skew (or a
|
||||
/// timestamp persisted by a node running ahead) cannot produce a negative age.
|
||||
fn seconds_between(earlier: &ZonedDateTime, now: &ZonedDateTime) -> f64 {
|
||||
fn seconds_between(earlier: &Zoned, now: &Zoned) -> f64 {
|
||||
(now.timestamp().as_second() - earlier.timestamp().as_second()).max(0) as f64
|
||||
}
|
||||
|
||||
@@ -273,7 +273,7 @@ impl DeletionWorker {
|
||||
}
|
||||
_ = ticker.tick() => {}
|
||||
}
|
||||
let report = self.sweep(&ZonedDateTime::now()).await;
|
||||
let report = self.sweep(&Zoned::now()).await;
|
||||
if !report.removed.is_empty() || !report.blocked.is_empty() || report.failed > 0 || report.unreadable > 0 {
|
||||
info!(
|
||||
removed = ?report.removed,
|
||||
@@ -293,7 +293,7 @@ impl DeletionWorker {
|
||||
/// Also feeds the lifecycle gauges: every key it lists is counted into a
|
||||
/// [`KeyCensus`] on the way past, so the observability comes out of the
|
||||
/// pages the sweep already had to fetch.
|
||||
pub(crate) async fn sweep(&self, now: &ZonedDateTime) -> SweepReport {
|
||||
pub(crate) async fn sweep(&self, now: &Zoned) -> SweepReport {
|
||||
let mut report = SweepReport::default();
|
||||
let mut census = KeyCensus::default();
|
||||
let mut marker: Option<String> = None;
|
||||
@@ -359,7 +359,7 @@ impl DeletionWorker {
|
||||
}
|
||||
|
||||
/// Handle one key that is on its way out; reports whether it was removed.
|
||||
async fn process_key(&self, key_id: &str, now: &ZonedDateTime, report: &mut SweepReport) -> bool {
|
||||
async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) -> bool {
|
||||
// Never remove a key that live configuration still points at. The
|
||||
// default key check is built in; broader references (bucket
|
||||
// encryption settings, ...) come from the injected checker.
|
||||
@@ -464,8 +464,8 @@ mod tests {
|
||||
DeletionWorker::new(backend, None, None, "local")
|
||||
}
|
||||
|
||||
fn after_window() -> ZonedDateTime {
|
||||
ZonedDateTime::now() + Duration::from_secs(8 * 86400)
|
||||
fn after_window() -> Zoned {
|
||||
Zoned::now() + Duration::from_secs(8 * 86400)
|
||||
}
|
||||
|
||||
async fn assert_key_gone(backend: &LocalKmsBackend, key_id: &str) {
|
||||
@@ -488,7 +488,7 @@ mod tests {
|
||||
let worker = worker(backend.clone());
|
||||
|
||||
// Not yet due: nothing happens.
|
||||
let report = worker.sweep(&ZonedDateTime::now()).await;
|
||||
let report = worker.sweep(&Zoned::now()).await;
|
||||
assert!(report.removed.is_empty());
|
||||
assert_eq!(report.skipped, 1);
|
||||
assert_eq!(report.failed, 0);
|
||||
@@ -635,7 +635,7 @@ mod tests {
|
||||
let worker = DeletionWorker::new(backend.clone(), None, None, "local").with_audit_sink(Some(sink.clone()));
|
||||
|
||||
// A key that is not yet due was never at risk, so it produces no record.
|
||||
worker.sweep(&ZonedDateTime::now()).await;
|
||||
worker.sweep(&Zoned::now()).await;
|
||||
assert!(
|
||||
sink.records.lock().expect("sink lock").is_empty(),
|
||||
"a key the sweep declined to touch must not be audited as a deletion"
|
||||
@@ -780,7 +780,7 @@ mod tests {
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn key_info(key_id: &str, status: KeyStatus, created_at: ZonedDateTime, rotated_at: Option<ZonedDateTime>) -> KeyInfo {
|
||||
fn key_info(key_id: &str, status: KeyStatus, created_at: Zoned, rotated_at: Option<Zoned>) -> KeyInfo {
|
||||
KeyInfo {
|
||||
key_id: key_id.to_string(),
|
||||
description: None,
|
||||
@@ -801,7 +801,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn census_ages_from_rotation_and_ignores_departing_keys() {
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
let day = Duration::from_secs(86400);
|
||||
let mut census = KeyCensus::default();
|
||||
|
||||
@@ -833,7 +833,7 @@ mod tests {
|
||||
/// material never wraps again, so its consumed nonce budget is moot.
|
||||
#[test]
|
||||
fn census_takes_the_max_wrap_reservation_of_usable_keys_only() {
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
let mut census = KeyCensus::default();
|
||||
|
||||
census.observe(&key_info("uncounted", KeyStatus::Active, now.clone(), None), &now);
|
||||
@@ -859,7 +859,7 @@ mod tests {
|
||||
fn wrap_budget_gauge_is_aggregate_and_carries_no_key_label() {
|
||||
let (snapshot, ()) = record_metrics(|| {
|
||||
Box::pin(async {
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
let mut census = KeyCensus::default();
|
||||
let mut wrapped = key_info("wrapped-key-id", KeyStatus::Active, now.clone(), None);
|
||||
wrapped.wrap_budget_reserved = Some(2_000_000);
|
||||
@@ -897,7 +897,7 @@ mod tests {
|
||||
// Three days in, still inside the seven-day window: the sweep
|
||||
// observes the scheduled key instead of removing it.
|
||||
let report = worker(backend.clone())
|
||||
.sweep(&(ZonedDateTime::now() + Duration::from_secs(3 * 86400)))
|
||||
.sweep(&(Zoned::now() + Duration::from_secs(3 * 86400)))
|
||||
.await;
|
||||
assert_eq!(report.skipped, 1);
|
||||
assert!(report.removed.is_empty());
|
||||
@@ -950,7 +950,7 @@ mod tests {
|
||||
|
||||
// Well inside the seven-day window: every key is observed as
|
||||
// pending rather than removed.
|
||||
let report = worker(backend.clone()).sweep(&ZonedDateTime::now()).await;
|
||||
let report = worker(backend.clone()).sweep(&Zoned::now()).await;
|
||||
assert_eq!(report.skipped, total, "every scheduled key must be inspected");
|
||||
assert!(report.removed.is_empty());
|
||||
})
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary};
|
||||
use async_trait::async_trait;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use rand::Rng;
|
||||
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -66,7 +66,7 @@ pub struct DataKeyEnvelope {
|
||||
pub nonce: Vec<u8>,
|
||||
pub encryption_context: HashMap<String, String>,
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub created_at: ZonedDateTime,
|
||||
pub created_at: Zoned,
|
||||
/// KEK version that wrapped `encrypted_key`; `None` on pre-versioning envelopes.
|
||||
///
|
||||
/// Optional and omitted when `None` so envelopes from non-rotating backends stay
|
||||
@@ -205,7 +205,7 @@ impl<'de> Deserialize<'de> for DataKeyEnvelope {
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] ZonedDateTime);
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] Zoned);
|
||||
|
||||
struct DataKeyEnvelopeVisitor;
|
||||
|
||||
@@ -591,7 +591,7 @@ mod tests {
|
||||
encrypted_key: vec![1, 2, 3, 4],
|
||||
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
encryption_context: HashMap::from([("bucket".to_string(), "b".to_string())]),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -625,7 +625,7 @@ mod tests {
|
||||
encrypted_key: vec![1, 2, 3, 4],
|
||||
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: Some(CONTEXT_BINDING_AAD_V1),
|
||||
};
|
||||
@@ -676,7 +676,7 @@ mod tests {
|
||||
map.insert("bucket".to_string(), "test-bucket".to_string());
|
||||
map
|
||||
},
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -694,7 +694,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_data_key_envelope_backward_compatibility() {
|
||||
// Test deserialization with current ZonedDateTime format (with timezone annotation)
|
||||
// Test deserialization with current Zoned format (with timezone annotation)
|
||||
let envelope_json = r#"{
|
||||
"key_id": "test-key-id",
|
||||
"master_key_id": "master-key-id",
|
||||
@@ -812,7 +812,7 @@ mod tests {
|
||||
encrypted_key: vec![1, 2, 3, 4],
|
||||
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -832,7 +832,7 @@ mod tests {
|
||||
encrypted_key: vec![1, 2, 3, 4],
|
||||
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: Some(7),
|
||||
context_binding: None,
|
||||
};
|
||||
|
||||
@@ -28,7 +28,7 @@ use crate::types::{
|
||||
MAX_PENDING_DELETION_WINDOW_DAYS, MIN_PENDING_DELETION_WINDOW_DAYS, OperationContext, RewrapDataKeyRequest,
|
||||
RewrapDataKeyResponse, RotationDueReason,
|
||||
};
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -335,7 +335,7 @@ impl KmsManager {
|
||||
let mut result = self.backend.list_keys(request).await;
|
||||
if let Ok(response) = result.as_mut() {
|
||||
let rotates = self.backend.capabilities().rotate;
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
for key in &mut response.keys {
|
||||
self.apply_rotation_readiness(key, rotates, &now);
|
||||
}
|
||||
@@ -351,7 +351,7 @@ impl KmsManager {
|
||||
/// about what "overdue" means, and so a backend that cannot rotate is never
|
||||
/// the one deciding whether it should. Nothing consults the verdict before
|
||||
/// encrypting or decrypting: a key reported as due keeps serving traffic.
|
||||
fn apply_rotation_readiness(&self, key: &mut KeyInfo, backend_rotates: bool, now: &ZonedDateTime) {
|
||||
fn apply_rotation_readiness(&self, key: &mut KeyInfo, backend_rotates: bool, now: &Zoned) {
|
||||
if !backend_rotates {
|
||||
// Reported, not silently omitted: an operator chasing an overdue key
|
||||
// needs to know the answer is "this backend cannot rotate at all",
|
||||
@@ -699,7 +699,7 @@ mod tests {
|
||||
use crate::error::KmsError;
|
||||
use crate::types::{KeyMetadata, KeySpec, KeyState, KeyStatus, KeyUsage};
|
||||
use async_trait::async_trait;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
@@ -764,7 +764,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "RUSTFS_KMS".to_string(),
|
||||
key_manager: "RUSTFS".to_string(),
|
||||
@@ -1810,7 +1810,7 @@ mod tests {
|
||||
manager
|
||||
}
|
||||
|
||||
fn aged_key(rotated_at: Option<ZonedDateTime>, created_at: ZonedDateTime) -> KeyInfo {
|
||||
fn aged_key(rotated_at: Option<Zoned>, created_at: Zoned) -> KeyInfo {
|
||||
KeyInfo {
|
||||
key_id: "key-a".to_string(),
|
||||
description: None,
|
||||
@@ -1833,7 +1833,7 @@ mod tests {
|
||||
/// a backend that cannot rotate is never told to rotate.
|
||||
#[test]
|
||||
fn rotation_readiness_reports_but_never_asks_the_impossible() {
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
let long_ago = &now - jiff::Span::new().days(400);
|
||||
let recently = &now - jiff::Span::new().hours(1);
|
||||
let day = Duration::from_secs(86_400);
|
||||
@@ -1887,13 +1887,13 @@ mod tests {
|
||||
/// independently of the age policy and reported under its own reason.
|
||||
#[test]
|
||||
fn rotation_readiness_reports_an_exhausted_wrap_budget() {
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
let recently = &now - jiff::Span::new().hours(1);
|
||||
let long_ago = &now - jiff::Span::new().days(400);
|
||||
let day = Duration::from_secs(86_400);
|
||||
let budget = 2_000_000;
|
||||
|
||||
let with_wraps = |manager: &KmsManager, wraps: Option<u64>, rotated_at: Option<ZonedDateTime>| {
|
||||
let with_wraps = |manager: &KmsManager, wraps: Option<u64>, rotated_at: Option<Zoned>| {
|
||||
let mut key = aged_key(rotated_at, recently.clone());
|
||||
key.wrap_budget_reserved = wraps;
|
||||
manager.apply_rotation_readiness(&mut key, true, &now);
|
||||
|
||||
@@ -595,7 +595,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: jiff::ZonedDateTime::now(),
|
||||
creation_date: jiff::Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "test".to_string(),
|
||||
key_manager: "test".to_string(),
|
||||
|
||||
@@ -23,7 +23,7 @@ use crate::encryption::context_aad;
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::types::*;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rand::random;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
@@ -543,7 +543,7 @@ impl ObjectEncryptionService {
|
||||
iv,
|
||||
tag: Some(tag),
|
||||
encryption_context: context,
|
||||
encrypted_at: ZonedDateTime::now(),
|
||||
encrypted_at: Zoned::now(),
|
||||
// Pinned to the bytes actually fed to the AEAD, so the projection
|
||||
// below can store them verbatim instead of re-deriving them.
|
||||
context_aad: Some(aad),
|
||||
@@ -702,7 +702,7 @@ impl ObjectEncryptionService {
|
||||
iv,
|
||||
tag: Some(tag),
|
||||
encryption_context: context,
|
||||
encrypted_at: ZonedDateTime::now(),
|
||||
encrypted_at: Zoned::now(),
|
||||
// Pinned to the bytes actually fed to the AEAD, so the projection
|
||||
// below can store them verbatim instead of re-deriving them.
|
||||
context_aad: Some(aad),
|
||||
@@ -953,7 +953,7 @@ impl ObjectEncryptionService {
|
||||
iv,
|
||||
tag,
|
||||
encryption_context,
|
||||
encrypted_at: ZonedDateTime::now(),
|
||||
encrypted_at: Zoned::now(),
|
||||
original_size: 0, // Not available from headers
|
||||
encrypted_data_key,
|
||||
context_aad,
|
||||
@@ -1070,7 +1070,7 @@ mod tests {
|
||||
iv: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
|
||||
tag: Some(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
|
||||
encryption_context: HashMap::from([("bucket".to_string(), "test-bucket".to_string())]),
|
||||
encrypted_at: ZonedDateTime::now(),
|
||||
encrypted_at: Zoned::now(),
|
||||
original_size: 100,
|
||||
encrypted_data_key: vec![1, 2, 3, 4],
|
||||
// A hand-built record with no sealed bytes to defer to.
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
use jiff::{Timestamp, ZonedDateTime, tz::TimeZone};
|
||||
use jiff::{Timestamp, Zoned, tz::TimeZone};
|
||||
use serde::{Deserialize, Deserializer, Serializer};
|
||||
|
||||
pub(crate) mod zoned {
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn serialize<S>(value: &ZonedDateTime, serializer: S) -> Result<S::Ok, S::Error>
|
||||
pub(crate) fn serialize<S>(value: &Zoned, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
serializer.serialize_str(&value.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<ZonedDateTime, D::Error>
|
||||
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Zoned, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
@@ -23,7 +23,7 @@ pub(crate) mod zoned {
|
||||
pub(crate) mod option_zoned {
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn serialize<S>(value: &Option<ZonedDateTime>, serializer: S) -> Result<S::Ok, S::Error>
|
||||
pub(crate) fn serialize<S>(value: &Option<Zoned>, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
@@ -33,7 +33,7 @@ pub(crate) mod option_zoned {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Option<ZonedDateTime>, D::Error>
|
||||
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Option<Zoned>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
@@ -44,15 +44,15 @@ pub(crate) mod option_zoned {
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_zoned_compat(value: &str) -> Result<ZonedDateTime, String> {
|
||||
if let Ok(zoned) = value.parse::<ZonedDateTime>() {
|
||||
fn parse_zoned_compat(value: &str) -> Result<Zoned, String> {
|
||||
if let Ok(zoned) = value.parse::<Zoned>() {
|
||||
return Ok(zoned);
|
||||
}
|
||||
|
||||
let timestamp = value
|
||||
.parse::<Timestamp>()
|
||||
.map_err(|err| format!("failed to parse legacy timestamp '{value}': {err}"))?;
|
||||
Ok(timestamp.to_zoned_datetime(TimeZone::UTC))
|
||||
Ok(timestamp.to_zoned(TimeZone::UTC))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
+13
-13
@@ -14,7 +14,7 @@
|
||||
|
||||
//! Core type definitions for KMS operations
|
||||
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use uuid::Uuid;
|
||||
@@ -37,7 +37,7 @@ pub struct DataKeyInfo {
|
||||
/// Associated metadata
|
||||
pub metadata: HashMap<String, String>,
|
||||
/// Key creation timestamp
|
||||
pub created_at: ZonedDateTime,
|
||||
pub created_at: Zoned,
|
||||
}
|
||||
|
||||
impl DataKeyInfo {
|
||||
@@ -61,7 +61,7 @@ impl DataKeyInfo {
|
||||
ciphertext,
|
||||
key_spec,
|
||||
metadata: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,14 +112,14 @@ pub struct MasterKeyInfo {
|
||||
/// Associated metadata
|
||||
pub metadata: HashMap<String, String>,
|
||||
/// Key creation timestamp
|
||||
pub created_at: ZonedDateTime,
|
||||
pub created_at: Zoned,
|
||||
/// Key last rotation timestamp
|
||||
pub rotated_at: Option<ZonedDateTime>,
|
||||
pub rotated_at: Option<Zoned>,
|
||||
/// Key creator/owner
|
||||
pub created_by: Option<String>,
|
||||
/// Scheduled deletion deadline while the key is pending deletion
|
||||
#[serde(default)]
|
||||
pub deletion_date: Option<ZonedDateTime>,
|
||||
pub deletion_date: Option<Zoned>,
|
||||
}
|
||||
|
||||
impl MasterKeyInfo {
|
||||
@@ -142,7 +142,7 @@ impl MasterKeyInfo {
|
||||
status: KeyStatus::Active,
|
||||
description: None,
|
||||
metadata: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
rotated_at: None,
|
||||
created_by,
|
||||
deletion_date: None,
|
||||
@@ -174,7 +174,7 @@ impl MasterKeyInfo {
|
||||
status: KeyStatus::Active,
|
||||
description,
|
||||
metadata: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
rotated_at: None,
|
||||
created_by,
|
||||
deletion_date: None,
|
||||
@@ -247,9 +247,9 @@ pub struct KeyInfo {
|
||||
/// Key tags
|
||||
pub tags: HashMap<String, String>,
|
||||
/// Key creation timestamp
|
||||
pub created_at: ZonedDateTime,
|
||||
pub created_at: Zoned,
|
||||
/// Key last rotation timestamp
|
||||
pub rotated_at: Option<ZonedDateTime>,
|
||||
pub rotated_at: Option<Zoned>,
|
||||
/// Key creator
|
||||
pub created_by: Option<String>,
|
||||
/// Whether the key has outlived the configured rotation age.
|
||||
@@ -690,7 +690,7 @@ pub struct EncryptionMetadata {
|
||||
/// Encryption context
|
||||
pub encryption_context: HashMap<String, String>,
|
||||
/// Timestamp when encrypted
|
||||
pub encrypted_at: ZonedDateTime,
|
||||
pub encrypted_at: Zoned,
|
||||
/// Size of original data
|
||||
pub original_size: u64,
|
||||
/// Encrypted data key
|
||||
@@ -789,9 +789,9 @@ pub struct KeyMetadata {
|
||||
/// Key description
|
||||
pub description: Option<String>,
|
||||
/// Key creation timestamp
|
||||
pub creation_date: ZonedDateTime,
|
||||
pub creation_date: Zoned,
|
||||
/// Key deletion timestamp
|
||||
pub deletion_date: Option<ZonedDateTime>,
|
||||
pub deletion_date: Option<Zoned>,
|
||||
/// Key origin
|
||||
pub origin: String,
|
||||
/// Key manager
|
||||
|
||||
@@ -39,7 +39,7 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use common::{TestKms, assert_key_not_found, assert_unsupported_capability, ctx};
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use rustfs_kms::backends::local::LocalKmsBackend;
|
||||
use rustfs_kms::backends::static_kms::StaticKmsBackend;
|
||||
use rustfs_kms::backends::{ExpiredKeyRemoval, KmsBackend};
|
||||
@@ -79,8 +79,8 @@ async fn schedule(backend: &LocalKmsBackend, key_id: &str, days: u32) {
|
||||
.expect("deletion should be scheduled");
|
||||
}
|
||||
|
||||
fn days_from_now(days: u64) -> ZonedDateTime {
|
||||
ZonedDateTime::now() + Duration::from_secs(days * 86_400)
|
||||
fn days_from_now(days: u64) -> Zoned {
|
||||
Zoned::now() + Duration::from_secs(days * 86_400)
|
||||
}
|
||||
|
||||
async fn exists(backend: &LocalKmsBackend, key_id: &str) -> bool {
|
||||
@@ -422,7 +422,7 @@ async fn a_backend_without_deletion_support_reports_the_capability_gap() {
|
||||
"the static backend must not advertise deletion scheduling"
|
||||
);
|
||||
assert_unsupported_capability(
|
||||
backend.remove_expired_key(common::STATIC_KEY_ID, &ZonedDateTime::now()).await,
|
||||
backend.remove_expired_key(common::STATIC_KEY_ID, &Zoned::now()).await,
|
||||
"remove_expired_key",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
|
||||
[dependencies]
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
flate2 = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -20,8 +20,7 @@ use crate::ingest::{IngestReport, SkipReason};
|
||||
use crate::model::{EventKind, LogEvent, LogLevel, ParseStats};
|
||||
use crate::redact;
|
||||
use crate::rules::{Finding, FindingsCollector, RuleEngine, Severity};
|
||||
use crate::time::Rfc3339;
|
||||
use jiff::{SignedDuration, ZonedDateTime};
|
||||
use chrono::{DateTime, Duration, FixedOffset, Utc};
|
||||
use regex::Regex;
|
||||
use serde::Serialize;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
@@ -36,9 +35,9 @@ const BUCKET_WIDTHS_MIN: [i64; 6] = [1, 5, 15, 60, 360, 1440];
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AnalyzeOptions {
|
||||
/// Keep events at/after this time. Timestamp-less events always pass.
|
||||
pub since: Option<ZonedDateTime>,
|
||||
pub since: Option<DateTime<FixedOffset>>,
|
||||
/// Keep events at/before this time. Timestamp-less events always pass.
|
||||
pub until: Option<ZonedDateTime>,
|
||||
pub until: Option<DateTime<FixedOffset>>,
|
||||
/// Drop events below this level; level-less events always pass.
|
||||
pub min_level: Option<LogLevel>,
|
||||
pub max_samples: usize,
|
||||
@@ -65,7 +64,7 @@ pub struct Summary {
|
||||
pub events_after_filter: u64,
|
||||
/// "ERROR" -> n, …, "no_level" -> n.
|
||||
pub level_counts: BTreeMap<String, u64>,
|
||||
pub time_range: Option<(ZonedDateTime, ZonedDateTime)>,
|
||||
pub time_range: Option<(DateTime<FixedOffset>, DateTime<FixedOffset>)>,
|
||||
pub nodes: Vec<String>,
|
||||
pub parse: ParseStats,
|
||||
pub files_parsed: u64,
|
||||
@@ -89,7 +88,7 @@ pub struct UnmatchedCluster {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct TimeBucket {
|
||||
/// Bucket start, normalized to UTC.
|
||||
pub start: ZonedDateTime,
|
||||
pub start: DateTime<Utc>,
|
||||
pub error: u64,
|
||||
pub warn: u64,
|
||||
pub other: u64,
|
||||
@@ -106,8 +105,8 @@ pub struct TimelineAnomaly {
|
||||
/// Involved node labels (node_ranges_disjoint); empty otherwise.
|
||||
pub nodes: Vec<String>,
|
||||
/// Gap boundaries in UTC (log_gap); null otherwise.
|
||||
pub gap_start: Option<ZonedDateTime>,
|
||||
pub gap_end: Option<ZonedDateTime>,
|
||||
pub gap_start: Option<DateTime<Utc>>,
|
||||
pub gap_end: Option<DateTime<Utc>>,
|
||||
/// log_gap only: a startup-class finding begins within 5min after the gap.
|
||||
pub restart_evidence: bool,
|
||||
}
|
||||
@@ -130,7 +129,7 @@ pub struct AnalysisReport {
|
||||
}
|
||||
|
||||
/// (min ts, max ts, timestamped event count) for one node label.
|
||||
type NodeRange = (ZonedDateTime, ZonedDateTime, u64);
|
||||
type NodeRange = (DateTime<FixedOffset>, DateTime<FixedOffset>, u64);
|
||||
|
||||
#[derive(Default)]
|
||||
struct BucketCounts {
|
||||
@@ -155,7 +154,7 @@ pub struct Analyzer {
|
||||
events_total: u64,
|
||||
events_after_filter: u64,
|
||||
level_counts: BTreeMap<String, u64>,
|
||||
time_range: Option<(ZonedDateTime, ZonedDateTime)>,
|
||||
time_range: Option<(DateTime<FixedOffset>, DateTime<FixedOffset>)>,
|
||||
nodes: BTreeSet<String>,
|
||||
offsets: BTreeSet<String>,
|
||||
/// node label -> (min ts, max ts, timestamped event count); feeds the
|
||||
@@ -190,9 +189,8 @@ impl Analyzer {
|
||||
pub fn observe(&mut self, mut event: LogEvent) {
|
||||
self.events_total += 1;
|
||||
|
||||
if let Some(ts) = event.timestamp.as_ref()
|
||||
&& (self.opts.since.as_ref().is_some_and(|since| ts < since)
|
||||
|| self.opts.until.as_ref().is_some_and(|until| ts > until))
|
||||
if let Some(ts) = event.timestamp
|
||||
&& (self.opts.since.is_some_and(|since| ts < since) || self.opts.until.is_some_and(|until| ts > until))
|
||||
{
|
||||
return;
|
||||
}
|
||||
@@ -219,26 +217,23 @@ impl Analyzer {
|
||||
self.nodes.insert(node.to_string());
|
||||
}
|
||||
|
||||
if let Some(ts) = event.timestamp.clone() {
|
||||
self.time_range = Some(match self.time_range.take() {
|
||||
None => (ts.clone(), ts.clone()),
|
||||
Some((first, last)) => (first.min(ts.clone()), last.max(ts.clone())),
|
||||
if let Some(ts) = event.timestamp {
|
||||
self.time_range = Some(match self.time_range {
|
||||
None => (ts, ts),
|
||||
Some((first, last)) => (first.min(ts), last.max(ts)),
|
||||
});
|
||||
self.offsets.insert(format_offset(&ts));
|
||||
if let Some(node) = event.node.as_deref() {
|
||||
self.node_ranges
|
||||
.entry(node.to_string())
|
||||
.and_modify(|(min, max, count)| {
|
||||
*min = min.clone().min(ts.clone());
|
||||
*max = max.clone().max(ts.clone());
|
||||
*min = (*min).min(ts);
|
||||
*max = (*max).max(ts);
|
||||
*count += 1;
|
||||
})
|
||||
.or_insert((ts.clone(), ts.clone(), 1));
|
||||
.or_insert((ts, ts, 1));
|
||||
}
|
||||
let bucket = self
|
||||
.minute_buckets
|
||||
.entry(ts.timestamp().as_second().div_euclid(60))
|
||||
.or_default();
|
||||
let bucket = self.minute_buckets.entry(ts.timestamp().div_euclid(60)).or_default();
|
||||
match event.level {
|
||||
Some(LogLevel::Error) => bucket.error += 1,
|
||||
Some(LogLevel::Warn) => bucket.warn += 1,
|
||||
@@ -361,8 +356,8 @@ impl Analyzer {
|
||||
}
|
||||
}
|
||||
|
||||
fn format_offset(ts: &ZonedDateTime) -> String {
|
||||
let seconds = crate::time::offset_seconds(ts);
|
||||
fn format_offset(ts: &DateTime<FixedOffset>) -> String {
|
||||
let seconds = ts.offset().local_minus_utc();
|
||||
if seconds == 0 {
|
||||
return "Z".to_string();
|
||||
}
|
||||
@@ -395,7 +390,7 @@ fn merge_timeline(minute_buckets: &BTreeMap<i64, BucketCounts>) -> (Vec<TimeBuck
|
||||
let buckets = merged
|
||||
.into_iter()
|
||||
.map(|(slot, counts)| TimeBucket {
|
||||
start: crate::time::utc_from_unix(slot * width * 60),
|
||||
start: DateTime::<Utc>::from_timestamp(slot * width * 60, 0).expect("valid timestamp"),
|
||||
error: counts.error,
|
||||
warn: counts.warn,
|
||||
other: counts.other,
|
||||
@@ -476,13 +471,13 @@ fn detect_timeline_anomalies(
|
||||
// span of at least max(15min, 3x display bucket width), then activity
|
||||
// again — the shape of a process restart or hang.
|
||||
let threshold = (3 * bucket_width_min).max(15);
|
||||
let startup_after_gap = |gap_end: &ZonedDateTime| {
|
||||
let startup_after_gap = |gap_end: DateTime<Utc>| {
|
||||
findings.iter().chain(low_confidence).find_map(|f| {
|
||||
if !STARTUP_RULE_IDS.contains(&f.rule_id.as_str()) {
|
||||
return None;
|
||||
}
|
||||
let first = f.first_seen.as_ref()?;
|
||||
(first >= gap_end && first <= gap_end + SignedDuration::from_mins(5)).then(|| f.rule_id.clone())
|
||||
let first = f.first_seen?.with_timezone(&Utc);
|
||||
(first >= gap_end && first <= gap_end + Duration::minutes(5)).then(|| f.rule_id.clone())
|
||||
})
|
||||
};
|
||||
let minutes: Vec<i64> = minute_buckets.keys().copied().collect();
|
||||
@@ -495,9 +490,9 @@ fn detect_timeline_anomalies(
|
||||
continue;
|
||||
}
|
||||
if run_len >= 3 && delta > threshold {
|
||||
let gap_start = crate::time::utc_from_unix((prev + 1) * 60);
|
||||
let gap_end = crate::time::utc_from_unix(next * 60);
|
||||
let restart = startup_after_gap(&gap_end);
|
||||
let gap_start = DateTime::<Utc>::from_timestamp((prev + 1) * 60, 0).expect("valid timestamp");
|
||||
let gap_end = DateTime::<Utc>::from_timestamp(next * 60, 0).expect("valid timestamp");
|
||||
let restart = startup_after_gap(gap_end);
|
||||
let span = human_duration_minutes(delta - 1);
|
||||
let message = match &restart {
|
||||
Some(rule_id) => format!(
|
||||
@@ -550,7 +545,7 @@ fn collapse_findings(findings: &mut [Finding]) {
|
||||
let index_of: HashMap<String, usize> = findings.iter().enumerate().map(|(i, f)| (f.rule_id.clone(), i)).collect();
|
||||
// Among several qualifying roots pick the earliest first_seen;
|
||||
// timestamp-less roots come last, rule id breaks remaining ties.
|
||||
let root_key = |f: &Finding| (f.first_seen.is_none(), f.first_seen.clone(), f.rule_id.clone());
|
||||
let root_key = |f: &Finding| (f.first_seen.is_none(), f.first_seen, f.rule_id.clone());
|
||||
|
||||
let mut chosen: Vec<Option<usize>> = vec![None; findings.len()];
|
||||
for (i, symptom) in findings.iter().enumerate() {
|
||||
@@ -560,10 +555,10 @@ fn collapse_findings(findings: &mut [Finding]) {
|
||||
continue;
|
||||
}
|
||||
let root = &findings[r];
|
||||
let qualifies = match (root.first_seen.as_ref(), symptom.first_seen.as_ref()) {
|
||||
let qualifies = match (root.first_seen, symptom.first_seen) {
|
||||
(Some(root_first), Some(symptom_first)) => {
|
||||
root_first <= symptom_first + SignedDuration::from_mins(5)
|
||||
&& root.last_seen.as_ref().unwrap_or(root_first) >= symptom_first - SignedDuration::from_mins(30)
|
||||
root_first <= symptom_first + Duration::minutes(5)
|
||||
&& root.last_seen.unwrap_or(root_first) >= symptom_first - Duration::minutes(30)
|
||||
}
|
||||
_ => true,
|
||||
};
|
||||
@@ -704,7 +699,7 @@ mod tests {
|
||||
|
||||
fn event(message: &str, level: Option<LogLevel>, ts: Option<&str>) -> LogEvent {
|
||||
LogEvent {
|
||||
timestamp: ts.map(|t| crate::time::parse_rfc3339(t).expect("ts")),
|
||||
timestamp: ts.map(|t| DateTime::parse_from_rfc3339(t).expect("ts")),
|
||||
level,
|
||||
target: Some("rustfs::server::http".to_string()),
|
||||
message: message.to_string(),
|
||||
@@ -820,8 +815,8 @@ mod tests {
|
||||
assert_eq!(gaps.len(), 1, "{:?}", report.timeline_anomalies);
|
||||
assert!(gaps[0].restart_evidence, "startup-fatal right after the gap: {:?}", gaps[0]);
|
||||
assert!(gaps[0].message.contains("重启证据"));
|
||||
assert_eq!(gaps[0].gap_start.as_ref().expect("start").to_rfc3339(), "2026-07-15T03:04:00+00:00");
|
||||
assert_eq!(gaps[0].gap_end.as_ref().expect("end").to_rfc3339(), "2026-07-15T03:44:00+00:00");
|
||||
assert_eq!(gaps[0].gap_start.expect("start").to_rfc3339(), "2026-07-15T03:04:00+00:00");
|
||||
assert_eq!(gaps[0].gap_end.expect("end").to_rfc3339(), "2026-07-15T03:44:00+00:00");
|
||||
|
||||
// Same hole but no startup finding afterwards: still a gap, no upgrade.
|
||||
let mut a = analyzer(AnalyzeOptions::default());
|
||||
@@ -1016,7 +1011,7 @@ mod tests {
|
||||
#[test]
|
||||
fn since_until_filters_timestamped_events_only() {
|
||||
let mut a = analyzer(AnalyzeOptions {
|
||||
since: Some(crate::time::parse_rfc3339("2026-07-15T03:00:00+08:00").expect("ts")),
|
||||
since: Some(DateTime::parse_from_rfc3339("2026-07-15T03:00:00+08:00").expect("ts")),
|
||||
..Default::default()
|
||||
});
|
||||
a.observe(event("Disk full", Some(LogLevel::Error), Some("2026-07-15T02:00:00+08:00")));
|
||||
|
||||
@@ -27,7 +27,6 @@ pub mod parse;
|
||||
mod redact;
|
||||
pub mod report;
|
||||
pub mod rules;
|
||||
pub(crate) mod time;
|
||||
|
||||
pub use analyze::{AnalysisReport, AnalyzeOptions, Analyzer, Summary, TimeBucket, TimelineAnomaly, UnmatchedCluster};
|
||||
pub use ingest::{IngestOptions, IngestReport, SkipReason, ingest_path, ingest_reader};
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
//! Unified event model shared by every stage of the analyzer.
|
||||
|
||||
use jiff::ZonedDateTime;
|
||||
use chrono::{DateTime, FixedOffset};
|
||||
use serde::{Deserialize, Serialize, Serializer};
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
@@ -102,7 +102,7 @@ pub enum EventKind {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct LogEvent {
|
||||
/// RFC-3339 timestamp with offset. None for panic/text lines that carry none.
|
||||
pub timestamp: Option<ZonedDateTime>,
|
||||
pub timestamp: Option<DateTime<FixedOffset>>,
|
||||
/// None when the line carries no recognizable level (text lines).
|
||||
pub level: Option<LogLevel>,
|
||||
/// tracing `target`, e.g. "rustfs::scanner::io". None for panic/text.
|
||||
|
||||
@@ -36,7 +36,7 @@ pub(crate) fn try_parse(line: &str, file: &Arc<str>, line_no: u64, node: &Option
|
||||
}
|
||||
|
||||
let timestamp = match map.remove("timestamp") {
|
||||
Some(Value::String(raw)) => match crate::time::parse_rfc3339(&raw) {
|
||||
Some(Value::String(raw)) => match chrono::DateTime::parse_from_rfc3339(&raw) {
|
||||
Ok(ts) => Some(ts),
|
||||
Err(_) => {
|
||||
map.insert("timestamp_raw".to_string(), Value::String(raw));
|
||||
|
||||
@@ -156,8 +156,8 @@ mod tests {
|
||||
assert_eq!(ev.level, Some(LogLevel::Error));
|
||||
assert_eq!(ev.target.as_deref(), Some("rustfs::scanner::io"));
|
||||
assert_eq!(ev.message, "Disk health check marked disk faulty");
|
||||
let ts = ev.timestamp.as_ref().expect("timestamp");
|
||||
assert_eq!(crate::time::offset_seconds(ts), 8 * 3600);
|
||||
let ts = ev.timestamp.expect("timestamp");
|
||||
assert_eq!(ts.offset().local_minus_utc(), 8 * 3600);
|
||||
assert_eq!(ev.field_str("reason"), Some("faulty_disk"));
|
||||
assert_eq!(ev.field_str("disk"), Some("/data/disk3"));
|
||||
assert!(ev.fields.contains_key("filename"));
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
|
||||
use super::{human_bytes, parse_percentage};
|
||||
use crate::analyze::AnalysisReport;
|
||||
use crate::time::Rfc3339;
|
||||
use std::io;
|
||||
|
||||
/// GFM parses a table row into cells before inline code, so a literal `|`
|
||||
@@ -34,7 +33,7 @@ pub(super) fn render(report: &AnalysisReport, w: &mut dyn io::Write) -> io::Resu
|
||||
writeln!(w)?;
|
||||
writeln!(w, "| 项 | 值 |")?;
|
||||
writeln!(w, "|---|---|")?;
|
||||
match report.summary.time_range.as_ref() {
|
||||
match report.summary.time_range {
|
||||
Some((first, last)) => writeln!(w, "| 时间范围 | {} → {} |", first.to_rfc3339(), last.to_rfc3339())?,
|
||||
None => writeln!(w, "| 时间范围 | (无带时间戳的事件) |")?,
|
||||
}
|
||||
@@ -105,7 +104,7 @@ pub(super) fn render(report: &AnalysisReport, w: &mut dyn io::Write) -> io::Resu
|
||||
finding.count
|
||||
)?;
|
||||
writeln!(w)?;
|
||||
if let (Some(first), Some(last)) = (finding.first_seen.as_ref(), finding.last_seen.as_ref()) {
|
||||
if let (Some(first), Some(last)) = (finding.first_seen, finding.last_seen) {
|
||||
writeln!(w, "- 首次: {} · 末次: {}", first.to_rfc3339(), last.to_rfc3339())?;
|
||||
}
|
||||
if !finding.caused.is_empty() {
|
||||
|
||||
@@ -74,13 +74,13 @@ mod tests {
|
||||
use crate::ingest::IngestReport;
|
||||
use crate::model::{EventKind, LogEvent, LogLevel, SourceRef};
|
||||
use crate::rules::{RuleEngine, seed_rule_set};
|
||||
use crate::time::parse_rfc3339;
|
||||
use chrono::DateTime;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn fixed_report() -> AnalysisReport {
|
||||
let mut analyzer = Analyzer::new(RuleEngine::new(seed_rule_set()), AnalyzeOptions::default());
|
||||
let mut ev = LogEvent {
|
||||
timestamp: Some(parse_rfc3339("2026-07-15T03:12:44+08:00").expect("ts")),
|
||||
timestamp: Some(DateTime::parse_from_rfc3339("2026-07-15T03:12:44+08:00").expect("ts")),
|
||||
level: Some(LogLevel::Error),
|
||||
target: Some("rustfs_ecstore".to_string() + "::erasure"),
|
||||
message: "erasure write quorum (required=8, achieved=5, failed=3)".to_string(),
|
||||
@@ -96,11 +96,11 @@ mod tests {
|
||||
.insert("required".to_string(), serde_json::Value::String("8".into()));
|
||||
analyzer.observe(ev.clone());
|
||||
analyzer.observe(LogEvent {
|
||||
timestamp: Some(parse_rfc3339("2026-07-15T03:20:44+08:00").expect("ts")),
|
||||
timestamp: Some(DateTime::parse_from_rfc3339("2026-07-15T03:20:44+08:00").expect("ts")),
|
||||
..ev
|
||||
});
|
||||
analyzer.observe(LogEvent {
|
||||
timestamp: Some(parse_rfc3339("2026-07-15T03:25:00+08:00").expect("ts")),
|
||||
timestamp: Some(DateTime::parse_from_rfc3339("2026-07-15T03:25:00+08:00").expect("ts")),
|
||||
level: Some(LogLevel::Error),
|
||||
target: None,
|
||||
message: "failed to sync /data/x9 after 5 retries".to_string(),
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
use super::{human_bytes, human_duration, parse_percentage};
|
||||
use crate::analyze::AnalysisReport;
|
||||
use crate::rules::Finding;
|
||||
use crate::time::Rfc3339;
|
||||
use std::io;
|
||||
|
||||
const HEAVY_RULE: &str = "════════════════════════════════════════";
|
||||
@@ -27,9 +26,9 @@ pub(super) fn render(report: &AnalysisReport, w: &mut dyn io::Write) -> io::Resu
|
||||
writeln!(w, "RustFS 日志诊断报告")?;
|
||||
writeln!(w, "{HEAVY_RULE}")?;
|
||||
|
||||
match report.summary.time_range.as_ref() {
|
||||
match report.summary.time_range {
|
||||
Some((first, last)) => {
|
||||
let span = human_duration(last.duration_since(first).as_secs());
|
||||
let span = human_duration(last.signed_duration_since(first).num_seconds());
|
||||
writeln!(w, "时间范围 {} → {} ({span})", first.to_rfc3339(), last.to_rfc3339())?;
|
||||
}
|
||||
None => writeln!(w, "时间范围 (无带时间戳的事件)")?,
|
||||
@@ -158,7 +157,7 @@ fn render_finding(finding: &Finding, counts: &std::collections::HashMap<&str, u6
|
||||
finding.count,
|
||||
nodes
|
||||
)?;
|
||||
if let (Some(first), Some(last)) = (finding.first_seen.as_ref(), finding.last_seen.as_ref()) {
|
||||
if let (Some(first), Some(last)) = (finding.first_seen, finding.last_seen) {
|
||||
writeln!(w, " 首次 {} 末次 {}", first.to_rfc3339(), last.to_rfc3339())?;
|
||||
}
|
||||
if !finding.caused.is_empty() {
|
||||
@@ -181,10 +180,7 @@ fn render_finding(finding: &Finding, counts: &std::collections::HashMap<&str, u6
|
||||
writeln!(w, " 证据: {field} = {}{overflow}", shown.join(", "))?;
|
||||
}
|
||||
for sample in &finding.samples {
|
||||
let ts = sample
|
||||
.timestamp
|
||||
.as_ref()
|
||||
.map_or_else(|| "(无时间戳)".to_string(), |t| t.to_rfc3339());
|
||||
let ts = sample.timestamp.map_or_else(|| "(无时间戳)".to_string(), |t| t.to_rfc3339());
|
||||
let node = sample.node.as_deref().unwrap_or("-");
|
||||
writeln!(w, " 样本: {ts} {node} {}:{}", sample.source.file, sample.source.line)?;
|
||||
writeln!(w, " {}", sample.message)?;
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
use super::model::{Rule, Severity};
|
||||
use crate::model::LogEvent;
|
||||
use jiff::ZonedDateTime;
|
||||
use chrono::{DateTime, FixedOffset};
|
||||
use serde::Serialize;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
@@ -44,8 +44,8 @@ pub struct Finding {
|
||||
pub suggestion: String,
|
||||
pub count: u64,
|
||||
/// Earliest / latest timestamps among hits that carry one.
|
||||
pub first_seen: Option<ZonedDateTime>,
|
||||
pub last_seen: Option<ZonedDateTime>,
|
||||
pub first_seen: Option<DateTime<FixedOffset>>,
|
||||
pub last_seen: Option<DateTime<FixedOffset>>,
|
||||
/// Node labels of hits; label-less events are recorded as "-".
|
||||
pub nodes: BTreeSet<String>,
|
||||
/// The earliest `max_samples` hit events (timestamp-less ones sort last).
|
||||
@@ -79,28 +79,28 @@ pub struct FindingsCollector {
|
||||
/// Sort key for sample selection: earliest first, timestamp-less last,
|
||||
/// (file, line) as the deterministic tiebreaker — line numbers alone are only
|
||||
/// unique within one file, so ties across inputs would be order-dependent.
|
||||
fn sample_key(event: &LogEvent) -> (bool, Option<ZonedDateTime>, &str, u64) {
|
||||
(event.timestamp.is_none(), event.timestamp.clone(), &event.source.file, event.source.line)
|
||||
fn sample_key(event: &LogEvent) -> (bool, Option<DateTime<FixedOffset>>, &str, u64) {
|
||||
(event.timestamp.is_none(), event.timestamp, &event.source.file, event.source.line)
|
||||
}
|
||||
|
||||
/// `ZonedDateTime` orders by instant, so two equal instants carrying
|
||||
/// `DateTime<FixedOffset>` orders by instant, so two equal instants carrying
|
||||
/// different offsets compare equal and `min`/`max` keep whichever arrived
|
||||
/// first — an input-order dependency that then leaks into the serialized
|
||||
/// RFC3339 offset. Break the tie on the offset itself to stay deterministic.
|
||||
fn earliest(a: ZonedDateTime, b: ZonedDateTime) -> ZonedDateTime {
|
||||
fn earliest(a: DateTime<FixedOffset>, b: DateTime<FixedOffset>) -> DateTime<FixedOffset> {
|
||||
match a.cmp(&b) {
|
||||
std::cmp::Ordering::Less => a,
|
||||
std::cmp::Ordering::Greater => b,
|
||||
std::cmp::Ordering::Equal if crate::time::offset_seconds(&a) <= crate::time::offset_seconds(&b) => a,
|
||||
std::cmp::Ordering::Equal if a.offset().local_minus_utc() <= b.offset().local_minus_utc() => a,
|
||||
std::cmp::Ordering::Equal => b,
|
||||
}
|
||||
}
|
||||
|
||||
fn latest(a: ZonedDateTime, b: ZonedDateTime) -> ZonedDateTime {
|
||||
fn latest(a: DateTime<FixedOffset>, b: DateTime<FixedOffset>) -> DateTime<FixedOffset> {
|
||||
match a.cmp(&b) {
|
||||
std::cmp::Ordering::Greater => a,
|
||||
std::cmp::Ordering::Less => b,
|
||||
std::cmp::Ordering::Equal if crate::time::offset_seconds(&a) >= crate::time::offset_seconds(&b) => a,
|
||||
std::cmp::Ordering::Equal if a.offset().local_minus_utc() >= b.offset().local_minus_utc() => a,
|
||||
std::cmp::Ordering::Equal => b,
|
||||
}
|
||||
}
|
||||
@@ -140,9 +140,9 @@ impl FindingsCollector {
|
||||
let finding = &mut acc.finding;
|
||||
finding.count += 1;
|
||||
|
||||
if let Some(ts) = event.timestamp.clone() {
|
||||
finding.first_seen = Some(finding.first_seen.take().map_or(ts.clone(), |cur| earliest(cur, ts.clone())));
|
||||
finding.last_seen = Some(finding.last_seen.take().map_or(ts.clone(), |cur| latest(cur, ts)));
|
||||
if let Some(ts) = event.timestamp {
|
||||
finding.first_seen = Some(finding.first_seen.map_or(ts, |cur| earliest(cur, ts)));
|
||||
finding.last_seen = Some(finding.last_seen.map_or(ts, |cur| latest(cur, ts)));
|
||||
}
|
||||
|
||||
finding.nodes.insert(event.node.as_deref().unwrap_or("-").to_string());
|
||||
|
||||
@@ -40,7 +40,6 @@ pub fn seed_rule_set() -> RuleSet {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::model::{EventKind, LogEvent, LogLevel, SourceRef};
|
||||
use crate::time::Rfc3339;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn event(message: &str, level: Option<LogLevel>, line: u64) -> LogEvent {
|
||||
@@ -61,7 +60,7 @@ mod tests {
|
||||
|
||||
fn event_at(message: &str, ts: &str, line: u64) -> LogEvent {
|
||||
LogEvent {
|
||||
timestamp: Some(crate::time::parse_rfc3339(ts).expect("ts")),
|
||||
timestamp: Some(chrono::DateTime::parse_from_rfc3339(ts).expect("ts")),
|
||||
..event(message, Some(LogLevel::Error), line)
|
||||
}
|
||||
}
|
||||
@@ -225,8 +224,8 @@ mod tests {
|
||||
};
|
||||
let f = &findings[0];
|
||||
assert_eq!(f.count, 5);
|
||||
assert_eq!(f.first_seen.as_ref().expect("first").to_rfc3339(), "2026-07-15T01:00:00+08:00");
|
||||
assert_eq!(f.last_seen.as_ref().expect("last").to_rfc3339(), "2026-07-15T04:00:00+08:00");
|
||||
assert_eq!(f.first_seen.expect("first").to_rfc3339(), "2026-07-15T01:00:00+08:00");
|
||||
assert_eq!(f.last_seen.expect("last").to_rfc3339(), "2026-07-15T04:00:00+08:00");
|
||||
let sample_messages: Vec<_> = f.samples.iter().map(|s| s.message.as_str()).collect();
|
||||
assert_eq!(sample_messages, vec!["quorum e2", "quorum e3", "quorum e1"]);
|
||||
}
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
use jiff::{Timestamp, ZonedDateTime, tz::TimeZone};
|
||||
|
||||
pub(crate) trait Rfc3339 {
|
||||
fn to_rfc3339(&self) -> String;
|
||||
fn format(&self, pattern: &str) -> String;
|
||||
}
|
||||
|
||||
impl Rfc3339 for ZonedDateTime {
|
||||
fn to_rfc3339(&self) -> String {
|
||||
format_rfc3339(self)
|
||||
}
|
||||
|
||||
fn format(&self, pattern: &str) -> String {
|
||||
self.strftime(pattern).to_string()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_rfc3339(value: &str) -> Result<ZonedDateTime, jiff::Error> {
|
||||
if let Some(prefix) = value.strip_suffix('Z') {
|
||||
let normalized = format!("{prefix}+00:00");
|
||||
return ZonedDateTime::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", normalized);
|
||||
}
|
||||
ZonedDateTime::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", value)
|
||||
}
|
||||
|
||||
pub(crate) fn format_rfc3339(value: &ZonedDateTime) -> String {
|
||||
value.strftime("%Y-%m-%dT%H:%M:%S%.f%:z").to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn utc_from_unix(seconds: i64) -> ZonedDateTime {
|
||||
Timestamp::new(seconds, 0)
|
||||
.expect("unix timestamp must fit jiff")
|
||||
.to_zoned_datetime(TimeZone::UTC)
|
||||
}
|
||||
|
||||
pub(crate) fn offset_seconds(value: &ZonedDateTime) -> i32 {
|
||||
value.offset().seconds()
|
||||
}
|
||||
@@ -30,13 +30,12 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Kept as no-op compatibility features for dependents that forward the
|
||||
# workspace profiling feature set to every crate.
|
||||
hotpath = []
|
||||
hotpath-alloc = []
|
||||
hotpath-cpu = []
|
||||
hotpath = ["hotpath/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
humantime.workspace = true
|
||||
http.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
|
||||
@@ -74,6 +74,7 @@ jiff = { workspace = true, features = ["serde"] }
|
||||
percent-encoding = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
rustc-hash = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
starshard = { workspace = true, features = ["rayon", "async", "serde"] }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
@@ -13,9 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{
|
||||
BucketNotificationConfig, NotificationError, config_manager::notify_configuration_hint,
|
||||
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
|
||||
rules::ParseConfigError,
|
||||
BucketNotificationConfig, NotificationError, config_manager::notify_configuration_hint, notifier::EventNotifier,
|
||||
rule_engine::NotifyRuleEngine, rules::ParseConfigError,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
use std::sync::Arc;
|
||||
@@ -30,26 +29,14 @@ const EVENT_NOTIFY_BUCKET_CONFIG_LOADED: &str = "notify_bucket_config_loaded";
|
||||
pub struct NotifyBucketConfigManager {
|
||||
notifier: Arc<EventNotifier>,
|
||||
rule_engine: NotifyRuleEngine,
|
||||
subscriber_view: Arc<NotificationSystemSubscriberView>,
|
||||
}
|
||||
|
||||
impl NotifyBucketConfigManager {
|
||||
pub fn new(
|
||||
notifier: Arc<EventNotifier>,
|
||||
rule_engine: NotifyRuleEngine,
|
||||
subscriber_view: Arc<NotificationSystemSubscriberView>,
|
||||
) -> Self {
|
||||
Self {
|
||||
notifier,
|
||||
rule_engine,
|
||||
subscriber_view,
|
||||
}
|
||||
pub fn new(notifier: Arc<EventNotifier>, rule_engine: NotifyRuleEngine) -> Self {
|
||||
Self { notifier, rule_engine }
|
||||
}
|
||||
|
||||
pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
|
||||
if !self.subscriber_view.has_subscriber(bucket, event) {
|
||||
return false;
|
||||
}
|
||||
self.rule_engine.has_subscriber(bucket, event).await
|
||||
}
|
||||
|
||||
@@ -105,7 +92,6 @@ impl NotifyBucketConfigManager {
|
||||
|
||||
/// Publishes an already validated `cfg` to the runtime rule state.
|
||||
pub async fn apply_bucket_notification_config(&self, bucket: &str, cfg: &BucketNotificationConfig) {
|
||||
self.subscriber_view.apply_bucket_config(bucket, cfg);
|
||||
self.rule_engine.set_bucket_rules(bucket, cfg.get_rules_map().clone()).await;
|
||||
info!(
|
||||
event = EVENT_NOTIFY_BUCKET_CONFIG_LOADED,
|
||||
@@ -128,8 +114,16 @@ impl NotifyBucketConfigManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn publish_bucket_notification_config(&self, bucket: &str, cfg: Option<(uuid::Uuid, &BucketNotificationConfig)>) {
|
||||
if let Some((incarnation_id, cfg)) = cfg {
|
||||
self.rule_engine
|
||||
.publish_bucket_rules(bucket, Some(incarnation_id), cfg.get_rules_map().clone());
|
||||
} else {
|
||||
self.rule_engine.withdraw_bucket_rules(bucket);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn remove_bucket_notification_config(&self, bucket: &str) {
|
||||
self.subscriber_view.clear_bucket(bucket);
|
||||
self.rule_engine.clear_bucket_rules(bucket).await;
|
||||
}
|
||||
}
|
||||
@@ -138,8 +132,8 @@ impl NotifyBucketConfigManager {
|
||||
mod tests {
|
||||
use super::NotifyBucketConfigManager;
|
||||
use crate::{
|
||||
BucketNotificationConfig, NotificationError, integration::NotificationMetrics,
|
||||
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
|
||||
BucketNotificationConfig, NotificationError, integration::NotificationMetrics, notifier::EventNotifier,
|
||||
rule_engine::NotifyRuleEngine,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
@@ -149,8 +143,7 @@ mod tests {
|
||||
let metrics = Arc::new(NotificationMetrics::new());
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
let notifier = Arc::new(EventNotifier::new(metrics, rule_engine.clone()));
|
||||
let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
|
||||
NotifyBucketConfigManager::new(notifier, rule_engine, subscriber_view)
|
||||
NotifyBucketConfigManager::new(notifier, rule_engine)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -166,11 +159,11 @@ mod tests {
|
||||
let mut cfg = BucketNotificationConfig::new("us-east-1");
|
||||
cfg.add_rule(&[EventName::ObjectCreatedPut], "*".to_string(), target_id);
|
||||
|
||||
manager.subscriber_view.apply_bucket_config("bucket", &cfg);
|
||||
assert!(manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
|
||||
manager.apply_bucket_notification_config("bucket", &cfg).await;
|
||||
assert!(manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
|
||||
manager.remove_bucket_notification_config("bucket").await;
|
||||
assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
|
||||
assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -186,7 +179,6 @@ mod tests {
|
||||
.expect_err("validation must fail when the runtime has no notify targets");
|
||||
assert!(matches!(err, NotificationError::Configuration(_)));
|
||||
assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::rules::TargetIdSet;
|
||||
use hashbrown::HashMap;
|
||||
use jiff::{Timestamp, tz::TimeZone};
|
||||
use rustfs_s3_ops::is_object_removed_event;
|
||||
@@ -170,6 +171,10 @@ pub struct Event {
|
||||
pub glacier_event_data: Option<GlacierEventData>,
|
||||
/// Information about the source of the event
|
||||
pub source: Source,
|
||||
#[serde(skip)]
|
||||
pub(crate) bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
#[serde(skip)]
|
||||
pub(crate) bucket_removed_targets: Option<TargetIdSet>,
|
||||
}
|
||||
|
||||
impl Event {
|
||||
@@ -195,6 +200,8 @@ impl Event {
|
||||
user_metadata.insert("x-request-time".to_string(), format_timestamp_rfc3339_millis(&Timestamp::now()));
|
||||
|
||||
Event {
|
||||
bucket_incarnation_id: None,
|
||||
bucket_removed_targets: None,
|
||||
event_version: event_schema_version(event_name).to_string(),
|
||||
event_source: "rustfs:s3".to_string(),
|
||||
aws_region: "us-east-1".to_string(),
|
||||
@@ -320,6 +327,8 @@ impl Event {
|
||||
};
|
||||
|
||||
Self {
|
||||
bucket_incarnation_id: args.bucket_incarnation_id,
|
||||
bucket_removed_targets: args.bucket_removed_targets,
|
||||
event_version: event_schema_version(args.event_name).to_string(),
|
||||
event_source: "rustfs:s3".to_string(),
|
||||
aws_region: args.req_params.get("region").cloned().unwrap_or_default(),
|
||||
@@ -359,7 +368,7 @@ where
|
||||
}
|
||||
|
||||
fn format_timestamp_rfc3339_millis(value: &Timestamp) -> String {
|
||||
let utc = value.to_zoned_datetime(TimeZone::UTC);
|
||||
let utc = value.to_zoned(TimeZone::UTC);
|
||||
format!(
|
||||
"{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:03}Z",
|
||||
utc.year(),
|
||||
@@ -387,6 +396,10 @@ fn initialize_response_elements(elements: &mut HashMap<String, String>, keys: &[
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EventArgs {
|
||||
/// Bucket identity established by the producing operation.
|
||||
pub bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
/// Delivery targets captured before a successful bucket removal withdraws its rules.
|
||||
pub bucket_removed_targets: Option<TargetIdSet>,
|
||||
pub event_name: EventName,
|
||||
pub bucket_name: String,
|
||||
pub object: NotifyObjectInfo,
|
||||
@@ -436,6 +449,8 @@ impl EventArgs {
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct EventArgsBuilder {
|
||||
bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
bucket_removed_targets: Option<TargetIdSet>,
|
||||
event_name: EventName,
|
||||
bucket_name: String,
|
||||
object: NotifyObjectInfo,
|
||||
@@ -458,6 +473,18 @@ impl EventArgsBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
/// Carries the operation's confirmed bucket identity to event dispatch.
|
||||
pub fn bucket_incarnation_id(mut self, incarnation_id: Option<uuid::Uuid>) -> Self {
|
||||
self.bucket_incarnation_id = incarnation_id;
|
||||
self
|
||||
}
|
||||
|
||||
/// Uses the subscription targets captured before deletion.
|
||||
pub fn bucket_removed_targets(mut self, targets: TargetIdSet) -> Self {
|
||||
self.bucket_removed_targets = Some(targets);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the event name.
|
||||
pub fn event_name(mut self, event_name: EventName) -> Self {
|
||||
self.event_name = event_name;
|
||||
@@ -529,6 +556,8 @@ impl EventArgsBuilder {
|
||||
/// This method consumes the builder and returns the constructed `EventArgs`.
|
||||
pub fn build(self) -> EventArgs {
|
||||
EventArgs {
|
||||
bucket_incarnation_id: self.bucket_incarnation_id,
|
||||
bucket_removed_targets: self.bucket_removed_targets,
|
||||
event_name: self.event_name,
|
||||
bucket_name: self.bucket_name,
|
||||
object: self.object,
|
||||
@@ -823,6 +852,8 @@ mod event_args_tests {
|
||||
req_params.insert((*k).to_string(), (*v).to_string());
|
||||
}
|
||||
EventArgs {
|
||||
bucket_incarnation_id: None,
|
||||
bucket_removed_targets: None,
|
||||
event_name: EventName::ObjectRemovedDelete,
|
||||
bucket_name: "b".to_string(),
|
||||
object: ObjectInfo::default(),
|
||||
|
||||
@@ -239,7 +239,8 @@ pub mod notifier_global {
|
||||
.await
|
||||
}
|
||||
|
||||
fn bucket_notification_config_for_rules(
|
||||
/// Compiles event rules into the bucket configuration used by validation and publication.
|
||||
pub fn bucket_notification_config_for_rules(
|
||||
region: &str,
|
||||
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
|
||||
) -> BucketNotificationConfig {
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::notification_system_subscriber::NotificationSystemSubscriberView;
|
||||
use crate::notifier::{EventNotifier, TargetList};
|
||||
use crate::services::NotifyServices;
|
||||
use crate::{
|
||||
@@ -182,7 +181,6 @@ impl NotificationSystem {
|
||||
rustfs_utils::get_env_usize(ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY);
|
||||
let (live_event_sender, _) = broadcast::channel(1024);
|
||||
let metrics = Arc::new(NotificationMetrics::new());
|
||||
let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
|
||||
let target_list = notifier.target_list();
|
||||
@@ -200,7 +198,6 @@ impl NotificationSystem {
|
||||
stream_cancellers,
|
||||
concurrency_limiter,
|
||||
metrics,
|
||||
subscriber_view,
|
||||
live_event_sender,
|
||||
live_event_history,
|
||||
);
|
||||
@@ -413,6 +410,13 @@ impl NotificationSystem {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Applies the current committed bucket rules while the caller holds the metadata publication fence.
|
||||
pub fn publish_bucket_notification_config(&self, bucket: &str, cfg: Option<(uuid::Uuid, &BucketNotificationConfig)>) {
|
||||
self.services
|
||||
.bucket_config_manager
|
||||
.publish_bucket_notification_config(bucket, cfg);
|
||||
}
|
||||
|
||||
/// Sends an event
|
||||
pub async fn send_event(&self, event: Arc<Event>) {
|
||||
self.services.pipeline.send_event(event).await;
|
||||
|
||||
@@ -26,7 +26,6 @@ pub mod factory;
|
||||
mod global;
|
||||
pub mod integration;
|
||||
mod lifecycle;
|
||||
mod notification_system_subscriber;
|
||||
pub mod notifier;
|
||||
mod pipeline;
|
||||
pub mod registry;
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::BucketNotificationConfig;
|
||||
use crate::rules::{BucketRulesSnapshot, DynRulesContainer, SubscriberIndex};
|
||||
use rustfs_s3_types::EventName;
|
||||
|
||||
/// NotificationSystemSubscriberView - Provides an interface to manage and query
|
||||
/// the subscription status of buckets in the notification system.
|
||||
#[derive(Debug)]
|
||||
pub struct NotificationSystemSubscriberView {
|
||||
index: SubscriberIndex,
|
||||
}
|
||||
|
||||
impl NotificationSystemSubscriberView {
|
||||
/// Creates a new NotificationSystemSubscriberView with an empty SubscriberIndex.
|
||||
///
|
||||
/// Returns a new instance of NotificationSystemSubscriberView.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
index: SubscriberIndex::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks if a bucket has any subscribers for a specific event.
|
||||
/// This is a quick check using the event mask in the snapshot.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `bucket` - The name of the bucket to check.
|
||||
/// * `event` - The event name to check for subscriptions.
|
||||
///
|
||||
/// Returns `true` if there are subscribers for the event, `false` otherwise.
|
||||
#[inline]
|
||||
pub fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
|
||||
self.index.has_subscriber(bucket, event)
|
||||
}
|
||||
|
||||
/// Builds and atomically replaces a bucket's subscription snapshot from the configuration.
|
||||
///
|
||||
/// Core principle: masks and rules are calculated and stored together in the same update.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `bucket` - The name of the bucket to update.
|
||||
/// * `cfg` - The bucket notification configuration to compile into a snapshot.
|
||||
pub fn apply_bucket_config(&self, bucket: &str, cfg: &BucketNotificationConfig) {
|
||||
// *It is recommended to merge compile into one function to ensure the same origin.
|
||||
let snapshot: BucketRulesSnapshot<DynRulesContainer> = cfg.compile_snapshot();
|
||||
|
||||
// *debug to prevent inconsistencies from being introduced when modifying the compile logic in the future.
|
||||
snapshot.debug_assert_mask_consistent();
|
||||
|
||||
self.index.store_snapshot(bucket, snapshot);
|
||||
}
|
||||
|
||||
/// Clears a bucket's subscription snapshot.
|
||||
///
|
||||
/// #Arguments
|
||||
/// * `bucket` - The name of the bucket to clear.
|
||||
#[inline]
|
||||
pub fn clear_bucket(&self, bucket: &str) {
|
||||
self.index.clear_bucket(bucket);
|
||||
}
|
||||
}
|
||||
@@ -13,8 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::NotificationError;
|
||||
use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
|
||||
use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine, rules::TargetIdSet};
|
||||
use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::Target;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use rustfs_targets::target::EntityTarget;
|
||||
@@ -235,6 +236,12 @@ impl EventNotifier {
|
||||
);
|
||||
}
|
||||
|
||||
/// Captures the deleted bucket's final delivery targets before its rules are withdrawn.
|
||||
pub fn bucket_removed_targets(&self, bucket: &str, incarnation_id: uuid::Uuid) -> TargetIdSet {
|
||||
self.rule_engine
|
||||
.match_bucket_targets(bucket, Some(incarnation_id), EventName::BucketRemoved, "")
|
||||
}
|
||||
|
||||
/// Sends an event to the appropriate targets based on the bucket rules
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -245,7 +252,25 @@ impl EventNotifier {
|
||||
let object_key = &event.s3.object.key;
|
||||
let event_name = event.event_name;
|
||||
|
||||
let target_ids = self.rule_engine.match_targets(bucket_name, event_name, object_key).await;
|
||||
let target_ids = match &event.bucket_removed_targets {
|
||||
Some(targets) if event_name == EventName::BucketRemoved => targets.clone(),
|
||||
_ => {
|
||||
let match_targets = || {
|
||||
self.rule_engine
|
||||
.match_bucket_targets(bucket_name, event.bucket_incarnation_id, event_name, object_key)
|
||||
};
|
||||
let targets = match_targets();
|
||||
if targets.is_empty()
|
||||
&& let Some(id) = event.bucket_incarnation_id
|
||||
&& self.rule_engine.request_incarnation_refresh(bucket_name, id)
|
||||
&& crate::storage_api::refresh_notify_bucket_metadata(bucket_name).await
|
||||
{
|
||||
match_targets()
|
||||
} else {
|
||||
targets
|
||||
}
|
||||
}
|
||||
};
|
||||
if target_ids.is_empty() {
|
||||
debug!(
|
||||
event = EVENT_NOTIFY_DISPATCH_SKIPPED,
|
||||
@@ -1110,6 +1135,46 @@ mod tests {
|
||||
assert_eq!(disabled_target.save_call_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_removal_targets_survive_withdrawal_without_routing_recreated_objects() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), engine.clone());
|
||||
let target = MockTarget::new("removed-bucket", "webhook");
|
||||
let old_id = uuid::Uuid::new_v4();
|
||||
let new_id = uuid::Uuid::new_v4();
|
||||
let mut rules = RulesMap::new();
|
||||
rules.add_rule_config(
|
||||
&[EventName::BucketRemoved, EventName::ObjectCreatedPut],
|
||||
"*".to_owned(),
|
||||
target.target_id(),
|
||||
);
|
||||
engine.publish_bucket_rules("bucket", Some(old_id), rules);
|
||||
notifier
|
||||
.target_list()
|
||||
.write()
|
||||
.await
|
||||
.add(Arc::new(target.clone()))
|
||||
.expect("register target");
|
||||
|
||||
let targets = notifier.bucket_removed_targets("bucket", old_id);
|
||||
assert!(targets.contains(&target.target_id()));
|
||||
engine.withdraw_bucket_rules("bucket");
|
||||
|
||||
let mut removed = Event::new_test_event("bucket", "", EventName::BucketRemoved);
|
||||
removed.bucket_incarnation_id = Some(old_id);
|
||||
removed.bucket_removed_targets = Some(targets);
|
||||
let encoded = serde_json::to_value(&removed).expect("serialize webhook event");
|
||||
assert!(encoded.get("bucketIncarnationId").is_none());
|
||||
assert!(encoded.get("bucketRemovedTargets").is_none());
|
||||
notifier.send(Arc::new(removed)).await;
|
||||
assert_eq!(target.save_call_count(), 1, "the deleted bucket earns its final removal event");
|
||||
|
||||
let mut created = Event::new_test_event("bucket", "fresh.txt", EventName::ObjectCreatedPut);
|
||||
created.bucket_incarnation_id = Some(new_id);
|
||||
notifier.send(Arc::new(created)).await;
|
||||
assert_eq!(target.save_call_count(), 1, "the new bucket has no subscription");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_event_respects_prefix_suffix_filters() {
|
||||
let rule_engine = NotifyRuleEngine::new();
|
||||
|
||||
@@ -16,9 +16,13 @@ use crate::rules::{RulesMap, TargetIdSet};
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode};
|
||||
use std::sync::Arc;
|
||||
use starshard::{DEFAULT_SHARDS, ShardedHashMap, SnapshotMode};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
|
||||
const LOG_COMPONENT_NOTIFY: &str = "notify";
|
||||
const LOG_SUBSYSTEM_RULE_ENGINE: &str = "rule_engine";
|
||||
@@ -33,34 +37,56 @@ fn decoded_object_key_for_matching(object_key: &str) -> Option<String> {
|
||||
(decoded != object_key).then(|| decoded.into_owned())
|
||||
}
|
||||
|
||||
struct BucketRules {
|
||||
incarnation_id: Option<Uuid>,
|
||||
rules: RulesMap,
|
||||
refresh_requested: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NotifyRuleEngine {
|
||||
bucket_rules_map: Arc<AsyncShardedHashMap<String, RulesMap, rustc_hash::FxBuildHasher>>,
|
||||
bucket_rules_map: Arc<ShardedHashMap<String, Arc<BucketRules>, rustc_hash::FxBuildHasher>>,
|
||||
}
|
||||
|
||||
impl NotifyRuleEngine {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
bucket_rules_map: Arc::new(AsyncShardedHashMap::with_snapshot_mode(DEFAULT_SHARDS, SnapshotMode::Cached)),
|
||||
bucket_rules_map: Arc::new(ShardedHashMap::with_snapshot_mode(DEFAULT_SHARDS, SnapshotMode::Cached)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool {
|
||||
let items = self.bucket_rules_map.iter().await;
|
||||
for (_bucket, rules_map) in items {
|
||||
if rules_map.contains_target_id(target_id) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
self.bucket_rules_map
|
||||
.iter()
|
||||
.any(|(_, snapshot)| snapshot.rules.contains_target_id(target_id))
|
||||
}
|
||||
|
||||
pub async fn set_bucket_rules(&self, bucket: &str, rules_map: RulesMap) {
|
||||
self.publish_bucket_rules(bucket, None, rules_map);
|
||||
}
|
||||
|
||||
/// Atomically publishes the rules and the identity used by event dispatch.
|
||||
pub(crate) fn publish_bucket_rules(&self, bucket: &str, incarnation_id: Option<Uuid>, rules_map: RulesMap) {
|
||||
let event_count = rules_map.iter_events().count();
|
||||
let bucket_key = bucket.to_owned();
|
||||
if rules_map.is_empty() {
|
||||
self.bucket_rules_map.remove(&bucket.to_string()).await;
|
||||
self.bucket_rules_map.remove(&bucket_key);
|
||||
} else {
|
||||
self.bucket_rules_map.insert(bucket.to_string(), rules_map).await;
|
||||
// A reload that confirms the current incarnation must not re-arm
|
||||
// the refresh triggered by delayed events from an older bucket.
|
||||
let refresh_requested = self
|
||||
.bucket_rules_map
|
||||
.get(&bucket_key)
|
||||
.filter(|current| current.incarnation_id == incarnation_id)
|
||||
.map_or_else(|| Arc::new(AtomicBool::new(false)), |current| current.refresh_requested.clone());
|
||||
self.bucket_rules_map.insert(
|
||||
bucket_key,
|
||||
Arc::new(BucketRules {
|
||||
incarnation_id,
|
||||
rules: rules_map,
|
||||
refresh_requested,
|
||||
}),
|
||||
);
|
||||
}
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RULES_UPDATED,
|
||||
@@ -74,11 +100,17 @@ impl NotifyRuleEngine {
|
||||
}
|
||||
|
||||
pub async fn get_bucket_rules(&self, bucket: &str) -> Option<RulesMap> {
|
||||
self.bucket_rules_map.get(&bucket.to_string()).await
|
||||
self.bucket_rules_map
|
||||
.get(&bucket.to_string())
|
||||
.map(|snapshot| snapshot.rules.clone())
|
||||
}
|
||||
|
||||
pub async fn clear_bucket_rules(&self, bucket: &str) {
|
||||
if self.bucket_rules_map.remove(&bucket.to_string()).await.is_some() {
|
||||
self.withdraw_bucket_rules(bucket);
|
||||
}
|
||||
|
||||
pub(crate) fn withdraw_bucket_rules(&self, bucket: &str) {
|
||||
if self.bucket_rules_map.remove(&bucket.to_string()).is_some() {
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RULES_UPDATED,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
@@ -91,21 +123,43 @@ impl NotifyRuleEngine {
|
||||
}
|
||||
|
||||
pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
|
||||
self.get_bucket_rules(bucket)
|
||||
.await
|
||||
.is_some_and(|rules_map| rules_map.has_subscriber(event))
|
||||
self.bucket_rules_map
|
||||
.get(&bucket.to_string())
|
||||
.is_some_and(|snapshot| snapshot.rules.has_subscriber(event))
|
||||
}
|
||||
|
||||
pub async fn match_targets(&self, bucket: &str, event_name: EventName, object_key: &str) -> TargetIdSet {
|
||||
self.get_bucket_rules(bucket)
|
||||
.await
|
||||
.map_or_else(TargetIdSet::new, |rules_map| {
|
||||
let mut target_ids = rules_map.match_rules(event_name, object_key);
|
||||
if let Some(decoded_key) = decoded_object_key_for_matching(object_key) {
|
||||
target_ids.extend(rules_map.match_rules(event_name, &decoded_key));
|
||||
}
|
||||
target_ids
|
||||
})
|
||||
self.match_bucket_targets(bucket, None, event_name, object_key)
|
||||
}
|
||||
|
||||
/// Allow one targeted reload per installed bucket incarnation. The ordinary
|
||||
/// metadata refresh loop can still recover a failed or interrupted reload.
|
||||
pub(crate) fn request_incarnation_refresh(&self, bucket: &str, incarnation_id: Uuid) -> bool {
|
||||
self.bucket_rules_map.get(&bucket.to_string()).is_some_and(|snapshot| {
|
||||
snapshot.incarnation_id.is_some_and(|current| current != incarnation_id)
|
||||
&& !snapshot.refresh_requested.swap(true, Ordering::Relaxed)
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn match_bucket_targets(
|
||||
&self,
|
||||
bucket: &str,
|
||||
incarnation_id: Option<Uuid>,
|
||||
event_name: EventName,
|
||||
object_key: &str,
|
||||
) -> TargetIdSet {
|
||||
let Some(snapshot) = self
|
||||
.bucket_rules_map
|
||||
.get(&bucket.to_string())
|
||||
.filter(|snapshot| snapshot.incarnation_id == incarnation_id)
|
||||
else {
|
||||
return TargetIdSet::new();
|
||||
};
|
||||
let mut targets = snapshot.rules.match_rules(event_name, object_key);
|
||||
if let Some(decoded_key) = decoded_object_key_for_matching(object_key) {
|
||||
targets.extend(snapshot.rules.match_rules(event_name, &decoded_key));
|
||||
}
|
||||
targets
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,6 +176,35 @@ mod tests {
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
|
||||
#[test]
|
||||
fn bucket_rules_route_events_only_with_the_matching_incarnation() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let old_id = uuid::Uuid::new_v4();
|
||||
let new_id = uuid::Uuid::new_v4();
|
||||
let old_target = TargetID::new("old".to_string(), "webhook".to_string());
|
||||
let new_target = TargetID::new("new".to_string(), "webhook".to_string());
|
||||
let rules = |target: TargetID| {
|
||||
let mut rules = RulesMap::new();
|
||||
rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_owned(), target);
|
||||
rules
|
||||
};
|
||||
let matched = |id| engine.match_bucket_targets("bucket", id, EventName::ObjectCreatedPut, "object");
|
||||
|
||||
engine.publish_bucket_rules("bucket", Some(old_id), rules(old_target.clone()));
|
||||
assert!(matched(Some(old_id)).contains(&old_target));
|
||||
assert!(matched(Some(new_id)).is_empty(), "a peer may still have the old rules");
|
||||
assert!(engine.request_incarnation_refresh("bucket", new_id));
|
||||
assert!(!engine.request_incarnation_refresh("bucket", new_id));
|
||||
assert!(matched(None).is_empty());
|
||||
|
||||
engine.withdraw_bucket_rules("bucket");
|
||||
assert!(matched(Some(old_id)).is_empty());
|
||||
assert!(matched(Some(new_id)).is_empty());
|
||||
engine.publish_bucket_rules("bucket", Some(new_id), rules(new_target.clone()));
|
||||
assert!(matched(Some(old_id)).is_empty());
|
||||
assert!(matched(Some(new_id)).contains(&new_target));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rule_engine_tracks_bucket_rule_lifecycle() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
|
||||
@@ -17,7 +17,6 @@ use crate::{
|
||||
bucket_config_manager::NotifyBucketConfigManager,
|
||||
config_manager::NotifyConfigManager,
|
||||
integration::NotificationMetrics,
|
||||
notification_system_subscriber::NotificationSystemSubscriberView,
|
||||
notifier::{EventNotifier, SharedNotifyTargetList},
|
||||
pipeline::{LiveEventHistory, NotifyPipeline},
|
||||
registry::TargetRegistry,
|
||||
@@ -52,7 +51,6 @@ impl NotifyServices {
|
||||
stream_cancellers: Arc<RwLock<ReplayWorkerManager>>,
|
||||
concurrency_limiter: Arc<Semaphore>,
|
||||
metrics: Arc<NotificationMetrics>,
|
||||
subscriber_view: Arc<NotificationSystemSubscriberView>,
|
||||
live_event_sender: broadcast::Sender<Arc<Event>>,
|
||||
live_event_history: Arc<RwLock<LiveEventHistory>>,
|
||||
) -> Self {
|
||||
@@ -65,7 +63,7 @@ impl NotifyServices {
|
||||
metrics.clone(),
|
||||
);
|
||||
let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone());
|
||||
let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view);
|
||||
let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine);
|
||||
let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history);
|
||||
let status_view = NotifyStatusView::new(metrics);
|
||||
|
||||
@@ -84,8 +82,8 @@ impl NotifyServices {
|
||||
mod tests {
|
||||
use super::NotifyServices;
|
||||
use crate::{
|
||||
integration::NotificationMetrics, notification_system_subscriber::NotificationSystemSubscriberView,
|
||||
notifier::EventNotifier, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
|
||||
integration::NotificationMetrics, notifier::EventNotifier, pipeline::LiveEventHistory, registry::TargetRegistry,
|
||||
rule_engine::NotifyRuleEngine,
|
||||
};
|
||||
use rustfs_config::server_config::Config;
|
||||
use rustfs_targets::ReplayWorkerManager;
|
||||
@@ -102,7 +100,6 @@ mod tests {
|
||||
let config = Arc::new(RwLock::new(Config::default()));
|
||||
let stream_cancellers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
|
||||
let concurrency_limiter = Arc::new(Semaphore::new(4));
|
||||
let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
|
||||
let (live_event_sender, _) = broadcast::channel(16);
|
||||
let live_event_history = Arc::new(RwLock::new(LiveEventHistory::default()));
|
||||
|
||||
@@ -115,7 +112,6 @@ mod tests {
|
||||
stream_cancellers,
|
||||
concurrency_limiter,
|
||||
metrics,
|
||||
subscriber_view,
|
||||
live_event_sender,
|
||||
live_event_history,
|
||||
);
|
||||
|
||||
@@ -27,6 +27,15 @@ pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
|
||||
resolve_notify_object_store_handle_from_backend()
|
||||
}
|
||||
|
||||
pub(crate) async fn refresh_notify_bucket_metadata(bucket: &str) -> bool {
|
||||
let Some(store) = resolve_notify_object_store_handle() else {
|
||||
return false;
|
||||
};
|
||||
rustfs_ecstore::api::bucket::metadata_sys::reload_bucket_metadata(store, bucket)
|
||||
.await
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
pub(crate) type NotifyServerConfigSnapshot = rustfs_ecstore::api::config::com::ServerConfigSnapshot;
|
||||
|
||||
pub(crate) async fn read_notify_server_config_snapshot(store: Arc<NotifyStore>) -> Result<NotifyServerConfigSnapshot, String> {
|
||||
|
||||
@@ -127,6 +127,7 @@ rustfs-security-governance = { workspace = true }
|
||||
rustfs-storage-api = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["ip"] }
|
||||
rustix = { workspace = true, features = ["fs", "stdio"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
flate2 = { workspace = true }
|
||||
glob = { workspace = true }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
|
||||
@@ -23,7 +23,8 @@ use crate::global::{
|
||||
METRIC_LOG_CLEANER_ACTIVE_FILE_SIZE_BYTES, METRIC_LOG_CLEANER_ROTATION_DURATION_SECONDS,
|
||||
METRIC_LOG_CLEANER_ROTATION_FAILURES_TOTAL, METRIC_LOG_CLEANER_ROTATION_TOTAL,
|
||||
};
|
||||
use jiff::{Timestamp, ZonedDateTime, tz::TimeZone};
|
||||
use chrono::{Local, TimeZone as _};
|
||||
use jiff::Zoned;
|
||||
use metrics::{counter, gauge, histogram};
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, Write};
|
||||
@@ -57,12 +58,8 @@ impl Rotation {
|
||||
Rotation::Minutely => now / 60 != last / 60,
|
||||
Rotation::Hourly => now / 3600 != last / 3600,
|
||||
Rotation::Daily => {
|
||||
let last_day = Timestamp::new(last, 0)
|
||||
.ok()
|
||||
.map(|ts| ts.to_zoned_datetime(TimeZone::system()).date());
|
||||
let now_day = Timestamp::new(now, 0)
|
||||
.ok()
|
||||
.map(|ts| ts.to_zoned_datetime(TimeZone::system()).date());
|
||||
let last_day = Local.timestamp_opt(last, 0).single().map(|ts| ts.date_naive());
|
||||
let now_day = Local.timestamp_opt(now, 0).single().map(|ts| ts.date_naive());
|
||||
|
||||
match (last_day, now_day) {
|
||||
(Some(last_day), Some(now_day)) => last_day != now_day,
|
||||
@@ -134,7 +131,7 @@ impl RollingAppender {
|
||||
match_mode,
|
||||
file: None,
|
||||
size: 0,
|
||||
last_roll_ts: ZonedDateTime::now().timestamp().as_second(),
|
||||
last_roll_ts: Zoned::now().timestamp().as_second(),
|
||||
rotation_retry_cooldown_until: None,
|
||||
roll_signal: None,
|
||||
};
|
||||
@@ -228,7 +225,7 @@ impl RollingAppender {
|
||||
|
||||
// 2. Time-based check
|
||||
// We check this after size check to avoid unnecessary time calls if size forces a roll.
|
||||
let now = ZonedDateTime::now().timestamp().as_second();
|
||||
let now = Zoned::now().timestamp().as_second();
|
||||
|
||||
self.rotation.check_should_roll(self.last_roll_ts, now)
|
||||
}
|
||||
@@ -252,7 +249,7 @@ impl RollingAppender {
|
||||
// 2. Generate archive name.
|
||||
// Format: YYYYMMDDHHMMSS.uuuuuu (Microsecond/Nanosecond precision)
|
||||
// We use jiff's strftime. "%Y%m%d%H%M%S%.6f" gives microsecond precision.
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
let timestamp_str = now.strftime("%Y%m%d%H%M%S%.6f").to_string();
|
||||
|
||||
// Add a unique counter to prevent collisions in high-concurrency/fast-rotation scenarios.
|
||||
@@ -670,12 +667,11 @@ mod tests {
|
||||
/// The fixtures below use mid-January dates, which sit on no DST or
|
||||
/// offset transition in any zone, so every wall-clock time exists exactly once.
|
||||
fn local_ts(year: i32, month: u32, day: u32, hour: u32, min: u32, sec: u32) -> i64 {
|
||||
jiff::civil::DateTime::new(year as i16, month as i8, day as i8, hour as i8, min as i8, sec as i8, 0)
|
||||
.expect("fixture local time must be valid")
|
||||
.to_zoned_datetime(TimeZone::system())
|
||||
Local
|
||||
.with_ymd_and_hms(year, month, day, hour, min, sec)
|
||||
.single()
|
||||
.expect("fixture local time must exist exactly once")
|
||||
.timestamp()
|
||||
.as_second()
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user