mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-18 14:05:27 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 205e6403f1 | |||
| b62f1ce808 |
Generated
+389
-443
File diff suppressed because it is too large
Load Diff
+33
-33
@@ -151,16 +151,16 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.2.0", default-features = false }
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.66.0", default-features = false }
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.66.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "d6840d54187382d31526317e61369734c759e2cd", version = "0.13.0", default-features = false }
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "ab5242b092f60c22a6cbace3950117f4702de36a", version = "0.13.0", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -168,13 +168,13 @@ async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.53.0", default-features = false }
|
||||
async-nats = { version = "0.52.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.2"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "=6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.3" }
|
||||
hyper = { version = "1.12.0" }
|
||||
@@ -209,8 +209,8 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
] }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.23.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.2" }
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
@@ -242,7 +242,7 @@ jsonwebtoken = { version = "12.0.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "0.10.0", features = ["pkcs1v15"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
rustls = { default-features = false, version = "0.23.46" }
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.16.0"
|
||||
@@ -250,13 +250,13 @@ x509-parser = "0.18.1"
|
||||
sha1 = "0.11.0"
|
||||
sha2 = "0.11.1"
|
||||
subtle = "2.6"
|
||||
zeroize = { version = "1.9.2" }
|
||||
zeroize = { version = "1.9.1" }
|
||||
proptest = "1"
|
||||
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "1.0.1" }
|
||||
jiff = { version = "0.2.43" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -273,18 +273,18 @@ tar-codec = "0.0.16"
|
||||
tar-framing = "0.0.16"
|
||||
atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.3" }
|
||||
aws-config = { version = "1.13.2" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.133.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.160.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.127.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.131.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.159.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.126.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.3" }
|
||||
aws-smithy-runtime-api = { version = "1.22.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.21.0" }
|
||||
aws-smithy-types = { version = "1.9.4" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.1.3"
|
||||
clap = { version = "4.6.8" }
|
||||
brotli = "9.1.1"
|
||||
clap = { version = "4.6.7" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -292,7 +292,7 @@ crossbeam-queue = "0.3.14"
|
||||
crossbeam-channel = "0.5.17"
|
||||
crossbeam-deque = "0.8.8"
|
||||
crossbeam-utils = "0.8.23"
|
||||
datafusion = { default-features = false, version = "55.3.0" }
|
||||
datafusion = { default-features = false, version = "55.2.0" }
|
||||
derive_builder = "0.20.2"
|
||||
ed25519-dalek = "3.0.0"
|
||||
enumset = "1.1.15"
|
||||
@@ -318,7 +318,7 @@ log = "0.4"
|
||||
memmap2 = "0.9.12"
|
||||
nixl-sys = { git = "https://github.com/houseme/nixl", rev = "cc636c83f13d692e85b67e55aed092413d16fb41" }
|
||||
lz4 = "1.28.1"
|
||||
lz4_flex = { version = "0.14.2", default-features = false, features = ["std", "frame"] }
|
||||
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
mime_guess = "2.0.5"
|
||||
@@ -337,14 +337,14 @@ percent-encoding = "2.3.2"
|
||||
qrcode-rs = { version = "2.2.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.4" }
|
||||
rand = { version = "0.10.3" }
|
||||
ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.1.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.2" }
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.2" }
|
||||
redis = { version = "1.8.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.14.0" }
|
||||
@@ -364,7 +364,7 @@ snap = "1.1.2"
|
||||
starshard = { version = "2.4.1" }
|
||||
strum = { version = "0.28.0" }
|
||||
sync_wrapper = "1.0.2"
|
||||
sysinfo = "0.40.4"
|
||||
sysinfo = "0.40.3"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.1"
|
||||
@@ -378,15 +378,15 @@ tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.29.3" }
|
||||
uuid = { version = "1.29.2" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.21" }
|
||||
zip = "9.7.0"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.5.1"
|
||||
zstd = "0.14.4"
|
||||
|
||||
# Observability and Metrics
|
||||
@@ -405,9 +405,9 @@ pyroscope = { version = "2.1.5" }
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.25.0" }
|
||||
unftp-core = "0.1.1"
|
||||
suppaftp = { version = "13.0.0" }
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.2" }
|
||||
russh = { version = "0.65.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
@@ -416,7 +416,7 @@ dav-server = "0.12.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.4" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.32.1", default-features = false }
|
||||
hotpath = { version = "0.31.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ jsonwebtoken = { workspace = true, features = ["aws_lc_rs"] }
|
||||
base64-simd = { workspace = true }
|
||||
pbkdf2 = { workspace = true, optional = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
rsa = { workspace = true }
|
||||
rsa = { workspace = true, features = ["sha2"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
thiserror.workspace = true
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use rsa::{
|
||||
Pkcs1v15Encrypt, PrivateKey, PublicKey,
|
||||
Pkcs1v15Encrypt, RsaPrivateKey, RsaPublicKey,
|
||||
pkcs8::{DecodePrivateKey, DecodePublicKey},
|
||||
pss::{Signature, SigningKey, VerifyingKey},
|
||||
pss::{BlindedSigningKey, Signature, VerifyingKey},
|
||||
sha2::Sha256,
|
||||
signature::{RandomizedSigner, Verifier},
|
||||
traits::PublicKeyParts,
|
||||
@@ -37,7 +37,7 @@ pub struct Token {
|
||||
pub fn gencode(token: &Token, key: &str) -> Result<String> {
|
||||
let data = serde_json::to_vec(token)?;
|
||||
let mut rng = rand::rng();
|
||||
let public_key = PublicKey::from_public_key_pem(key).map_err(Error::other)?;
|
||||
let public_key = RsaPublicKey::from_public_key_pem(key).map_err(Error::other)?;
|
||||
let encrypted_data = public_key.encrypt(&mut rng, Pkcs1v15Encrypt, &data).map_err(Error::other)?;
|
||||
Ok(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&encrypted_data))
|
||||
}
|
||||
@@ -51,7 +51,7 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
|
||||
let encrypted_data = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(Error::other)?;
|
||||
let private_key = PrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
|
||||
let private_key = RsaPrivateKey::from_pkcs8_pem(key).map_err(Error::other)?;
|
||||
let decrypted_data = private_key.decrypt(Pkcs1v15Encrypt, &encrypted_data).map_err(Error::other)?;
|
||||
serde_json::from_slice(&decrypted_data).map_err(Error::other)
|
||||
}
|
||||
@@ -63,8 +63,8 @@ pub fn parse(token: &str, key: &str) -> Result<Token> {
|
||||
pub fn sign_license_token(token: &Token, private_key_pem: &str) -> Result<String> {
|
||||
let payload = serde_json::to_vec(token)?;
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
|
||||
let signing_key = SigningKey::<Sha256>::new(private_key);
|
||||
let private_key = RsaPrivateKey::from_pkcs8_pem(private_key_pem).map_err(Error::other)?;
|
||||
let signing_key = BlindedSigningKey::<Sha256>::new(private_key);
|
||||
let signature: Signature = signing_key.try_sign_with_rng(&mut rng, &payload).map_err(Error::other)?;
|
||||
let signature: Box<[u8]> = signature.into();
|
||||
|
||||
@@ -80,7 +80,7 @@ pub fn parse_signed_license_token(token: &str, public_key_pem: &str) -> Result<T
|
||||
let signed_payload = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(Error::other)?;
|
||||
let public_key = PublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
|
||||
let public_key = RsaPublicKey::from_public_key_pem(public_key_pem).map_err(Error::other)?;
|
||||
let signature_len = public_key.size();
|
||||
|
||||
if signed_payload.len() <= signature_len {
|
||||
@@ -103,7 +103,7 @@ pub fn parse_license_with_public_key(license: &str, public_key: &str) -> Result<
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rsa::{
|
||||
PrivateKey,
|
||||
RsaPrivateKey,
|
||||
pkcs8::{EncodePrivateKey, EncodePublicKey, LineEnding},
|
||||
};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -112,8 +112,8 @@ mod tests {
|
||||
fn test_sign_license_token_and_parse_signed_license_token() {
|
||||
let mut rng = rand::rng();
|
||||
let bits = 2048;
|
||||
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
@@ -144,8 +144,8 @@ mod tests {
|
||||
fn test_legacy_gencode_and_parse_roundtrip() {
|
||||
let mut rng = rand::rng();
|
||||
let bits = 2048;
|
||||
let private_key = PrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, bits).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
@@ -173,8 +173,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_signed_license_token_rejects_tampered_payload() {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let private_key_pem = private_key
|
||||
.to_pkcs8_pem(LineEnding::LF)
|
||||
.expect("failed to encode private key pem");
|
||||
@@ -206,8 +206,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_parse_signed_license_token_rejects_invalid_token() {
|
||||
let mut rng = rand::rng();
|
||||
let private_key = PrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = PublicKey::from(&private_key);
|
||||
let private_key = RsaPrivateKey::new(&mut rng, 2048).expect("Failed to generate private key");
|
||||
let public_key = RsaPublicKey::from(&private_key);
|
||||
let public_key_pem = public_key
|
||||
.to_public_key_pem(LineEnding::LF)
|
||||
.expect("failed to encode public key pem");
|
||||
|
||||
@@ -98,6 +98,7 @@ rustfs-madmin.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
aws-smithy-types.workspace = true
|
||||
async-trait = { workspace = true }
|
||||
@@ -123,7 +124,7 @@ urlencoding.workspace = true
|
||||
walkdir.workspace = true
|
||||
base64-simd = { workspace = true }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
hex-simd = { workspace = true }
|
||||
md-5 = { workspace = true }
|
||||
prost.workspace = true
|
||||
@@ -141,7 +142,6 @@ clap = { workspace = true, features = ["derive", "env"] }
|
||||
[dev-dependencies]
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
opentelemetry-proto = { workspace = true }
|
||||
russh = { workspace = true, features = ["serde"] }
|
||||
russh-sftp = { workspace = true }
|
||||
|
||||
@@ -170,15 +170,6 @@ impl DiskFaultHarness {
|
||||
/// Start (or restart after [`Self::kill_server`]) the RustFS server with
|
||||
/// all volume directories and the same address.
|
||||
pub async fn start_server(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(false).await
|
||||
}
|
||||
|
||||
/// Start a process once its node read-readiness probe passes.
|
||||
pub async fn start_server_at_read_quorum(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(true).await
|
||||
}
|
||||
|
||||
async fn start_server_with_readiness(&mut self, read_quorum_only: bool) -> ChaosResult<()> {
|
||||
// The common helper always appends `env.temp_dir` as the final storage
|
||||
// path: point it at the last disk for startup and pass the remaining
|
||||
// disks explicitly, then restore it to the root so Drop cleans up
|
||||
@@ -190,13 +181,7 @@ impl DiskFaultHarness {
|
||||
let arg_refs: Vec<&str> = disk_args.iter().map(String::as_str).collect();
|
||||
let env_refs: Vec<(&str, &str)> = self.extra_env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
|
||||
|
||||
let result = if read_quorum_only {
|
||||
self.env
|
||||
.start_rustfs_server_with_env_at_read_quorum(arg_refs, &env_refs)
|
||||
.await
|
||||
} else {
|
||||
self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await
|
||||
};
|
||||
let result = self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await;
|
||||
self.env.temp_dir = self.root.to_string_lossy().to_string();
|
||||
result
|
||||
}
|
||||
|
||||
@@ -722,7 +722,7 @@ impl RustFSTestEnvironment {
|
||||
cleanup_existing: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing, false)
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -732,7 +732,6 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
cleanup_existing: bool,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
if cleanup_existing {
|
||||
self.cleanup_existing_processes().await?;
|
||||
@@ -757,7 +756,7 @@ impl RustFSTestEnvironment {
|
||||
self.process = Some(process);
|
||||
|
||||
// Wait for server to be ready
|
||||
self.wait_for_server_ready_with_probe(read_quorum_only).await?;
|
||||
self.wait_for_server_ready().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -771,7 +770,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -783,7 +782,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -801,18 +800,6 @@ impl RustFSTestEnvironment {
|
||||
self.start_rustfs_server_inner(extra_args, extra_env, true).await
|
||||
}
|
||||
|
||||
/// Start a degraded read-serving process using the node readiness probe.
|
||||
/// ListBuckets retains its namespace write-quorum requirement.
|
||||
pub async fn start_rustfs_server_with_env_at_read_quorum(
|
||||
&mut self,
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, true, true)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start RustFS server without cleaning up other running RustFS processes.
|
||||
///
|
||||
/// This is useful for tests that need multiple independent RustFS instances
|
||||
@@ -841,16 +828,8 @@ impl RustFSTestEnvironment {
|
||||
/// Fails immediately if the spawned server process has already exited
|
||||
/// (e.g. a port bind failure) instead of polling out the full timeout.
|
||||
pub async fn wait_for_server_ready(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_server_ready_with_probe(false).await
|
||||
}
|
||||
|
||||
async fn wait_for_server_ready_with_probe(
|
||||
&mut self,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
info!("Waiting for RustFS server to be ready on {}", self.address);
|
||||
let client = self.create_s3_client();
|
||||
let health_client = HttpClient::builder().no_proxy().timeout(Duration::from_secs(2)).build()?;
|
||||
|
||||
for i in 0..60 {
|
||||
if let Some(process) = self.process.as_mut()
|
||||
@@ -859,16 +838,7 @@ impl RustFSTestEnvironment {
|
||||
return Err(format!("RustFS server process exited before becoming ready: {status}").into());
|
||||
}
|
||||
|
||||
let ready = if read_quorum_only {
|
||||
health_client
|
||||
.get(format!("{}/minio/health/ready", self.url))
|
||||
.send()
|
||||
.await
|
||||
.is_ok_and(|response| response.status().is_success())
|
||||
} else {
|
||||
TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok()
|
||||
};
|
||||
if ready {
|
||||
if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
|
||||
info!("✅ RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! Over-the-wire smoke net for the embedded console listener (backlog#1154
|
||||
//! peri-4). The console is a second HTTP listener for the control-plane and
|
||||
//! unauthenticated console routes under `/rustfs/console`;
|
||||
//! peri-4). The console is a second HTTP listener that serves the full S3/admin
|
||||
//! surface plus the unauthenticated console routes under `/rustfs/console`;
|
||||
//! before this test its behaviour was only covered by in-process router tests
|
||||
//! (`rustfs/src/admin/console.rs`), so a misconfiguration that exposed
|
||||
//! credentials through the public console endpoints or fail-opened the
|
||||
@@ -26,13 +26,13 @@
|
||||
//! * `/rustfs/console/license` exposes only the coarse `licensed` flag.
|
||||
//! * the console SPA prefix dispatches to the static handler, never to the
|
||||
//! S3 API (no S3 error XML), whether or not console assets are embedded.
|
||||
//! * unauthenticated admin API requests are denied with 403 AccessDenied.
|
||||
//! * S3 root/object requests on the console listener return an empty 404,
|
||||
//! including authenticated requests, before S3 dispatch.
|
||||
//! * unauthenticated requests to the admin API and the S3 root on the console
|
||||
//! listener are denied (403 AccessDenied) — the extra listener does not
|
||||
//! fail-open the protected surface.
|
||||
//! * a listener with the console disabled (the main S3 port here) does not
|
||||
//! serve the unauthenticated console endpoints at all.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, signed_request_with_client};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
@@ -166,38 +166,18 @@ async fn test_console_over_the_wire_smoke() -> TestResult {
|
||||
let admin_body = admin_response.text().await?;
|
||||
assert!(admin_body.contains("AccessDenied"), "admin denial must be AccessDenied: {admin_body}");
|
||||
|
||||
let signed_data_root = signed_request_with_client(
|
||||
&client,
|
||||
http::Method::GET,
|
||||
&format!("{}/", env.url),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
// The console listener serves no S3 surface at all: an object route is not found there,
|
||||
// signed or not, instead of reaching the S3 service and being denied by it.
|
||||
let s3_root_response = client.get(format!("{console_base}/")).send().await?;
|
||||
assert_eq!(
|
||||
signed_data_root.status(),
|
||||
reqwest::StatusCode::OK,
|
||||
"the same credentials must reach S3 on the data listener"
|
||||
s3_root_response.status(),
|
||||
reqwest::StatusCode::NOT_FOUND,
|
||||
"the S3 root must not be served on the console listener"
|
||||
);
|
||||
assert!(
|
||||
s3_root_response.text().await?.is_empty(),
|
||||
"the console listener must not answer an object route with an S3 document"
|
||||
);
|
||||
|
||||
for path in ["/", "/console-object-bucket/key"] {
|
||||
let url = format!("{console_base}{path}");
|
||||
for signed in [false, true] {
|
||||
let response = if signed {
|
||||
signed_request_with_client(&client, http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?
|
||||
} else {
|
||||
client.get(&url).send().await?
|
||||
};
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
reqwest::StatusCode::NOT_FOUND,
|
||||
"S3 route {path}, signed={signed} must be isolated from the console listener"
|
||||
);
|
||||
assert!(response.bytes().await?.is_empty(), "isolated S3 routes must not reach a handler");
|
||||
}
|
||||
}
|
||||
|
||||
// --- console disabled on a listener means no console surface at all ------
|
||||
// The main S3 listener of this same process runs with the console disabled,
|
||||
|
||||
@@ -26,7 +26,7 @@ use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ObjectLockRule,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
|
||||
fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
@@ -66,7 +66,7 @@ async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let retain_until = Timestamp::now() + SignedDuration::from_hours(24);
|
||||
let retain_until = Utc::now() + ChronoDuration::days(1);
|
||||
|
||||
let compliance_key = "compliance.bin";
|
||||
let compliance_version = put_object_with_retention(
|
||||
@@ -176,10 +176,7 @@ async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_conf
|
||||
let retain_until = retention
|
||||
.retain_until_date()
|
||||
.ok_or("default retention omitted retain-until date")?;
|
||||
assert!(
|
||||
retain_until.secs() > Timestamp::now().as_second(),
|
||||
"default retention is not in the future"
|
||||
);
|
||||
assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
|
||||
|
||||
let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
|
||||
assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
|
||||
|
||||
@@ -38,8 +38,8 @@ use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use http::{Method, StatusCode};
|
||||
use jiff::Timestamp;
|
||||
use serde_json::Value;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -526,8 +526,8 @@ async fn put_multipart(client: &Client, bucket: &str, key: &str) -> TestResult<V
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
fn retention_deadline() -> TestResult<Timestamp> {
|
||||
Ok(Timestamp::now() + jiff::SignedDuration::from_hours(24))
|
||||
fn retention_deadline() -> TestResult<DateTime<Utc>> {
|
||||
DateTime::<Utc>::from_timestamp(Utc::now().timestamp() + 24 * 3600, 0).ok_or_else(|| "retention deadline out of range".into())
|
||||
}
|
||||
|
||||
/// Seed the deterministic inventory into the seed pool and record it.
|
||||
@@ -654,7 +654,7 @@ pub(crate) async fn seed_manifest(run: &TopologyRun) -> TestResult<TopologyManif
|
||||
let mut entry = ManifestEntry::body(&locked, &key, "compliance", Some(version_id.clone()), Some(version_id), &body);
|
||||
entry.retention = Some(RetentionRecord {
|
||||
mode: "COMPLIANCE".to_string(),
|
||||
retain_until_unix: retain_until.as_second(),
|
||||
retain_until_unix: retain_until.timestamp(),
|
||||
});
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
@@ -29,7 +29,6 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use jiff::Timestamp;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
@@ -387,7 +386,7 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
|
||||
"usage": "EncryptDecrypt",
|
||||
"status": "Active",
|
||||
"metadata": HashMap::<String, String>::new(),
|
||||
"created_at": format!("{}[UTC]", Timestamp::now().strftime("%Y-%m-%dT%H:%M:%S%.fZ")),
|
||||
"created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
|
||||
"rotated_at": serde_json::Value::Null,
|
||||
"created_by": "e2e-test",
|
||||
"encrypted_key_material": BASE64.encode_to_string(key_data),
|
||||
|
||||
@@ -23,13 +23,13 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use flate2::{
|
||||
Compression,
|
||||
write::{GzEncoder, ZlibEncoder},
|
||||
};
|
||||
use http::HeaderValue;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_s3_types::Body;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
@@ -42,8 +42,8 @@ use tokio::io::AsyncWriteExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn encode_post_policy(conditions: Vec<serde_json::Value>) -> String {
|
||||
let expiration = (Timestamp::now() + SignedDuration::from_hours(1))
|
||||
.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
let expiration = (Utc::now() + ChronoDuration::hours(1))
|
||||
.format("%Y-%m-%dT%H:%M:%S.000Z")
|
||||
.to_string();
|
||||
let policy = serde_json::json!({
|
||||
"expiration": expiration,
|
||||
|
||||
@@ -283,7 +283,6 @@ async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> Te
|
||||
#[tokio::test]
|
||||
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
assert_single_node_cold_start_with_missing_volume().await?;
|
||||
|
||||
for (node_count, parity) in [(3, 1), (4, 2), (6, 3), (6, 2), (4, 0)] {
|
||||
let read_quorum = node_count - parity;
|
||||
@@ -431,138 +430,6 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_single_node_cold_start_with_missing_volume() -> TestResult {
|
||||
for drive_count in [2, 3, 4] {
|
||||
let mut server = crate::chaos::DiskFaultHarness::new(drive_count).await?;
|
||||
server.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{}", drive_count / 2));
|
||||
server.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
server.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
server.set_env("RUSTFS_HEALTH_ENDPOINT_ENABLE", "true");
|
||||
server.set_env("RUST_LOG", "warn");
|
||||
server.start_server_at_read_quorum().await?;
|
||||
let client = server.env.create_s3_client();
|
||||
let client = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
);
|
||||
let bucket = format!("missing-volume-{drive_count}");
|
||||
client.create_bucket().bucket(&bucket).send().await?;
|
||||
let small = vec![17; 4096];
|
||||
let large = vec![73; 8 * 1024 * 1024];
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
for disk in 0..drive_count {
|
||||
let census = crate::chaos::census_object_version_on_disk(server.disk_path(disk), &bucket, key, None)?;
|
||||
assert!(census.is_complete(), "seed shard must exist before restart: {census:?}");
|
||||
}
|
||||
}
|
||||
server.kill_server();
|
||||
let missing = drive_count - 1;
|
||||
server.take_disk_offline(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
let head = client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
|
||||
}
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
let keys: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(keys, ["large", "small"]);
|
||||
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("degraded-write")
|
||||
.body(Bytes::from_static(b"degraded write").into())
|
||||
.send()
|
||||
.await;
|
||||
if drive_count == 2 {
|
||||
let error = put.expect_err("one of two drives cannot commit a new object");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
let error = client
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key("large")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("one of two drives cannot commit an object deletion");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_quorum_object_body(&client, &bucket, "large", &large).await?;
|
||||
} else {
|
||||
put?;
|
||||
assert_quorum_object_body(&client, &bucket, "degraded-write", b"degraded write").await?;
|
||||
client.delete_object().bucket(&bucket).key("degraded-write").send().await?;
|
||||
}
|
||||
assert!(
|
||||
!server.disk_path(missing).exists(),
|
||||
"degraded startup and S3 operations must leave the root absent"
|
||||
);
|
||||
|
||||
if drive_count == 2 {
|
||||
server.kill_server();
|
||||
std::fs::create_dir(server.disk_path(missing))?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
}
|
||||
assert_eq!(client.list_objects_v2().bucket(&bucket).send().await?.contents().len(), 2);
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("empty-root-write")
|
||||
.body(Bytes::from_static(b"must remain unavailable").into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an empty replacement cannot restore write quorum");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert!(
|
||||
!server
|
||||
.disk_path(missing)
|
||||
.join(crate::storage_api::RUSTFS_META_BUCKET)
|
||||
.join("format.json")
|
||||
.exists(),
|
||||
"read admission must not format the empty replacement"
|
||||
);
|
||||
}
|
||||
server.kill_server();
|
||||
server.bring_disk_online(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
}
|
||||
for key in ["degraded-write", "empty-root-write"] {
|
||||
let error = client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("rejected or deleted degraded write must stay absent after recovery");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("recovery-write")
|
||||
.body(Bytes::from_static(b"recovered write").into())
|
||||
.send()
|
||||
.await?;
|
||||
assert_quorum_object_body(&client, &bucket, "recovery-write", b"recovered write").await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cold_start_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
@@ -29,7 +29,7 @@ use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockMode,
|
||||
ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use tracing::info;
|
||||
|
||||
use crate::common::RustFSTestEnvironment;
|
||||
@@ -126,10 +126,10 @@ pub async fn put_object_with_retention(
|
||||
key: &str,
|
||||
data: &[u8],
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: Timestamp,
|
||||
retain_until: DateTime<Utc>,
|
||||
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
|
||||
// AWS SDK requires UTC time without timezone offset (e.g., "2026-01-24T11:20:14Z")
|
||||
let retain_until_str = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_str = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_datetime =
|
||||
aws_sdk_s3::primitives::DateTime::from_str(&retain_until_str, aws_sdk_s3::primitives::DateTimeFormat::DateTime)?;
|
||||
|
||||
@@ -181,10 +181,10 @@ pub async fn put_object_retention(
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
mode: ObjectLockRetentionMode,
|
||||
retain_until: Timestamp,
|
||||
retain_until: DateTime<Utc>,
|
||||
bypass_governance: bool,
|
||||
) -> Result<(), Box<SdkError<PutObjectRetentionError>>> {
|
||||
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.as_second());
|
||||
let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.timestamp());
|
||||
|
||||
let retention = ObjectLockRetention::builder()
|
||||
.mode(mode.clone())
|
||||
@@ -253,6 +253,6 @@ pub async fn delete_object_with_bypass(
|
||||
}
|
||||
|
||||
/// Calculate a future retain_until date
|
||||
pub fn future_retain_until(days: i64) -> Timestamp {
|
||||
Timestamp::now() + SignedDuration::from_hours(days * 24)
|
||||
pub fn future_retain_until(days: i64) -> DateTime<Utc> {
|
||||
Utc::now() + Duration::days(days)
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use aws_sdk_s3::types::{
|
||||
CompletedMultipartUpload, CompletedPart, Delete, MetadataDirective, ObjectIdentifier, ObjectLockLegalHoldStatus,
|
||||
ObjectLockMode, ObjectLockRetentionMode,
|
||||
};
|
||||
use jiff::{SignedDuration, Timestamp};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use tracing::info;
|
||||
|
||||
/// Initialize test logging
|
||||
@@ -67,7 +67,7 @@ async fn put_bucket_deny_policy(
|
||||
}
|
||||
|
||||
fn retention_timestamp(days: i64) -> aws_sdk_s3::primitives::DateTime {
|
||||
let retain_until = future_retain_until(days).strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until = future_retain_until(days).format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
aws_sdk_s3::primitives::DateTime::from_str(&retain_until, aws_sdk_s3::primitives::DateTimeFormat::DateTime)
|
||||
.expect("retention timestamp should parse")
|
||||
}
|
||||
@@ -118,14 +118,13 @@ where
|
||||
);
|
||||
}
|
||||
|
||||
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> Timestamp {
|
||||
fn parse_s3_datetime(value: &aws_sdk_s3::primitives::DateTime) -> DateTime<Utc> {
|
||||
let formatted = value
|
||||
.fmt(DateTimeFormat::DateTime)
|
||||
.expect("S3 timestamp should format as RFC3339");
|
||||
let normalized = formatted
|
||||
.strip_suffix('Z')
|
||||
.map_or(formatted.clone(), |prefix| format!("{prefix}+00:00"));
|
||||
Timestamp::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", normalized).expect("S3 timestamp should parse as RFC3339")
|
||||
DateTime::parse_from_rfc3339(&formatted)
|
||||
.expect("S3 timestamp should parse as RFC3339")
|
||||
.with_timezone(&Utc)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -471,7 +470,7 @@ async fn test_get_object_retention_returns_configured_values() {
|
||||
let bucket = "test-get-retention";
|
||||
let key = "retained-object";
|
||||
let retain_until = future_retain_until(30);
|
||||
let retain_until_expected = retain_until.strftime("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let retain_until_expected = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
|
||||
env.create_object_lock_bucket(bucket).await.unwrap();
|
||||
|
||||
@@ -1925,7 +1924,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let copy_started = Timestamp::now();
|
||||
let copy_started = Utc::now();
|
||||
client
|
||||
.copy_object()
|
||||
.copy_source(format!("{src_bucket}/{src_key}"))
|
||||
@@ -1948,7 +1947,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
|
||||
assert!(
|
||||
retain_until < copy_started + SignedDuration::from_hours(3 * 24),
|
||||
retain_until < copy_started + Duration::days(3),
|
||||
"destination default retention should not inherit the source's longer retention"
|
||||
);
|
||||
|
||||
@@ -2009,7 +2008,7 @@ async fn test_copy_object_retention_uses_destination_policy() {
|
||||
.expect("explicit retain-until date should be present"),
|
||||
);
|
||||
assert!(
|
||||
explicit_retain_until > Timestamp::now() + SignedDuration::from_hours(20 * 24),
|
||||
explicit_retain_until > Utc::now() + Duration::days(20),
|
||||
"explicit retention should override the shorter bucket default"
|
||||
);
|
||||
|
||||
@@ -2089,7 +2088,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let create_started = Timestamp::now();
|
||||
let create_started = Utc::now();
|
||||
let create_output = client.create_multipart_upload().bucket(bucket).key(key).send().await.unwrap();
|
||||
let upload_id = create_output.upload_id().unwrap();
|
||||
|
||||
@@ -2132,7 +2131,7 @@ async fn test_multipart_default_retention_fixed_at_create() {
|
||||
assert_eq!(retention.mode().map(|value| value.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = parse_s3_datetime(retention.retain_until_date().expect("retain-until date should be present"));
|
||||
assert!(
|
||||
retain_until < create_started + SignedDuration::from_hours(3 * 24),
|
||||
retain_until < create_started + Duration::days(3),
|
||||
"CompleteMultipartUpload should keep the default retention calculated at create time"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -573,15 +573,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
type RenameDataBatchStream = ResponseStream<rustfs_protos::proto_gen::node_service::RenameDataBatchResult>;
|
||||
|
||||
async fn rename_data_batch(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataBatchRequest>,
|
||||
) -> Result<Response<Self::RenameDataBatchStream>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_multipart_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -154,6 +154,7 @@ arc-swap.workspace = true
|
||||
async-trait.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
byteorder = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
thiserror.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
|
||||
@@ -90,9 +90,7 @@ fn remote_credentials(credentials: &Credentials, account_id: &str) -> RemoteCred
|
||||
access_key: credentials.access_key.clone(),
|
||||
secret_key: credentials.secret_key.clone(),
|
||||
session_token: credentials.effective_session_token().map(str::to_string),
|
||||
expiration: credentials
|
||||
.effective_expiration()
|
||||
.and_then(|timestamp| SystemTime::try_from(timestamp).ok()),
|
||||
expiration: credentials.effective_expiration().map(SystemTime::from),
|
||||
account_id: account_id.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1070,8 +1070,8 @@ mod tests {
|
||||
.unwrap_or_else(|| panic!("signed request must contain {name}"))
|
||||
}
|
||||
|
||||
fn signing_time(headers: &[(String, String)]) -> jiff::Timestamp {
|
||||
jiff::Timestamp::strptime("%Y%m%dT%H%M%SZ", recorded_header(headers, "x-amz-date"))
|
||||
fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
|
||||
chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
|
||||
.expect("SDK signing timestamp must use the SigV4 format")
|
||||
}
|
||||
|
||||
@@ -1086,8 +1086,8 @@ mod tests {
|
||||
.map(|(key, value)| (key.to_string(), value.to_string()))
|
||||
.collect(),
|
||||
);
|
||||
let server_time = jiff::Timestamp::try_from(self.clock.now()).expect("clock timestamp")
|
||||
+ jiff::SignedDuration::from_secs(self.skew_seconds);
|
||||
let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
|
||||
+ chrono::Duration::seconds(self.skew_seconds);
|
||||
let (status, body) = if headers.len() == 1 {
|
||||
(
|
||||
403,
|
||||
@@ -1098,7 +1098,7 @@ mod tests {
|
||||
};
|
||||
let response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("date", server_time.strftime("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len())
|
||||
.body(SdkBody::from(body))
|
||||
@@ -1136,7 +1136,7 @@ mod tests {
|
||||
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
|
||||
for skew_seconds in [-600, 600] {
|
||||
let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
|
||||
let initial = jiff::Timestamp::try_from(clock.now()).expect("clock timestamp");
|
||||
let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
|
||||
client
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
@@ -1163,12 +1163,12 @@ mod tests {
|
||||
assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
|
||||
assert_eq!(
|
||||
signing_time(&headers[1]),
|
||||
initial + jiff::SignedDuration::from_secs(skew_seconds),
|
||||
initial + chrono::Duration::seconds(skew_seconds),
|
||||
"{error_code}: retry must apply the measured offset exactly"
|
||||
);
|
||||
assert_eq!(
|
||||
signing_time(&headers[2]),
|
||||
initial + jiff::SignedDuration::from_secs(skew_seconds + 17),
|
||||
initial + chrono::Duration::seconds(skew_seconds + 17),
|
||||
"{error_code}: the next operation must apply cached skew to the advanced signing clock"
|
||||
);
|
||||
let signature = |index: usize| {
|
||||
|
||||
@@ -73,11 +73,10 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataBatchItem, RenameDataBatchRequest,
|
||||
RenameDataBatchResult, RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
|
||||
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest,
|
||||
VerifyFileRequest, WriteAllRequest, WriteAllResponse, WriteFormatControlRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteAllResponse,
|
||||
WriteFormatControlRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use rustfs_protos::{
|
||||
ChannelClass, ChannelRebuildReason, ConnectionEvictionLogLevel, evict_failed_connection_with_reason, prewarm_channel_class,
|
||||
@@ -105,7 +104,7 @@ use tokio::{
|
||||
time::timeout,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tonic::{Code, Request, Streaming, service::interceptor::InterceptedService};
|
||||
use tonic::{Code, Request, service::interceptor::InterceptedService};
|
||||
use tracing::{debug, trace, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -1046,7 +1045,6 @@ pub struct RemoteDisk {
|
||||
/// Optional per-peer gate held through complete bulk-unary decoding and validation.
|
||||
bulk_body_admission: Option<Arc<Semaphore>>,
|
||||
bulk_body_wait_timeout: Duration,
|
||||
rename_admission: Option<Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
/// Shortest gap between two failure warnings of one kind from one remote disk.
|
||||
@@ -1062,251 +1060,6 @@ struct BulkBodyAdmissionRegistry {
|
||||
static BULK_BODY_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<BulkBodyAdmissionRegistry>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(BulkBodyAdmissionRegistry::default()));
|
||||
|
||||
const ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH: &str = "RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH";
|
||||
const RENAME_ADMISSION_REGISTRY_CAPACITY: usize = 512;
|
||||
|
||||
#[derive(Default)]
|
||||
struct RenameAdmissionRegistry {
|
||||
peers: HashMap<String, Weak<Semaphore>>,
|
||||
overflow: Option<Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
static RENAME_ADMISSION_REGISTRY: std::sync::LazyLock<StdMutex<RenameAdmissionRegistry>> =
|
||||
std::sync::LazyLock::new(|| StdMutex::new(RenameAdmissionRegistry::default()));
|
||||
|
||||
fn peer_rename_admission(addr: &str, enabled: bool) -> Option<Arc<Semaphore>> {
|
||||
if !enabled {
|
||||
return None;
|
||||
}
|
||||
let mut registry = RENAME_ADMISSION_REGISTRY
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(semaphore) = registry.peers.get(addr).and_then(Weak::upgrade) {
|
||||
return Some(semaphore);
|
||||
}
|
||||
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
|
||||
registry.peers.retain(|_, semaphore| semaphore.strong_count() > 0);
|
||||
}
|
||||
if registry.peers.len() >= RENAME_ADMISSION_REGISTRY_CAPACITY {
|
||||
return Some(Arc::clone(registry.overflow.get_or_insert_with(|| {
|
||||
Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT))
|
||||
})));
|
||||
}
|
||||
let semaphore = Arc::new(Semaphore::new(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT));
|
||||
registry.peers.insert(addr.to_owned(), Arc::downgrade(&semaphore));
|
||||
Some(semaphore)
|
||||
}
|
||||
|
||||
struct RenameAdmissionGrant {
|
||||
permit: Option<OwnedSemaphorePermit>,
|
||||
batch: bool,
|
||||
}
|
||||
|
||||
impl RenameAdmissionGrant {
|
||||
fn release_one(&mut self) -> bool {
|
||||
self.permit.as_mut().is_some_and(|permit| permit.split(1).is_some())
|
||||
}
|
||||
|
||||
fn release_all(mut self) {
|
||||
drop(self.permit.take());
|
||||
}
|
||||
|
||||
fn quarantine_until(mut self, deadline: tokio::time::Instant) {
|
||||
let Some(permit) = self.permit.take() else {
|
||||
return;
|
||||
};
|
||||
let items = permit.num_permits();
|
||||
if items == 0 {
|
||||
return;
|
||||
}
|
||||
let hold = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine(self.batch, items, hold);
|
||||
let batch = self.batch;
|
||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||
handle.spawn(async move {
|
||||
tokio::time::sleep_until(deadline).await;
|
||||
drop(permit);
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine_released(batch, items);
|
||||
});
|
||||
} else {
|
||||
// Without a runtime there is no safe deadline task. Leaking until
|
||||
// process exit is safer than admitting work over a still-running mutation.
|
||||
rustfs_io_metrics::record_ecstore_rename_quarantine_no_runtime(batch, items);
|
||||
std::mem::forget(permit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct RenameBatchCallGuard {
|
||||
admission: Option<RenameAdmissionGrant>,
|
||||
deadline: tokio::time::Instant,
|
||||
dispatch_state: RenameAdmissionDispatchState,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum RenameAdmissionDispatchState {
|
||||
NotDispatched,
|
||||
MayHavePublished,
|
||||
}
|
||||
|
||||
impl RenameBatchCallGuard {
|
||||
fn mark_dispatched(&mut self) {
|
||||
self.dispatch_state = RenameAdmissionDispatchState::MayHavePublished;
|
||||
}
|
||||
|
||||
fn complete(mut self) {
|
||||
debug_assert_eq!(self.dispatch_state, RenameAdmissionDispatchState::MayHavePublished);
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.release_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn release_without_quarantine(mut self) {
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.release_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn into_stream(mut self, inner: Streaming<RenameDataBatchResult>, expected_mask: u16) -> RemoteRenameBatchStream {
|
||||
debug_assert_eq!(self.dispatch_state, RenameAdmissionDispatchState::MayHavePublished);
|
||||
RemoteRenameBatchStream {
|
||||
inner,
|
||||
expected_mask,
|
||||
seen_mask: 0,
|
||||
admission: self.admission.take(),
|
||||
deadline: self.deadline,
|
||||
terminal: false,
|
||||
stream_started_at: rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now),
|
||||
first_item_at: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RenameBatchCallGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.dispatch_state == RenameAdmissionDispatchState::MayHavePublished
|
||||
&& let Some(admission) = self.admission.take()
|
||||
{
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenameBatchClientItem<'a> {
|
||||
pub slot: usize,
|
||||
pub disk: String,
|
||||
pub file_info: &'a FileInfo,
|
||||
}
|
||||
|
||||
pub(crate) enum RenameBatchStartError {
|
||||
NotDispatched(Error),
|
||||
UnsupportedBeforeStream,
|
||||
MayHavePublished(Error),
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteRenameBatchOutcome {
|
||||
pub slot: usize,
|
||||
pub result: Result<RenameDataResp>,
|
||||
}
|
||||
|
||||
pub(crate) struct RemoteRenameBatchStream {
|
||||
inner: Streaming<RenameDataBatchResult>,
|
||||
expected_mask: u16,
|
||||
seen_mask: u16,
|
||||
admission: Option<RenameAdmissionGrant>,
|
||||
deadline: tokio::time::Instant,
|
||||
terminal: bool,
|
||||
stream_started_at: Option<Instant>,
|
||||
first_item_at: Option<Instant>,
|
||||
}
|
||||
|
||||
impl RemoteRenameBatchStream {
|
||||
fn terminate_ambiguous(&mut self, error: Error) -> Error {
|
||||
if !self.terminal {
|
||||
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
self.terminal = true;
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
error
|
||||
}
|
||||
|
||||
pub(crate) async fn next_outcome(&mut self) -> Result<Option<RemoteRenameBatchOutcome>> {
|
||||
if self.terminal {
|
||||
return Ok(None);
|
||||
}
|
||||
let response = match tokio::time::timeout_at(self.deadline, self.inner.message()).await {
|
||||
Err(_) => return Err(self.terminate_ambiguous(DiskError::Timeout)),
|
||||
Ok(Err(status)) => return Err(self.terminate_ambiguous(status.into())),
|
||||
Ok(Ok(None)) if self.seen_mask == self.expected_mask => {
|
||||
self.terminal = true;
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Ok(None)) => return Err(self.terminate_ambiguous(DiskError::Unexpected)),
|
||||
Ok(Ok(Some(response))) => response,
|
||||
};
|
||||
let Ok(slot) = usize::try_from(response.item_index) else {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
};
|
||||
if slot >= 16 {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
let bit = 1_u16 << slot;
|
||||
if self.expected_mask & bit == 0 || self.seen_mask & bit != 0 {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
self.seen_mask |= bit;
|
||||
if self.admission.as_mut().is_some_and(|admission| !admission.release_one()) {
|
||||
return Err(self.terminate_ambiguous(DiskError::Unexpected));
|
||||
}
|
||||
if let Some(stream_started_at) = self.stream_started_at {
|
||||
let completed_at = Instant::now();
|
||||
match self.first_item_at {
|
||||
Some(first_item_at) if self.seen_mask == self.expected_mask => {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_first_to_last(
|
||||
completed_at.saturating_duration_since(first_item_at),
|
||||
);
|
||||
}
|
||||
Some(_) => {}
|
||||
None => {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_stream_to_first(
|
||||
completed_at.saturating_duration_since(stream_started_at),
|
||||
);
|
||||
self.first_item_at = Some(completed_at);
|
||||
if self.seen_mask == self.expected_mask {
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_client_first_to_last(Duration::ZERO);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_item_completed();
|
||||
let result = if response.success {
|
||||
if response.error.is_some() {
|
||||
Err(DiskError::FileCorrupt)
|
||||
} else {
|
||||
decode_msgpack_strict(&response.rename_data_resp_msgpack)
|
||||
}
|
||||
} else {
|
||||
Err(response.error.unwrap_or_default().into())
|
||||
};
|
||||
Ok(Some(RemoteRenameBatchOutcome { slot, result }))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RemoteRenameBatchStream {
|
||||
fn drop(&mut self) {
|
||||
if !self.terminal {
|
||||
let missing = (self.expected_mask & !self.seen_mask).count_ones() as usize;
|
||||
rustfs_io_metrics::record_ecstore_rename_batch_terminal_missing(missing);
|
||||
}
|
||||
if let Some(admission) = self.admission.take() {
|
||||
admission.quarantine_until(self.deadline);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BulkBodyPermit {
|
||||
_metrics: rustfs_io_metrics::internode_metrics::InternodeBulkBodyAdmissionGuard,
|
||||
_permit: OwnedSemaphorePermit,
|
||||
@@ -1473,10 +1226,6 @@ fn spawn_control_channel_prewarm(addr: String) {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
pub(crate) fn rename_stream_batch_enabled(&self) -> bool {
|
||||
self.rename_admission.is_some()
|
||||
}
|
||||
|
||||
async fn guarded_delete_versions_rpc(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1802,8 +1551,6 @@ impl RemoteDisk {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
|
||||
let bulk_body_admission = peer_bulk_body_admission(&addr, bulk_body_concurrency_limit()?);
|
||||
let bulk_body_wait_timeout = bulk_body_wait_timeout();
|
||||
let rename_admission =
|
||||
peer_rename_admission(&addr, rustfs_utils::get_env_bool(ENV_RUSTFS_EXPERIMENTAL_RENAME_STREAM_BATCH, false));
|
||||
|
||||
let disk = Self {
|
||||
handle_id: Uuid::new_v4(),
|
||||
@@ -1826,7 +1573,6 @@ impl RemoteDisk {
|
||||
network_error_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
|
||||
bulk_body_admission,
|
||||
bulk_body_wait_timeout,
|
||||
rename_admission,
|
||||
};
|
||||
record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);
|
||||
|
||||
@@ -2562,54 +2308,6 @@ impl RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_peer_batch_with_timeout<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
operation: F,
|
||||
timeout_duration: Duration,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
if timeout_duration == Duration::ZERO {
|
||||
self.record_timeout(op, timeout_duration);
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match time::timeout(timeout_duration, operation()).await {
|
||||
Ok(operation_result) => {
|
||||
if operation_result.is_ok() {
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
let evict = operation_result
|
||||
.as_ref()
|
||||
.err()
|
||||
.is_some_and(|error| !error.is_metacache_output_stream_closed() && is_network_like_disk_error(error));
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, FailureHealthAction::IgnoreFailure)
|
||||
.await;
|
||||
if evict {
|
||||
evict_failed_connection_with_reason(
|
||||
&self.addr,
|
||||
ChannelRebuildReason::RpcFailure,
|
||||
ConnectionEvictionLogLevel::Debug,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
operation_result
|
||||
}
|
||||
Err(_) => {
|
||||
self.record_timeout(op, timeout_duration);
|
||||
evict_failed_connection_with_reason(
|
||||
&self.addr,
|
||||
ChannelRebuildReason::OperationTimeout,
|
||||
ConnectionEvictionLogLevel::Debug,
|
||||
)
|
||||
.await;
|
||||
Err(DiskError::Timeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_timeout(&self, op: &'static str, timeout_duration: Duration) {
|
||||
counter!(
|
||||
"rustfs_drive_op_timeout_total",
|
||||
@@ -2796,38 +2494,6 @@ impl RemoteDisk {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn acquire_rename_admission_until(
|
||||
&self,
|
||||
item_count: usize,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<Option<RenameAdmissionGrant>> {
|
||||
let Some(semaphore) = &self.rename_admission else {
|
||||
return Ok(None);
|
||||
};
|
||||
if item_count == 0 || item_count > rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT {
|
||||
return Err(Error::other("invalid internode rename admission weight"));
|
||||
}
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
}
|
||||
let weight = u32::try_from(item_count).map_err(Error::other)?;
|
||||
let started = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
let (result, timed_out) = match tokio::time::timeout_at(deadline, Arc::clone(semaphore).acquire_many_owned(weight)).await
|
||||
{
|
||||
Ok(Ok(permit)) => (Ok(permit), false),
|
||||
Ok(Err(_)) => (Err(Error::RemoteClientUnavailable("internode rename admission closed".to_owned())), false),
|
||||
Err(_) => (Err(DiskError::DiskOngoingReq), true),
|
||||
};
|
||||
if let Some(started) = started {
|
||||
rustfs_io_metrics::record_ecstore_rename_admission(item_count > 1, item_count, started.elapsed(), timed_out);
|
||||
}
|
||||
let permit = result?;
|
||||
Ok(Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: item_count > 1,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn get_bulk_client(&self) -> Result<NodeRpcClient> {
|
||||
if let Some(err) = self.offline_bypass_error() {
|
||||
return Err(err);
|
||||
@@ -2968,18 +2634,6 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_msgpack_strict<T: DeserializeOwned>(binary: &[u8]) -> Result<T> {
|
||||
if binary.is_empty() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let mut cursor = Cursor::new(binary);
|
||||
let value = rmp_serde::from_read(&mut cursor).map_err(|_| DiskError::FileCorrupt)?;
|
||||
if cursor.position() != u64::try_from(binary.len()).map_err(Error::other)? {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn decode_multipart_publication_response(binary: &[u8]) -> Result<RenameDataResp> {
|
||||
let result = if binary.len() > MultipartSourceOperation::max_wire_payload_bytes() {
|
||||
Err(DiskError::FileCorrupt)
|
||||
@@ -3177,97 +2831,6 @@ impl RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn start_rename_data_batch(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
items: Vec<RenameBatchClientItem<'_>>,
|
||||
) -> std::result::Result<RemoteRenameBatchStream, RenameBatchStartError> {
|
||||
if self.rename_admission.is_none() || items.is_empty() || items.len() > 16 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::UnsupportedDisk));
|
||||
}
|
||||
let deadline = tokio::time::Instant::now() + get_max_timeout_duration();
|
||||
let mut expected_mask = 0_u16;
|
||||
let mut request_items = Vec::with_capacity(items.len());
|
||||
for item in items {
|
||||
if item.slot >= 16 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
|
||||
}
|
||||
let bit = 1_u16 << item.slot;
|
||||
if expected_mask & bit != 0 {
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Unexpected));
|
||||
}
|
||||
expected_mask |= bit;
|
||||
let item_index = u32::try_from(item.slot)
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let file_info_msgpack = encode_file_info_msgpack(item.file_info).map_err(RenameBatchStartError::NotDispatched)?;
|
||||
request_items.push(RenameDataBatchItem {
|
||||
item_index,
|
||||
disk: item.disk,
|
||||
file_info_msgpack: file_info_msgpack.into(),
|
||||
});
|
||||
}
|
||||
let mut request = Request::new(RenameDataBatchRequest {
|
||||
src_volume: src_volume.to_owned(),
|
||||
src_path: src_path.to_owned(),
|
||||
dst_volume: dst_volume.to_owned(),
|
||||
dst_path: dst_path.to_owned(),
|
||||
items: request_items,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_batch_request_body(request.get_ref())
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body)
|
||||
.map_err(Error::other)
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let mut client = match tokio::time::timeout_at(deadline, self.get_client()).await {
|
||||
Ok(result) => result.map_err(RenameBatchStartError::NotDispatched)?,
|
||||
Err(_) => return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout)),
|
||||
};
|
||||
let admission = self
|
||||
.acquire_rename_admission_until(request.get_ref().items.len(), deadline)
|
||||
.await
|
||||
.map_err(RenameBatchStartError::NotDispatched)?;
|
||||
let guard = RenameBatchCallGuard {
|
||||
admission,
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
let call_timeout = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
if call_timeout == Duration::ZERO {
|
||||
guard.release_without_quarantine();
|
||||
return Err(RenameBatchStartError::NotDispatched(DiskError::Timeout));
|
||||
}
|
||||
let mut guard = guard;
|
||||
guard.mark_dispatched();
|
||||
let response = match self
|
||||
.execute_peer_batch_with_timeout(
|
||||
"rename_data_batch",
|
||||
|| async {
|
||||
match client.rename_data_batch(request).await {
|
||||
Ok(response) => Ok(response),
|
||||
Err(status) if status.code() == Code::Unimplemented => Err(DiskError::UnsupportedDisk),
|
||||
Err(status) => Err(status.into()),
|
||||
}
|
||||
},
|
||||
call_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(DiskError::UnsupportedDisk) => {
|
||||
guard.release_without_quarantine();
|
||||
return Err(RenameBatchStartError::UnsupportedBeforeStream);
|
||||
}
|
||||
Err(error) => return Err(RenameBatchStartError::MayHavePublished(error)),
|
||||
Ok(response) => response,
|
||||
};
|
||||
Ok(guard.into_stream(response.into_inner(), expected_mask))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
@@ -3339,27 +2902,6 @@ impl RemoteDisk {
|
||||
let scanner_publication_lease = scanner_publication_lease_token
|
||||
.map(|token| Bytes::copy_from_slice(token.as_bytes()))
|
||||
.unwrap_or_default();
|
||||
let rename_deadline = tokio::time::Instant::now() + get_max_timeout_duration();
|
||||
let rename_admission = self.acquire_rename_admission_until(1, rename_deadline).await?;
|
||||
let mut rename_admission_guard = RenameBatchCallGuard {
|
||||
admission: rename_admission,
|
||||
deadline: rename_deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
let rpc_kind = if multipart_rename {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_MULTIPART
|
||||
} else if pool_meta_cas {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_POOL_META_CAS
|
||||
} else if quota_fenced {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE
|
||||
} else if incarnation_bound {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_INCARNATION
|
||||
} else if scanner_publication_lease_token.is_some() {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE
|
||||
} else {
|
||||
rustfs_io_metrics::ECSTORE_RENAME_RPC_KIND_ORDINARY
|
||||
};
|
||||
rustfs_io_metrics::record_ecstore_rename_remote_rpc(rpc_kind);
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let mut retried_dual_write = false;
|
||||
let response = loop {
|
||||
@@ -3416,7 +2958,6 @@ impl RemoteDisk {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
|
||||
attempt_prepare_started,
|
||||
);
|
||||
rename_admission_guard.mark_dispatched();
|
||||
let result = rustfs_io_metrics::observe_put_stage_future(
|
||||
async {
|
||||
if multipart_rename {
|
||||
@@ -3447,7 +2988,6 @@ impl RemoteDisk {
|
||||
Err(status) => return Err(status.into()),
|
||||
}
|
||||
};
|
||||
rename_admission_guard.complete();
|
||||
let response_process_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response_headers_started = rustfs_io_metrics::put_stage_timer();
|
||||
let _ = response.metadata().len();
|
||||
@@ -13423,127 +12963,6 @@ mod tests {
|
||||
.expect("shared gate remains open");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_admission_is_weighted_shared_and_released_per_item() {
|
||||
use std::future::Future;
|
||||
use std::task::{Context, Poll, Waker};
|
||||
|
||||
let first = peer_rename_admission("http://rename-peer-a", true).expect("enabled rename gate");
|
||||
let same = peer_rename_admission("http://rename-peer-a", true).expect("same peer rename gate");
|
||||
let other = peer_rename_admission("http://rename-peer-b", true).expect("other peer rename gate");
|
||||
assert!(Arc::ptr_eq(&first, &same));
|
||||
assert!(!Arc::ptr_eq(&first, &other));
|
||||
assert!(peer_rename_admission("http://rename-disabled", false).is_none());
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&first)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let mut grant = RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
};
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
assert!(grant.release_one());
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3);
|
||||
drop(grant);
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let held = Arc::clone(&first)
|
||||
.acquire_many_owned(u32::try_from(rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 3).expect("small limit"))
|
||||
.await
|
||||
.expect("reserve all but three permits");
|
||||
let mut waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
|
||||
assert!(matches!(waiting.as_mut().poll(&mut Context::from_waker(Waker::noop())), Poll::Pending));
|
||||
assert_eq!(first.available_permits(), 0, "the FIFO waiter reserves currently available permits");
|
||||
drop(waiting);
|
||||
assert_eq!(
|
||||
first.available_permits(),
|
||||
3,
|
||||
"cancelling a queued weighted acquire must return its reservation"
|
||||
);
|
||||
let waiting = Box::pin(Arc::clone(&same).acquire_many_owned(4));
|
||||
drop(held);
|
||||
let acquired = tokio::time::timeout(Duration::from_secs(1), waiting)
|
||||
.await
|
||||
.expect("weighted waiter wakes")
|
||||
.expect("rename gate remains open");
|
||||
drop(acquired);
|
||||
assert_eq!(first.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_admission_quarantine_holds_until_absolute_deadline() {
|
||||
let gate = peer_rename_admission("http://rename-quarantine-peer", true).expect("enabled rename gate");
|
||||
let permit = Arc::clone(&gate)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("weighted batch admission");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
|
||||
RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}
|
||||
.quarantine_until(deadline);
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
|
||||
tokio::time::advance(Duration::from_millis(999)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_millis(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rename_call_guard_quarantines_only_after_dispatch() {
|
||||
let gate = peer_rename_admission("http://rename-call-guard-peer", true).expect("enabled rename gate");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("pre-dispatch permits");
|
||||
drop(RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
});
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&gate).acquire_many_owned(4).await.expect("dispatched permits");
|
||||
let mut guard = RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
guard.mark_dispatched();
|
||||
drop(guard);
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT - 4);
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
|
||||
let permit = Arc::clone(&gate)
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.expect("proven pre-mutation permits");
|
||||
let mut guard = RenameBatchCallGuard {
|
||||
admission: Some(RenameAdmissionGrant {
|
||||
permit: Some(permit),
|
||||
batch: true,
|
||||
}),
|
||||
deadline,
|
||||
dispatch_state: RenameAdmissionDispatchState::NotDispatched,
|
||||
};
|
||||
guard.mark_dispatched();
|
||||
guard.release_without_quarantine();
|
||||
assert_eq!(gate.available_permits(), rustfs_protos::INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_disk_builds_optional_complete_body_gate_from_configuration() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, Some("3"))], async {
|
||||
|
||||
@@ -12378,7 +12378,6 @@ struct DecommissionCapacityLockOrderBarrierState {
|
||||
external_heal_target_lock_attempted: tokio::sync::Notify,
|
||||
target_gate_retry_entered: tokio::sync::Notify,
|
||||
target_gate_retry_entries: AtomicUsize,
|
||||
target_gate_retry_pause: std::sync::OnceLock<tokio::sync::Semaphore>,
|
||||
target_gate_inline_retry_entered: tokio::sync::Notify,
|
||||
target_gate_inline_retry_entries: AtomicUsize,
|
||||
target_gate_exact_reloads: AtomicUsize,
|
||||
@@ -12426,7 +12425,6 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
external_heal_target_lock_attempted: tokio::sync::Notify::new(),
|
||||
target_gate_retry_entered: tokio::sync::Notify::new(),
|
||||
target_gate_retry_entries: AtomicUsize::new(0),
|
||||
target_gate_retry_pause: std::sync::OnceLock::new(),
|
||||
target_gate_inline_retry_entered: tokio::sync::Notify::new(),
|
||||
target_gate_inline_retry_entries: AtomicUsize::new(0),
|
||||
target_gate_exact_reloads: AtomicUsize::new(0),
|
||||
@@ -12522,13 +12520,6 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
.expect("decommission entries should observe target gate contention");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn pause_target_gate_retries(&self) -> &tokio::sync::Semaphore {
|
||||
self.state
|
||||
.target_gate_retry_pause
|
||||
.get_or_init(|| tokio::sync::Semaphore::new(0))
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn target_gate_inline_retries(&self) -> usize {
|
||||
self.state.target_gate_inline_retry_entries.load(Ordering::Acquire)
|
||||
@@ -12639,9 +12630,6 @@ impl DecommissionCapacityLockOrderBarrier {
|
||||
#[cfg(test)]
|
||||
impl Drop for DecommissionCapacityLockOrderBarrier {
|
||||
fn drop(&mut self) {
|
||||
if let Some(pause) = self.state.target_gate_retry_pause.get() {
|
||||
pause.close();
|
||||
}
|
||||
self.state.owner_release.notify_one();
|
||||
self.state.external_object_capacity_probe_release.notify_one();
|
||||
self.state.external_object_commit_phase_release.notify_one();
|
||||
@@ -12770,7 +12758,7 @@ async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn observe_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
@@ -12781,11 +12769,6 @@ async fn observe_decommission_target_gate_retry(store_id: uuid::Uuid) {
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.target_gate_retry_entries.fetch_add(1, Ordering::AcqRel);
|
||||
barrier.target_gate_retry_entered.notify_one();
|
||||
if let Some(pause) = barrier.target_gate_retry_pause.get()
|
||||
&& let Ok(permit) = pause.acquire().await
|
||||
{
|
||||
permit.forget();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17655,7 +17638,7 @@ impl ECStore {
|
||||
target_busy_attempt: usize,
|
||||
) -> Result<Option<DecommissionCapacityTargetPermit>> {
|
||||
#[cfg(test)]
|
||||
observe_decommission_target_gate_retry(self.id).await;
|
||||
notify_decommission_target_gate_retry(self.id);
|
||||
let mut wait_attempt = target_busy_attempt;
|
||||
let target_guard = loop {
|
||||
let retry_delay =
|
||||
|
||||
@@ -8360,16 +8360,13 @@ mod decommission_lock_order_tests {
|
||||
.await
|
||||
.expect("hold the target gate before starting both waiters");
|
||||
let retry_observer = DecommissionCapacityLockOrderBarrier::install(other_store.id, other_store.id);
|
||||
// Both workers must observe the original gate as busy, but slow disk I/O
|
||||
// in the holder or first waiter must not consume the second waiter's retry budget.
|
||||
let retry_pause = retry_observer.pause_target_gate_retries();
|
||||
|
||||
let mut workers = tokio::task::JoinSet::new();
|
||||
let mut workers = Vec::new();
|
||||
for object in objects {
|
||||
let worker_store = Arc::clone(&other_store);
|
||||
let worker_bucket = bucket.clone();
|
||||
let source_set = other_store.pools[0].get_disks_by_key(object);
|
||||
workers.spawn(async move {
|
||||
workers.push(tokio::spawn(async move {
|
||||
worker_store
|
||||
.decommission_entry_for_test(
|
||||
0,
|
||||
@@ -8381,7 +8378,7 @@ mod decommission_lock_order_tests {
|
||||
source_set,
|
||||
)
|
||||
.await
|
||||
});
|
||||
}));
|
||||
}
|
||||
retry_observer.wait_until_target_gate_retries(2).await;
|
||||
assert_eq!(
|
||||
@@ -8428,7 +8425,6 @@ mod decommission_lock_order_tests {
|
||||
);
|
||||
|
||||
drop(target_guard);
|
||||
retry_pause.add_permits(1);
|
||||
retry_observer.wait_until_owner_paused().await;
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
@@ -8447,13 +8443,6 @@ mod decommission_lock_order_tests {
|
||||
.expect("a pinned waiter must not hold the alternate target gate");
|
||||
drop(alternate_probe);
|
||||
retry_observer.release_owner();
|
||||
tokio::time::timeout(Duration::from_secs(30), workers.join_next())
|
||||
.await
|
||||
.expect("the pinned waiter should finish before the other retries")
|
||||
.expect("the pinned waiter must exist")
|
||||
.expect("the pinned waiter task should not panic")
|
||||
.expect("the pinned waiter migration should succeed");
|
||||
retry_pause.add_permits(1);
|
||||
retry_observer.wait_until_owner_paused().await;
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
@@ -8473,12 +8462,13 @@ mod decommission_lock_order_tests {
|
||||
drop(original_probe);
|
||||
retry_observer.release_owner();
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), workers.join_next())
|
||||
.await
|
||||
.expect("the reselected waiter should finish after permit transfer")
|
||||
.expect("the reselected waiter must exist")
|
||||
.expect("the reselected waiter task should not panic")
|
||||
.expect("the reselected waiter migration should succeed");
|
||||
for worker in workers {
|
||||
tokio::time::timeout(Duration::from_secs(30), worker)
|
||||
.await
|
||||
.expect("same-target waiter should finish after permit transfer")
|
||||
.expect("same-target waiter task should not panic")
|
||||
.expect("same-target waiter migration should succeed");
|
||||
}
|
||||
assert_eq!(
|
||||
retry_observer.target_gate_exact_reloads(),
|
||||
2,
|
||||
|
||||
@@ -604,10 +604,10 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
|
||||
let disk = match new_disk(ep, opt).await {
|
||||
Ok(disk) => disk,
|
||||
Err(crate::disk::error::DiskError::FileAccessDenied | crate::disk::error::DiskError::VolumeNotFound) => {
|
||||
// Preserve the configured slot when a local volume is unavailable.
|
||||
// LocalDisk logs the cause; format loading retains authority
|
||||
// over fatal errors and quorum. Never create a missing root.
|
||||
Err(crate::disk::error::DiskError::FileAccessDenied) => {
|
||||
// LocalDisk already logged the filesystem cause and repair
|
||||
// hint. Keep this slot unavailable so ECStore initialization
|
||||
// can apply its existing fatal-error and quorum rules.
|
||||
global_local_disk_map.insert(ep.to_string(), None);
|
||||
continue;
|
||||
}
|
||||
@@ -888,95 +888,6 @@ mod tests {
|
||||
disk_map.keys().collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn missing_local_volume_preserves_offline_slot_without_creating_root() {
|
||||
use crate::disk::{DiskOption, error::DiskError};
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
|
||||
for drive_count in [2, 3, 4] {
|
||||
let directory = tempfile::tempdir().expect("create volume parent");
|
||||
let mut endpoints = Vec::new();
|
||||
for index in 0..drive_count {
|
||||
let path = directory.path().join(format!("disk{index}"));
|
||||
if index != 0 {
|
||||
std::fs::create_dir(&path).expect("create surviving volume");
|
||||
}
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
let missing = endpoints[0].to_string();
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: drive_count,
|
||||
endpoints: endpoints.into(),
|
||||
cmd_line: "missing-volume".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let options = DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
};
|
||||
super::initialize_local_disk_maps(&ctx, pools.clone(), &options)
|
||||
.await
|
||||
.expect("missing volume should remain offline for quorum evaluation");
|
||||
let map = ctx.local_disk_map();
|
||||
let map = map.read().await;
|
||||
assert_eq!(map.len(), drive_count);
|
||||
assert!(matches!(map.get(&missing), Some(None)));
|
||||
assert_eq!(map.values().flatten().count(), drive_count - 1);
|
||||
drop(map);
|
||||
let drives = ctx.local_disk_set_drives();
|
||||
let drives = drives.read().await;
|
||||
assert_eq!(drives[0][0].len(), drive_count);
|
||||
assert!(drives[0][0][0].is_none());
|
||||
assert!(drives[0][0][1..].iter().all(Option::is_some));
|
||||
drop(drives);
|
||||
|
||||
let (disks, errors) = crate::store::init_format::init_disks(&ctx, &pools.0[0].endpoints, &options).await;
|
||||
assert!(disks[0].is_none());
|
||||
assert_eq!(errors[0], Some(DiskError::VolumeNotFound));
|
||||
assert!(disks[1..].iter().all(Option::is_some));
|
||||
assert!(errors[1..].iter().all(Option::is_none));
|
||||
assert!(!directory.path().join("disk0").exists(), "startup must not create a missing mount root");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_local_volume_still_fails_registry_initialization() {
|
||||
use crate::disk::DiskOption;
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
|
||||
let directory = tempfile::tempdir().expect("create volume parent");
|
||||
let path = directory.path().join("regular-file");
|
||||
std::fs::write(&path, b"preserve existing file").expect("create invalid volume");
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 1,
|
||||
endpoints: vec![endpoint].into(),
|
||||
cmd_line: "invalid-volume".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let options = DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
};
|
||||
super::initialize_local_disk_maps(&ctx, pools, &options)
|
||||
.await
|
||||
.expect_err("a non-directory volume must retain its initialization error");
|
||||
assert_eq!(std::fs::read(&path).expect("read existing file"), b"preserve existing file");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_invalid_registry_pool_index_has_no_authority_or_registry_effect() {
|
||||
use crate::disk::DiskOption;
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
use crate::diagnostics::admin_server_info::get_local_server_property;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::admin::StorageAdminApi;
|
||||
#[cfg(test)]
|
||||
use chrono::Utc;
|
||||
use jiff::Timestamp;
|
||||
use rustfs_heal_contracts::heal_channel::DriveState;
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
@@ -598,6 +600,10 @@ mod test {
|
||||
use serial_test::serial;
|
||||
use std::time::Duration;
|
||||
|
||||
fn chrono_to_jiff_timestamp(timestamp: chrono::DateTime<Utc>) -> jiff::Timestamp {
|
||||
jiff::Timestamp::try_from(std::time::SystemTime::from(timestamp)).expect("test timestamp should fit in jiff")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tes_types() {
|
||||
let t = MetricType::ALL;
|
||||
@@ -679,10 +685,10 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_mapping_preserves_partial_source_status() {
|
||||
let current_started = Timestamp::now() - jiff::SignedDuration::from_secs(5);
|
||||
let current_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
|
||||
current_cycle_active: true,
|
||||
current_started,
|
||||
current_started: chrono_to_jiff_timestamp(current_started),
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
cycle_timeout_total: 3,
|
||||
@@ -697,7 +703,7 @@ mod test {
|
||||
});
|
||||
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, current_started);
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
assert_eq!(scanner.cycle_timeout_total, 3);
|
||||
@@ -717,8 +723,8 @@ mod test {
|
||||
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
|
||||
let previous_init_time = *rustfs_scanner_metrics::GLOBAL_INIT_TIME.read().await;
|
||||
let previous_cycle = global_metrics().get_cycle().await;
|
||||
let init_time = Timestamp::now() - jiff::SignedDuration::from_hours(1);
|
||||
let cycle_started = Timestamp::now() - jiff::SignedDuration::from_secs(5);
|
||||
let init_time = Utc::now() - chrono::Duration::hours(1);
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
*rustfs_scanner_metrics::GLOBAL_INIT_TIME.write().await = Some(init_time);
|
||||
let cycle = CurrentCycle {
|
||||
current: 0,
|
||||
@@ -742,7 +748,7 @@ mod test {
|
||||
aggregated.merge(decoded);
|
||||
let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
|
||||
assert_eq!(scanner.current_cycle_active, Some(true));
|
||||
assert_eq!(scanner.current_started, cycle_started);
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(cycle_started));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17333,12 +17333,8 @@ mod tests {
|
||||
source.metadata_only = true;
|
||||
source.etag = Some("metadata-copy-etag".to_string());
|
||||
|
||||
let dst_opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..opts.clone()
|
||||
};
|
||||
let copied = set_disks
|
||||
.copy_object(bucket, object, bucket, object, &mut source, &opts, &dst_opts)
|
||||
.copy_object(bucket, object, bucket, object, &mut source, &opts, &opts)
|
||||
.await
|
||||
.expect("metadata self-copy should succeed");
|
||||
assert_eq!(copied.user_defined.get("x-amz-meta-copy-check").map(String::as_str), Some("present"));
|
||||
|
||||
@@ -17182,12 +17182,7 @@ mod metadata_mutation_generation_tests {
|
||||
copy_object,
|
||||
&mut copy_info,
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("metadata COPY should succeed");
|
||||
@@ -17365,11 +17360,7 @@ mod metadata_mutation_generation_tests {
|
||||
let metadata = Arc::make_mut(&mut source.user_defined);
|
||||
metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "new=value".to_string());
|
||||
metadata.insert("expires".to_string(), expires.format(&Rfc3339).expect("expiry"));
|
||||
let dst_opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..Default::default()
|
||||
};
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut source, &opts, &dst_opts)
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut source, &opts, &opts)
|
||||
.await
|
||||
.expect("explicit metadata wins over externalized fields");
|
||||
let mut copied = set.get_object_info(bucket, "object", &opts).await.expect("copied metadata");
|
||||
@@ -17380,7 +17371,7 @@ mod metadata_mutation_generation_tests {
|
||||
copied.metadata_only = true;
|
||||
copied.user_tags = Arc::new(String::new());
|
||||
copied.expires = None;
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut copied, &opts, &dst_opts)
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut copied, &opts, &opts)
|
||||
.await
|
||||
.expect("explicit empty replacement");
|
||||
let cleared = set.get_object_info(bucket, "object", &opts).await.expect("cleared metadata");
|
||||
@@ -17397,10 +17388,6 @@ mod metadata_mutation_generation_tests {
|
||||
}
|
||||
|
||||
let valid_object = "valid-sidecar";
|
||||
let dst_opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..Default::default()
|
||||
};
|
||||
let (mut valid_source, _) = put_and_prime(&set_disks, bucket, valid_object, b"valid sidecar body").await;
|
||||
persist_part_checksum_sidecar(&set_disks, bucket, valid_object, r#"[[1,[["CRC32C","AAAAAA=="]]]]"#).await;
|
||||
valid_source.metadata_only = true;
|
||||
@@ -17413,7 +17400,7 @@ mod metadata_mutation_generation_tests {
|
||||
valid_object,
|
||||
&mut valid_source,
|
||||
&ObjectOptions::default(),
|
||||
&dst_opts,
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("metadata-only copy should preserve a valid checksum sidecar");
|
||||
@@ -17486,7 +17473,7 @@ mod metadata_mutation_generation_tests {
|
||||
conflicting_object,
|
||||
&mut conflicting_source,
|
||||
&ObjectOptions::default(),
|
||||
&dst_opts,
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("metadata-only copy must reject conflicting checksum aliases");
|
||||
@@ -25532,9 +25519,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()),
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
|
||||
@@ -11043,12 +11043,7 @@ mod dangling_read_tests {
|
||||
object,
|
||||
&mut source,
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("a CopyObject source without a version id must resolve like a latest GET");
|
||||
|
||||
@@ -205,12 +205,6 @@ fn heal_format_fence_lost_error() -> Error {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Whether this exact storage instance requires an incarnation-bound write
|
||||
/// for the object currently owned by a bucket-heal scope.
|
||||
pub fn object_requires_incarnation(&self, bucket: &str, object: &str) -> bool {
|
||||
bucket_heal_scope_for_object(bucket, object).is_some_and(|scope| std::ptr::eq(Arc::as_ptr(&scope.store), self))
|
||||
}
|
||||
|
||||
pub async fn purge_delete_marker_with_proof(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
|
||||
@@ -752,60 +752,18 @@ impl ECStore {
|
||||
Ok(ec)
|
||||
}
|
||||
|
||||
async fn init_pool_metadata_for_startup(self: &Arc<Self>) -> Result<(PoolMeta, PoolMetaReplicaState)> {
|
||||
#[instrument(level = "debug", skip(self, rx))]
|
||||
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
runtime_sources::ensure_boot_time().await;
|
||||
|
||||
let should_persist_pool_meta = self
|
||||
.pools
|
||||
.first()
|
||||
.is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
|
||||
// Lock order is save gate -> distributed metadata fence, matching
|
||||
// Lock order is save gate -> distributed metadata writer, matching
|
||||
// runtime mutations. RPC locks are available before ECStore publication
|
||||
// and do not depend on the later fleet proof.
|
||||
let mut startup_write_state = self.pool_meta_save_gate.write().await;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let startup_attempt = uuid::Uuid::new_v4();
|
||||
// A committed snapshot needs only a shared namespace fence. A writer
|
||||
// transition releases this reader and re-reads every observation below
|
||||
// under the existing exclusive fence before applying side effects.
|
||||
if !startup_write_state.bootstrap_identity_proven() && startup_write_state.attested_pool_indices().is_empty() {
|
||||
let pool = self
|
||||
.pools
|
||||
.first()
|
||||
.ok_or_else(|| Error::other("startup has no storage pools"))?;
|
||||
let lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, "pool.bin").await?;
|
||||
let reader = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let load = load_pool_meta_for_startup(self.pools.clone(), &mut startup_write_state);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load);
|
||||
match load.await {
|
||||
Ok((meta, replicas)) => {
|
||||
let topology_update = meta.validate(self.pools.clone())?;
|
||||
let requires_writer = should_persist_pool_meta
|
||||
&& (topology_update
|
||||
|| (replicas.needs_repair && replicas.repair_write_safe)
|
||||
|| startup_write_state.identity_requires_repair());
|
||||
if !topology_update && !requires_writer {
|
||||
startup_write_state.ensure_startup_identity_initialized()?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
crate::core::pools::startup_cas_test_observe(serde_json::json!({
|
||||
"kind": "startup-classifier", "attempt": startup_attempt,
|
||||
"elected_writer": should_persist_pool_meta,
|
||||
"needs_repair": replicas.needs_repair,
|
||||
"repair_write_safe": replicas.repair_write_safe,
|
||||
"topology_update": topology_update,
|
||||
}));
|
||||
self.install_startup_pool_metadata(&meta, Some(&reader)).await?;
|
||||
return Ok((meta, replicas));
|
||||
}
|
||||
}
|
||||
Err(err)
|
||||
if should_persist_pool_meta
|
||||
&& err
|
||||
.pool_metadata_failure()
|
||||
.is_some_and(|failure| failure.kind == crate::error::PoolMetadataFailure::ReadUnavailable) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
drop(reader);
|
||||
}
|
||||
let startup_pool_meta_guard = if should_persist_pool_meta {
|
||||
let pool = self
|
||||
.pools
|
||||
@@ -866,6 +824,7 @@ impl ECStore {
|
||||
"repair_write_safe": pool_meta_replica_state.repair_write_safe,
|
||||
"topology_update": update,
|
||||
}));
|
||||
let endpoints = runtime_sources::endpoint_pools_or_default();
|
||||
|
||||
let mut installed_pool_meta = if update {
|
||||
PoolMeta::new(&self.pools, &meta)?
|
||||
@@ -922,34 +881,19 @@ impl ECStore {
|
||||
installed_pool_meta = persist.await?;
|
||||
}
|
||||
|
||||
self.install_startup_pool_metadata(&installed_pool_meta, startup_pool_meta_guard.as_deref())
|
||||
.await?;
|
||||
if startup_pool_meta_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
|
||||
{
|
||||
return Err(Error::other("startup pool metadata writer fence was lost before publication"));
|
||||
}
|
||||
{
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
*pool_meta = installed_pool_meta.clone();
|
||||
}
|
||||
drop(startup_pool_meta_guard);
|
||||
drop(startup_write_state);
|
||||
|
||||
Ok((installed_pool_meta, pool_meta_replica_state))
|
||||
}
|
||||
|
||||
async fn install_startup_pool_metadata(
|
||||
&self,
|
||||
metadata: &PoolMeta,
|
||||
fence: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<()> {
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
if fence.is_some_and(|guard| guard.is_lock_lost() || guard.is_released()) {
|
||||
return Err(Error::other("startup pool metadata fence was lost before publication"));
|
||||
}
|
||||
*pool_meta = metadata.clone();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, rx))]
|
||||
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
runtime_sources::ensure_boot_time().await;
|
||||
|
||||
let (installed_pool_meta, pool_meta_replica_state) = self.init_pool_metadata_for_startup().await?;
|
||||
let endpoints = runtime_sources::endpoint_pools_or_default();
|
||||
|
||||
resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?;
|
||||
let rebalance_resume_required = {
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
@@ -3787,89 +3731,6 @@ mod tests {
|
||||
(instance_ctx, store, shutdown)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn committed_startup_pool_metadata_uses_shared_fence_without_disk_writes() {
|
||||
use crate::disk::{FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking;
|
||||
|
||||
for reopen in [false, true] {
|
||||
let directory = tempfile::tempdir().expect("create committed metadata deployment");
|
||||
let (_, seeded, seed_shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(directory.path(), "startup-read-fence", &[4])).await;
|
||||
let (store, shutdown) = if reopen {
|
||||
seed_shutdown.cancel();
|
||||
drop(seeded);
|
||||
let (_, reopened, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(directory.path(), "startup-read-fence", &[4])).await;
|
||||
(reopened, shutdown)
|
||||
} else {
|
||||
(seeded, seed_shutdown)
|
||||
};
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
let snapshot = || {
|
||||
let mut files = std::collections::BTreeMap::new();
|
||||
for (index, slot) in disks.iter().enumerate() {
|
||||
let disk = slot.as_ref().expect("committed drive handle");
|
||||
for object in [POOL_META_NAME, POOL_META_IDENTITY_NAME, FORMAT_CONFIG_FILE] {
|
||||
let root = disk.path().join(RUSTFS_META_BUCKET).join(object);
|
||||
assert!(root.exists(), "committed authority must be present: {root:?}");
|
||||
let mut pending = vec![root];
|
||||
while let Some(path) = pending.pop() {
|
||||
let metadata = std::fs::symlink_metadata(&path).expect("inspect authority file type");
|
||||
assert!(!metadata.file_type().is_symlink(), "authority fixture must not follow symlinks");
|
||||
if metadata.is_dir() {
|
||||
for entry in std::fs::read_dir(&path).expect("inspect authority object directory") {
|
||||
pending.push(entry.expect("authority directory entry").path());
|
||||
}
|
||||
} else {
|
||||
assert!(metadata.is_file(), "authority fixture must contain regular files");
|
||||
let contents = std::fs::read(&path).expect("read physical authority bytes");
|
||||
let relative = path
|
||||
.strip_prefix(disk.path())
|
||||
.expect("authority stays under fixture drive")
|
||||
.to_path_buf();
|
||||
files.insert((index, relative), contents);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
files
|
||||
};
|
||||
let before = snapshot();
|
||||
let lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("metadata namespace lock");
|
||||
let mut reader = lock
|
||||
.get_read_lock(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("hold a competing metadata reader");
|
||||
let (metadata, replicas) = tokio::time::timeout(Duration::from_secs(2), store.init_pool_metadata_for_startup())
|
||||
.await
|
||||
.expect("committed startup must not wait for an exclusive lock")
|
||||
.expect("committed startup metadata is readable under a shared fence");
|
||||
assert_eq!(metadata.pools[0].cmd_line, "startup-read-fence-pool-0");
|
||||
assert!(!replicas.needs_repair);
|
||||
assert!(!reader.is_lock_lost());
|
||||
let after = snapshot();
|
||||
assert_eq!(
|
||||
after, before,
|
||||
"shared startup observation cannot mutate metadata, identity, or format authority"
|
||||
);
|
||||
assert!(reader.release(), "release the snapshot's shared fence");
|
||||
let mut invalid = metadata.clone();
|
||||
invalid.pools[0].cmd_line = "must-not-be-published".to_string();
|
||||
store
|
||||
.install_startup_pool_metadata(&invalid, Some(&reader))
|
||||
.await
|
||||
.expect_err("a released fence cannot publish startup metadata");
|
||||
assert_eq!(store.pool_meta.read().await.pools[0].cmd_line, metadata.pools[0].cmd_line);
|
||||
drop(reader);
|
||||
shutdown.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn interrupted_format_bootstrap_recovers_two_pools_through_store_init() {
|
||||
|
||||
@@ -169,34 +169,36 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
{
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
let native_read_candidate = if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum))) {
|
||||
select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let partial_native_with_blanks = formats_present > 0
|
||||
if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum)))
|
||||
&& let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
{
|
||||
retain_format_quorum_members(
|
||||
instance_ctx,
|
||||
disks,
|
||||
&format,
|
||||
&members,
|
||||
Some(&unformatted_bootstrap_routes),
|
||||
set_drive_count,
|
||||
)
|
||||
.await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
});
|
||||
}
|
||||
let resumable_partial_migration = formats_present > 0
|
||||
&& errs.iter().any(|error| matches!(error, Some(DiskError::UnformattedDisk)))
|
||||
&& format_quorum.as_ref().is_some_and(Result::is_err);
|
||||
let resumable_partial_migration = partial_native_with_blanks
|
||||
&& format_quorum.as_ref().is_some_and(Result::is_err)
|
||||
&& formats.iter().zip(&errs).all(|(format, error)| {
|
||||
format.is_some() || matches!(error, Some(DiskError::UnformattedDisk | DiskError::DiskNotFound))
|
||||
});
|
||||
|
||||
let allow_migration = first_disk && (all_unformatted || resumable_partial_migration);
|
||||
if partial_native_with_blanks && !allow_migration && native_read_candidate.is_none() {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
if allow_migration || (partial_native_with_blanks && native_read_candidate.is_some()) {
|
||||
if first_disk && (all_unformatted || resumable_partial_migration) {
|
||||
info!(all_unformatted, resumable_partial_migration, "checking for a legacy storage format");
|
||||
let mode = match native_read_candidate.as_ref() {
|
||||
Some((reference, _)) => LegacyFormatMode::NativeRead {
|
||||
reference,
|
||||
native_errors: &errs,
|
||||
allow_migration,
|
||||
},
|
||||
None => LegacyFormatMode::Migration,
|
||||
};
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count, mode).await {
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => {
|
||||
info!("Migrated format from MinIO config");
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, None, set_drive_count).await?;
|
||||
@@ -215,10 +217,10 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
);
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::ReadCompatible) => {}
|
||||
Ok(LegacyFormatOutcome::None) => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
if first_disk && fresh_bootstrap_proven {
|
||||
if fresh_bootstrap_proven {
|
||||
let fm = init_format_erasure(instance_ctx, disks, set_count, set_drive_count, deployment_id).await?;
|
||||
return Ok(LoadedFormat {
|
||||
format: fm,
|
||||
@@ -227,26 +229,6 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve legacy migration before admitting native reads. Blank replacement
|
||||
// slots contribute no votes and receive no formatting/bootstrap authority.
|
||||
if let Some((format, members)) = native_read_candidate {
|
||||
retain_format_quorum_members(
|
||||
instance_ctx,
|
||||
disks,
|
||||
&format,
|
||||
&members,
|
||||
Some(&unformatted_bootstrap_routes),
|
||||
set_drive_count,
|
||||
)
|
||||
.await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
});
|
||||
}
|
||||
info!(
|
||||
"first_disk: {}, should_init_erasure_disks: {}",
|
||||
first_disk,
|
||||
@@ -289,11 +271,9 @@ async fn verified_legacy_adoption_source(
|
||||
set_count: usize,
|
||||
set_drive_count: usize,
|
||||
) -> Result<PoolMetaBootstrapAuthority> {
|
||||
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count, LegacyFormatMode::Migration).await {
|
||||
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { .. }) => Ok(PoolMetaBootstrapAuthority::LegacyAdoption),
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::ReadCompatible | LegacyFormatOutcome::Incompatible) => {
|
||||
Ok(PoolMetaBootstrapAuthority::None)
|
||||
}
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::Incompatible) => Ok(PoolMetaBootstrapAuthority::None),
|
||||
Err(err) => {
|
||||
debug!(
|
||||
error = %err,
|
||||
@@ -463,49 +443,25 @@ enum LegacyFormatOutcome {
|
||||
/// version mismatch, or a parse failure). The caller must decide how to
|
||||
/// proceed; creating a fresh format would leave the legacy objects unreadable.
|
||||
Incompatible,
|
||||
/// Observed legacy formats agree with the native read quorum, without granting writer authority.
|
||||
ReadCompatible,
|
||||
/// No MinIO `format.json` was present on any disk (a normal fresh install).
|
||||
None,
|
||||
}
|
||||
|
||||
/// Native reads validate observed legacy identities without electing a migration writer.
|
||||
enum LegacyFormatMode<'a> {
|
||||
Migration,
|
||||
NativeRead {
|
||||
reference: &'a FormatV3,
|
||||
native_errors: &'a [Option<DiskError>],
|
||||
allow_migration: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// Tries to migrate an on-disk MinIO `format.json` into RustFS format files.
|
||||
///
|
||||
/// Returns [`LegacyFormatOutcome`] describing whether a legacy format was present
|
||||
/// and, if so, whether it was compatible. `Err` is returned for read or persist
|
||||
/// failures that prevent a conclusive migration decision. Native read admission
|
||||
/// checks every observed legacy format against its existing layout and tolerates
|
||||
/// unavailable nonvoters. Migration requires complete observations and a strict majority.
|
||||
/// failures that prevent a conclusive migration decision.
|
||||
async fn try_migrate_format(
|
||||
disks: &[Option<DiskStore>],
|
||||
rustfs_formats: &[Option<FormatV3>],
|
||||
set_count: usize,
|
||||
set_drive_count: usize,
|
||||
mode: LegacyFormatMode<'_>,
|
||||
) -> Result<LegacyFormatOutcome> {
|
||||
let (native_reference, native_errors, allow_migration) = match mode {
|
||||
LegacyFormatMode::Migration => (None, None, true),
|
||||
LegacyFormatMode::NativeRead {
|
||||
reference,
|
||||
native_errors,
|
||||
allow_migration,
|
||||
} => (Some(reference), Some(native_errors), allow_migration),
|
||||
};
|
||||
let legacy_format_max_bytes = legacy_format_max_bytes(disks.len())?;
|
||||
let mut legacy_seen = false;
|
||||
let mut invalid_legacy_format = false;
|
||||
let mut read_error: Option<Error> = None;
|
||||
let mut unavailable_error = None;
|
||||
let mut legacy_formats = vec![None; disks.len()];
|
||||
|
||||
let reads = stream::iter(
|
||||
@@ -517,13 +473,6 @@ async fn try_migrate_format(
|
||||
let Some(disk) = disk else {
|
||||
return (index, None);
|
||||
};
|
||||
// Native format loading already proved this peer unavailable.
|
||||
// Reuse that observation and keep it ineligible for migration.
|
||||
if let Some(error) = native_errors.and_then(|errors| errors.get(index)).and_then(Option::as_ref)
|
||||
&& is_network_like_disk_error(error)
|
||||
{
|
||||
return (index, Some(Err(error.clone())));
|
||||
}
|
||||
let reader = match disk.read_file(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE).await {
|
||||
Ok(reader) => reader,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return (index, None),
|
||||
@@ -563,9 +512,6 @@ async fn try_migrate_format(
|
||||
match read {
|
||||
Ok(Some(format)) => legacy_formats[index] = Some(format),
|
||||
Ok(None) => invalid_legacy_format = true,
|
||||
Err(err) if native_reference.is_some() && is_network_like_disk_error(&err) => {
|
||||
unavailable_error.get_or_insert(err);
|
||||
}
|
||||
Err(err) => {
|
||||
read_error.get_or_insert_with(|| err.into());
|
||||
}
|
||||
@@ -582,15 +528,6 @@ async fn try_migrate_format(
|
||||
return Ok(LegacyFormatOutcome::Incompatible);
|
||||
}
|
||||
|
||||
if let Some(reference) = native_reference {
|
||||
if !formats_match_reference_slots(&legacy_formats, reference, 0) {
|
||||
return Ok(LegacyFormatOutcome::Incompatible);
|
||||
}
|
||||
if !allow_migration || unavailable_error.is_some() || legacy_formats.iter().flatten().count() <= disks.len() / 2 {
|
||||
return Ok(LegacyFormatOutcome::ReadCompatible);
|
||||
}
|
||||
}
|
||||
|
||||
let format = match get_format_erasure_in_quorum(&legacy_formats, 0) {
|
||||
Ok(format) => format,
|
||||
Err(_) => return Ok(LegacyFormatOutcome::Incompatible),
|
||||
@@ -740,7 +677,7 @@ fn select_existing_format_read_quorum(
|
||||
|| check_format_erasure_value_for_topology(reference, formats.len(), set_drive_count).is_err()
|
||||
|| formats.iter().zip(errors).any(|(format, error)| match (format, error) {
|
||||
(Some(_), None) => false,
|
||||
(None, Some(DiskError::DiskNotFound | DiskError::UnformattedDisk)) => false,
|
||||
(None, Some(DiskError::DiskNotFound)) => false,
|
||||
(None, Some(error)) => !is_network_like_disk_error(error),
|
||||
_ => true,
|
||||
})
|
||||
@@ -1508,23 +1445,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn existing_format_read_quorum_excludes_blank_and_network_unavailable_slots() {
|
||||
let (formats, mut errors) = half_pool_formats();
|
||||
errors[2] = Some(DiskError::UnformattedDisk);
|
||||
errors[3] = Some(DiskError::Timeout);
|
||||
let (_, members) = select_existing_format_read_quorum(&formats, &errors, 4)
|
||||
.expect("blank and unreachable slots cannot block an existing native read quorum");
|
||||
assert!(!members[2] && !members[3]);
|
||||
assert_eq!(members.iter().filter(|member| **member).count(), 12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn existing_format_read_quorum_rejects_conflicts_and_invalid_slots() {
|
||||
fn existing_format_read_quorum_rejects_conflicts_invalid_slots_and_blank_disks() {
|
||||
for failure in [
|
||||
"conflict",
|
||||
"wrong_slot",
|
||||
"nil",
|
||||
"duplicate",
|
||||
"blank",
|
||||
"corrupt",
|
||||
"access_denied",
|
||||
"tie",
|
||||
@@ -1546,6 +1473,7 @@ mod tests {
|
||||
format.erasure.sets[5][3] = format.erasure.sets[0][0];
|
||||
}
|
||||
}
|
||||
"blank" => errors[2] = Some(DiskError::UnformattedDisk),
|
||||
"corrupt" => errors[2] = Some(DiskError::CorruptedFormat),
|
||||
"access_denied" => errors[2] = Some(DiskError::FileAccessDenied),
|
||||
"tie" => {
|
||||
@@ -1604,186 +1532,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_format_read_quorum_quarantines_blank_replacement_without_writes() {
|
||||
for first_disk in [false, true] {
|
||||
for surviving_slot in 0..2 {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
|
||||
let blank = 1 - surviving_slot;
|
||||
let mut format = FormatV3::new(1, 2);
|
||||
format.erasure.this = format.erasure.sets[0][surviving_slot];
|
||||
save_format_file(&disks[surviving_slot], &Some(format.clone()))
|
||||
.await
|
||||
.expect("persist surviving format");
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&original).await;
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(
|
||||
&Arc::new(InstanceContext::new()),
|
||||
first_disk,
|
||||
&mut disks,
|
||||
1,
|
||||
2,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("existing native format must load at read quorum with a blank replacement");
|
||||
assert_eq!(loaded.format.shared_identity(), format.shared_identity());
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[surviving_slot].is_some());
|
||||
assert!(disks[blank].is_none(), "blank replacement cannot vote or serve data");
|
||||
assert_eq!(format_bytes(&original).await, before, "read admission cannot write either format");
|
||||
assert!(matches!(
|
||||
load_format_erasure(original[blank].as_ref().expect("blank handle"), false).await,
|
||||
Err(DiskError::UnformattedDisk)
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_read_quorum_with_blank_and_unavailable_slots_validates_legacy_without_writes() {
|
||||
for first_disk in [false, true] {
|
||||
for scenario in [
|
||||
"native-remote",
|
||||
"legacy-missing",
|
||||
"foreign-remote",
|
||||
"corrupt-remote",
|
||||
"wrong-slot-missing",
|
||||
] {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(4, 4).await;
|
||||
let reference = FormatV3::new(1, 4);
|
||||
for slot in 0..2 {
|
||||
let mut format = reference.clone();
|
||||
format.erasure.this = format.erasure.sets[0][slot];
|
||||
save_format_file(&disks[slot], &Some(format.clone()))
|
||||
.await
|
||||
.expect("persist native read quorum");
|
||||
if scenario != "native-remote" {
|
||||
write_legacy_format(&disks[slot], &format).await;
|
||||
}
|
||||
}
|
||||
match scenario {
|
||||
"foreign-remote" => {
|
||||
let mut foreign = FormatV3::new(1, 4);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][2];
|
||||
write_legacy_format(&disks[2], &foreign).await;
|
||||
}
|
||||
"corrupt-remote" => write_legacy_bytes(&disks[2], bytes::Bytes::from_static(b"corrupt legacy")).await,
|
||||
"wrong-slot-missing" => {
|
||||
let mut wrong_slot = reference.clone();
|
||||
wrong_slot.erasure.this = reference.erasure.sets[0][0];
|
||||
write_legacy_format(&disks[2], &wrong_slot).await;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&original).await;
|
||||
if scenario.ends_with("remote") {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve unavailable peer address");
|
||||
let address = listener.local_addr().expect("peer address");
|
||||
drop(listener);
|
||||
let mut endpoint = Endpoint::try_from(format!("http://{address}/disk").as_str()).expect("remote endpoint");
|
||||
endpoint.is_local = false;
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(3);
|
||||
let remote = crate::cluster::rpc::RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
Arc::new(crate::cluster::rpc::TcpHttpInternodeDataTransport),
|
||||
)
|
||||
.await
|
||||
.expect("construct unavailable remote handle");
|
||||
let remote = Arc::new(crate::disk::Disk::Remote(Box::new(remote)));
|
||||
let error = remote
|
||||
.read_all(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
|
||||
.await
|
||||
.expect_err("remote peer must be unavailable");
|
||||
assert!(is_network_like_disk_error(&error), "expected unavailable peer, got {error:?}");
|
||||
disks[3] = Some(remote);
|
||||
} else {
|
||||
disks[3] = None;
|
||||
}
|
||||
let result = connect_load_init_formats_with_instance_ctx(
|
||||
&Arc::new(InstanceContext::new()),
|
||||
first_disk,
|
||||
&mut disks,
|
||||
1,
|
||||
4,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
if matches!(scenario, "native-remote" | "legacy-missing") {
|
||||
let loaded = result.unwrap_or_else(|error| panic!("{scenario} must admit native reads: {error:?}"));
|
||||
assert_eq!(loaded.format.shared_identity(), reference.shared_identity());
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[..2].iter().all(Option::is_some));
|
||||
assert!(disks[2..].iter().all(Option::is_none), "unavailable and blank slots cannot vote");
|
||||
} else {
|
||||
assert!(
|
||||
matches!(result, Err(Error::CorruptedFormat)),
|
||||
"{scenario} observed legacy evidence must reject reads"
|
||||
);
|
||||
}
|
||||
assert_eq!(format_bytes(&original).await, before, "{scenario} cannot write native formats");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_native_format_with_legacy_quorum_reads_without_migration_writes() {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
|
||||
let legacy = FormatV3::new(1, 2);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut native = legacy.clone();
|
||||
native.erasure.this = native.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(native))
|
||||
.await
|
||||
.expect("persist partial native format");
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&disks).await;
|
||||
let loaded =
|
||||
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), false, &mut disks, 1, 2, None)
|
||||
.await
|
||||
.expect("follower can read the existing native layout without migrating legacy formats");
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[0].is_some() && disks[1].is_none(), "legacy-only slots cannot vote for native reads");
|
||||
assert_eq!(format_bytes(&original).await, before, "follower legacy inspection cannot persist formats");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn follower_below_native_read_quorum_skips_legacy_reads_without_writes() {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(3, 3).await;
|
||||
let mut format = FormatV3::new(1, 3);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(format))
|
||||
.await
|
||||
.expect("persist one native format below the two-drive read quorum");
|
||||
for disk in &disks {
|
||||
write_legacy_bytes(disk, bytes::Bytes::from_static(b"invalid legacy format")).await;
|
||||
}
|
||||
let before = format_bytes(&disks).await;
|
||||
|
||||
let result = connect_load_init_formats(false, &mut disks, 1, 3, None).await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(Error::ErasureReadQuorum)),
|
||||
"a non-reader must reject quorum before probing corrupt legacy files"
|
||||
);
|
||||
assert_eq!(format_bytes(&disks).await, before);
|
||||
for disk in disks.iter().flatten() {
|
||||
assert_eq!(
|
||||
disk.read_all(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE)
|
||||
.await
|
||||
.expect("legacy bytes must remain untouched"),
|
||||
bytes::Bytes::from_static(b"invalid legacy format")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_format_load_succeeds_with_a_strict_majority() {
|
||||
let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await;
|
||||
@@ -2627,7 +2375,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_format_migration_resumes_partial_rustfs_writes() {
|
||||
for drive_count in [2, 3, 4] {
|
||||
for drive_count in [3, 4] {
|
||||
let (_temp_dir, mut disks) = local_disks(drive_count).await;
|
||||
let legacy = FormatV3::new(1, drive_count);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
@@ -2686,32 +2434,23 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_format_migration_rejects_a_foreign_partial_rustfs_format() {
|
||||
for first_disk in [false, true] {
|
||||
for drive_count in [2, 3] {
|
||||
let (_temp_dir, mut disks) = local_disks(drive_count).await;
|
||||
let legacy = FormatV3::new(1, drive_count);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut foreign = FormatV3::new(1, drive_count);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][0];
|
||||
let foreign_id = foreign.id;
|
||||
save_format_file(&disks[0], &Some(foreign))
|
||||
.await
|
||||
.expect("foreign partial RustFS format should be written");
|
||||
let (_temp_dir, mut disks) = local_disks(3).await;
|
||||
let legacy = FormatV3::new(1, 3);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut foreign = FormatV3::new(1, 3);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][0];
|
||||
let foreign_id = foreign.id;
|
||||
save_format_file(&disks[0], &Some(foreign))
|
||||
.await
|
||||
.expect("foreign partial RustFS format should be written");
|
||||
|
||||
let result = connect_load_init_formats(first_disk, &mut disks, 1, drive_count, None).await;
|
||||
if !first_disk && drive_count == 3 {
|
||||
assert!(matches!(result, Err(Error::ErasureReadQuorum)), "a follower below read quorum must wait");
|
||||
} else {
|
||||
assert!(
|
||||
matches!(result, Err(Error::CorruptedFormat)),
|
||||
"an eligible reader or migration writer must reject the foreign legacy layout"
|
||||
);
|
||||
}
|
||||
let (formats, _) = load_format_erasure_all(&disks, false).await;
|
||||
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
|
||||
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
|
||||
}
|
||||
}
|
||||
assert!(matches!(
|
||||
connect_load_init_formats(true, &mut disks, 1, 3, None).await,
|
||||
Err(Error::CorruptedFormat)
|
||||
));
|
||||
let (formats, _) = load_format_erasure_all(&disks, false).await;
|
||||
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
|
||||
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -83,7 +83,6 @@ rustfs-common = { workspace = true }
|
||||
rustfs-heal-contracts = { workspace = true }
|
||||
rustfs-madmin = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync", "io-util", "time", "macros", "fs", "rt-multi-thread"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
|
||||
tonic = { workspace = true }
|
||||
@@ -111,7 +110,7 @@ walkdir = { workspace = true }
|
||||
http = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
jiff = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -2871,7 +2871,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn w13_timestamp() -> String {
|
||||
jiff::Timestamp::now().strftime("%Y-%m-%dT%H:%M:%SZ").to_string()
|
||||
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
|
||||
}
|
||||
|
||||
fn w13_selection_contains(selection: &str, lane: &str) -> bool {
|
||||
|
||||
+126
-92
@@ -18,7 +18,7 @@ use crate::sys::{get_claims_from_token_with_secret, get_claims_from_token_with_s
|
||||
use crate::{
|
||||
cache::{Cache, CacheEntity, LockedCache},
|
||||
error::{Error as IamError, is_err_no_such_group, is_err_no_such_policy, is_err_no_such_user},
|
||||
store::{GroupEdit, GroupInfo, MappedPolicy, Store, UserType, object::IAM_CONFIG_PREFIX},
|
||||
store::{GroupInfo, MappedPolicy, Store, UserType, object::IAM_CONFIG_PREFIX},
|
||||
sys::{
|
||||
MAX_SVCSESSION_POLICY_SIZE, SESSION_POLICY_NAME, SESSION_POLICY_NAME_EXTRACTED, SITE_REPLICATOR_CLAIM,
|
||||
SITE_REPLICATOR_SERVICE_ACCOUNT, STATUS_DISABLED, STATUS_ENABLED, UpdateServiceAccountOpts,
|
||||
@@ -1122,9 +1122,7 @@ where
|
||||
|
||||
let direct_policies = mapped_policy.to_slice();
|
||||
let has_identity_mapping = !direct_policies.is_empty();
|
||||
// An uncached group mapping can contain a Deny even when a direct policy exists.
|
||||
let mut policies = direct_policies;
|
||||
policies.extend(self.policy_db_get_with_groups(Vec::new(), groups).await?);
|
||||
let policies = self.policy_db_get_with_groups(direct_policies, groups).await?;
|
||||
Ok((policies, has_identity_mapping))
|
||||
}
|
||||
|
||||
@@ -1580,7 +1578,7 @@ where
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
self.add_user_at_with_secret_change(access_key, args, updated_at, true)
|
||||
self.add_user_at_with_secret_change(access_key, args, updated_at)
|
||||
.await
|
||||
.map(|(updated_at, _, _guard)| updated_at)
|
||||
}
|
||||
@@ -1591,7 +1589,6 @@ where
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
allow_status_change: bool,
|
||||
) -> Result<(OffsetDateTime, bool, tokio::sync::OwnedMutexGuard<crate::cache::MutationLockState>)> {
|
||||
// A virtual OIDC parent is a canonical base64url-encoded SHA-256 digest.
|
||||
// It has no stored user record to collide with in the checks below.
|
||||
@@ -1649,14 +1646,9 @@ where
|
||||
});
|
||||
user_entry.update_at = Some(updated_at);
|
||||
|
||||
let secret_changed = if allow_status_change {
|
||||
self.api
|
||||
.save_user_identity(access_key, UserType::Reg, user_entry.clone(), None)
|
||||
.await?;
|
||||
secret_changed
|
||||
} else {
|
||||
self.api.save_regular_user_preserving_status(access_key, &user_entry).await?
|
||||
};
|
||||
self.api
|
||||
.save_user_identity(access_key, UserType::Reg, user_entry.clone(), None)
|
||||
.await?;
|
||||
|
||||
self.update_user_with_claims(access_key, user_entry)?;
|
||||
|
||||
@@ -2001,25 +1993,38 @@ where
|
||||
// whose time predates the entity's load time — a replicated edit
|
||||
// whose source time is older than this node's startup would
|
||||
// otherwise never reach the cache.
|
||||
drop(cache);
|
||||
let (edit, current) = self.api.begin_group_edit(group).await?;
|
||||
let gi = match current {
|
||||
Some(mut gi) => {
|
||||
let gi = match cache.groups.get(group) {
|
||||
Some(res) => {
|
||||
let mut gi = res.clone();
|
||||
let mut uniq_set: HashSet<String, std::collections::hash_map::RandomState> =
|
||||
HashSet::from_iter(gi.members.iter().cloned());
|
||||
uniq_set.extend(members);
|
||||
uniq_set.extend(members.iter().cloned());
|
||||
|
||||
gi.members = uniq_set.into_iter().collect();
|
||||
gi.update_at = Some(updated_at);
|
||||
gi
|
||||
}
|
||||
None => {
|
||||
let mut gi = GroupInfo::new(members);
|
||||
let mut gi = GroupInfo::new(members.clone());
|
||||
gi.update_at = Some(updated_at);
|
||||
gi
|
||||
}
|
||||
};
|
||||
self.save_group_and_publish(group, &*edit, gi).await?;
|
||||
drop(cache);
|
||||
|
||||
self.api.save_group_info(group, gi.clone()).await?;
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(group, &gi, now);
|
||||
|
||||
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
|
||||
members.iter().for_each(|member| {
|
||||
let mut m = user_group_memberships.get(member).cloned().unwrap_or_default();
|
||||
m.insert(group.to_string());
|
||||
cache.add_or_update_user_group_membership(member, &m, now);
|
||||
});
|
||||
});
|
||||
|
||||
Ok(updated_at)
|
||||
}
|
||||
@@ -2035,8 +2040,13 @@ where
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
|
||||
let (edit, current) = self.api.begin_group_edit(name).await?;
|
||||
let mut gi = current.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
|
||||
let cache = self.cache.snapshot();
|
||||
let groups = Arc::clone(&cache.groups);
|
||||
let mut gi = match groups.get(name) {
|
||||
Some(gi) => gi.clone(),
|
||||
None => return Err(Error::NoSuchGroup(name.to_string())),
|
||||
};
|
||||
drop(cache);
|
||||
|
||||
if enable {
|
||||
gi.status = STATUS_ENABLED.to_owned();
|
||||
@@ -2045,7 +2055,11 @@ where
|
||||
}
|
||||
gi.update_at = Some(updated_at);
|
||||
|
||||
self.save_group_and_publish(name, &*edit, gi).await?;
|
||||
self.api.save_group_info(name, gi.clone()).await?;
|
||||
|
||||
// Cache publication time is the local clock, not the record stamp
|
||||
// (see `add_users_to_group_at`).
|
||||
self.cache.add_or_update_group(name, &gi, OffsetDateTime::now_utc());
|
||||
|
||||
Ok(updated_at)
|
||||
}
|
||||
@@ -2132,27 +2146,37 @@ where
|
||||
update_cache_only: bool,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let removed: HashSet<_> = members.into_iter().collect();
|
||||
let remove = |mut gi: GroupInfo| {
|
||||
gi.members.retain(|m| !removed.contains(m));
|
||||
gi.update_at = Some(updated_at);
|
||||
gi
|
||||
};
|
||||
if update_cache_only {
|
||||
return self.cache.with_write_lock(|cache| {
|
||||
let gi = cache
|
||||
.state()
|
||||
.groups
|
||||
.get(name)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
|
||||
self.publish_group_unlocked(cache, name, &remove(gi));
|
||||
Ok(updated_at)
|
||||
});
|
||||
let cache = self.cache.snapshot();
|
||||
let mut gi = cache
|
||||
.groups
|
||||
.get(name)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
|
||||
drop(cache);
|
||||
|
||||
let s: HashSet<&String> = HashSet::from_iter(gi.members.iter());
|
||||
let d: HashSet<&String> = HashSet::from_iter(members.iter());
|
||||
gi.members = s.difference(&d).map(|v| v.to_string()).collect::<Vec<String>>();
|
||||
gi.update_at = Some(updated_at);
|
||||
if !update_cache_only {
|
||||
self.api.save_group_info(name, gi.clone()).await?;
|
||||
}
|
||||
let (edit, current) = self.api.begin_group_edit(name).await?;
|
||||
let gi = current.ok_or_else(|| Error::NoSuchGroup(name.to_string()))?;
|
||||
self.save_group_and_publish(name, &*edit, remove(gi)).await?;
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
// Sample after storage completes so a concurrent reload cannot
|
||||
// make this publication older than the cache it must update.
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(name, &gi, now);
|
||||
|
||||
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
|
||||
members.iter().for_each(|member| {
|
||||
if let Some(m) = user_group_memberships.get(member) {
|
||||
let mut m = m.clone();
|
||||
m.remove(name);
|
||||
cache.add_or_update_user_group_membership(member, &m, now);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Ok(updated_at)
|
||||
}
|
||||
@@ -2177,6 +2201,7 @@ where
|
||||
|
||||
let cache = self.cache.snapshot();
|
||||
let users_cache = Arc::clone(&cache.users);
|
||||
let groups_cache = Arc::clone(&cache.groups);
|
||||
drop(cache);
|
||||
|
||||
for member in members.iter() {
|
||||
@@ -2189,60 +2214,65 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
if !members.is_empty() {
|
||||
return self.remove_members_from_group_at(group, members, false, updated_at).await;
|
||||
}
|
||||
let (edit, current) = self.api.begin_group_edit(group).await?;
|
||||
let gi = current.ok_or_else(|| Error::NoSuchGroup(group.to_string()))?;
|
||||
if !gi.members.is_empty() {
|
||||
let gi = if members.is_empty() {
|
||||
// Reload from backend so we see latest members (e.g. after user was deleted elsewhere)
|
||||
let mut m = HashMap::new();
|
||||
self.api.load_group(group, &mut m).await?;
|
||||
m.get(group).cloned().ok_or_else(|| Error::NoSuchGroup(group.to_string()))?
|
||||
} else {
|
||||
groups_cache
|
||||
.get(group)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::NoSuchGroup(group.to_string()))?
|
||||
};
|
||||
|
||||
if members.is_empty() && !gi.members.is_empty() {
|
||||
return Err(Error::GroupNotEmpty);
|
||||
}
|
||||
|
||||
edit.delete_empty().await?;
|
||||
if members.is_empty() {
|
||||
if let Err(err) = self.api.delete_mapped_policy(group, UserType::Reg, true).await
|
||||
&& !is_err_no_such_policy(&err)
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
Self::replace_cached_group_members(cache, group, &[], now);
|
||||
cache.delete_group(group, now);
|
||||
cache.delete_group_policy(group, now);
|
||||
});
|
||||
if let Err(err) = self.api.delete_group_info(group).await
|
||||
&& !is_err_no_such_group(&err)
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(updated_at)
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
self.remove_group_from_memberships_map_unlocked(cache, group, now);
|
||||
cache.delete_group(group, now);
|
||||
cache.delete_group_policy(group, now);
|
||||
});
|
||||
|
||||
return Ok(updated_at);
|
||||
}
|
||||
|
||||
self.remove_members_from_group_at(group, members, false, updated_at).await
|
||||
}
|
||||
|
||||
async fn save_group_and_publish(&self, group: &str, edit: &dyn GroupEdit, gi: GroupInfo) -> Result<()> {
|
||||
edit.save(&gi).await?;
|
||||
self.cache
|
||||
.with_write_lock(|cache| self.publish_group_unlocked(cache, group, &gi));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn publish_group_unlocked(&self, cache: &mut LockedCache, group: &str, gi: &GroupInfo) {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(group, gi, now);
|
||||
Self::replace_cached_group_members(cache, group, &gi.members, now);
|
||||
}
|
||||
|
||||
fn replace_cached_group_members(cache: &mut LockedCache, group: &str, members: &[String], now: OffsetDateTime) {
|
||||
// This node may have missed peer edits, so reconcile against the complete record,
|
||||
// including memberships that were already stale before this edit began.
|
||||
let mut missing: HashSet<_> = members.iter().collect();
|
||||
let reverse = Arc::clone(&cache.state().user_group_memberships);
|
||||
for (user, groups) in reverse.iter() {
|
||||
if groups.contains(group) && !missing.remove(user) {
|
||||
let mut groups = groups.clone();
|
||||
groups.remove(group);
|
||||
if groups.is_empty() {
|
||||
cache.delete_user_group_membership(user, now);
|
||||
} else {
|
||||
cache.add_or_update_user_group_membership(user, &groups, now);
|
||||
}
|
||||
fn remove_group_from_memberships_map_unlocked(&self, cache: &mut LockedCache, group: &str, now: OffsetDateTime) {
|
||||
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
|
||||
for (k, v) in user_group_memberships.iter() {
|
||||
if v.contains(group) {
|
||||
let mut m = v.clone();
|
||||
m.remove(group);
|
||||
cache.add_or_update_user_group_membership(k, &m, now);
|
||||
}
|
||||
}
|
||||
for user in missing {
|
||||
let mut groups = reverse.get(user).cloned().unwrap_or_default();
|
||||
groups.insert(group.to_owned());
|
||||
cache.add_or_update_user_group_membership(user, &groups, now);
|
||||
}
|
||||
|
||||
fn update_group_memberships_map_unlocked(&self, cache: &mut LockedCache, group: &str, gi: &GroupInfo, now: OffsetDateTime) {
|
||||
let user_group_memberships = Arc::clone(&cache.state().user_group_memberships);
|
||||
for member in gi.members.iter() {
|
||||
let mut m = user_group_memberships.get(member).cloned().unwrap_or_default();
|
||||
m.insert(group.to_string());
|
||||
cache.add_or_update_user_group_membership(member, &m, now);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2267,7 +2297,7 @@ where
|
||||
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
Self::replace_cached_group_members(cache, group, &[], now);
|
||||
self.remove_group_from_memberships_map_unlocked(cache, group, now);
|
||||
cache.delete_group(group, now);
|
||||
cache.delete_group_policy(group, now);
|
||||
});
|
||||
@@ -2277,8 +2307,12 @@ where
|
||||
|
||||
let gi = m[group].clone();
|
||||
|
||||
self.cache
|
||||
.with_write_lock(|cache| self.publish_group_unlocked(cache, group, &gi));
|
||||
self.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_group(group, &gi, now);
|
||||
self.remove_group_from_memberships_map_unlocked(cache, group, now);
|
||||
self.update_group_memberships_map_unlocked(cache, group, &gi, now);
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -104,12 +104,8 @@ pub(crate) mod crate_boundary {
|
||||
}
|
||||
|
||||
pub(crate) mod object_store {
|
||||
pub(crate) use super::storage_contracts::{
|
||||
HTTPPreconditions, ListOperations, NamespaceLocking, ObjectInfoOrErr, ObjectOperations,
|
||||
};
|
||||
pub(crate) use super::storage_contracts::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
|
||||
pub(crate) use rustfs_ecstore::api::config::com::read_config_preserve_empty_with_metadata_opts;
|
||||
pub(crate) use rustfs_ecstore::api::object::WriteCommitGuard;
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::get_lock_acquire_timeout;
|
||||
}
|
||||
|
||||
pub(crate) mod runtime {
|
||||
|
||||
@@ -21,14 +21,6 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// A backend-owned transaction for one named group. Persistence must fail if its lock is lost.
|
||||
#[async_trait::async_trait]
|
||||
pub trait GroupEdit: Send + Sync {
|
||||
async fn save(&self, group: &GroupInfo) -> Result<()>;
|
||||
/// Remove the policy mapping, then the group record, after the caller verified emptiness.
|
||||
async fn delete_empty(&self) -> Result<()>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait Store: Clone + Send + Sync + 'static {
|
||||
fn has_watcher(&self) -> bool;
|
||||
@@ -42,11 +34,6 @@ pub trait Store: Clone + Send + Sync + 'static {
|
||||
async fn delete_iam_config(&self, path: impl AsRef<str> + Send) -> Result<()>;
|
||||
|
||||
async fn save_user_identity(&self, name: &str, user_type: UserType, item: UserIdentity, ttl: Option<usize>) -> Result<()>;
|
||||
/// Create a regular identity or replace one without changing its persisted status.
|
||||
/// Return whether a previous secret was replaced. Backends without conditional writes fail closed.
|
||||
async fn save_regular_user_preserving_status(&self, _name: &str, _item: &UserIdentity) -> Result<bool> {
|
||||
Err(crate::error::Error::IAMActionNotAllowed)
|
||||
}
|
||||
async fn delete_user_identity(&self, name: &str, user_type: UserType) -> Result<()>;
|
||||
async fn load_user_identity(&self, name: &str, user_type: UserType) -> Result<UserIdentity>;
|
||||
|
||||
@@ -54,12 +41,6 @@ pub trait Store: Clone + Send + Sync + 'static {
|
||||
async fn load_users(&self, user_type: UserType, m: &mut HashMap<String, UserIdentity>) -> Result<()>;
|
||||
async fn load_secret_key(&self, name: &str, user_type: UserType) -> Result<String>;
|
||||
|
||||
/// Serialize edits to this group across all writers sharing the backend, then read its
|
||||
/// durable state. Only confirmed absence yields `None`. Keep the returned guard through
|
||||
/// save/delete and local cache publication; dropping it releases the transaction lock.
|
||||
async fn begin_group_edit(&self, _name: &str) -> Result<(Box<dyn GroupEdit>, Option<GroupInfo>)> {
|
||||
Err(crate::error::Error::other("backend does not support serialized group edits"))
|
||||
}
|
||||
async fn save_group_info(&self, name: &str, item: GroupInfo) -> Result<()>;
|
||||
async fn delete_group_info(&self, name: &str) -> Result<()>;
|
||||
async fn load_group(&self, name: &str, m: &mut HashMap<String, GroupInfo>) -> Result<()>;
|
||||
|
||||
@@ -12,9 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{GroupEdit, GroupInfo, MappedPolicy, Store, UserType};
|
||||
use super::{GroupInfo, MappedPolicy, Store, UserType};
|
||||
use crate::error::{Error, Result, is_err_config_not_found, is_err_no_such_group};
|
||||
use crate::storage_api::object_store::WriteCommitGuard;
|
||||
use crate::{
|
||||
IAM_CONFIG_ROOT_PREFIX, IamStorageError, IamStore, classify_iam_system_path_failure_reason, delete_iam_config,
|
||||
read_iam_config_no_lock, read_iam_config_with_metadata, save_iam_config, save_iam_config_with_opts,
|
||||
@@ -42,8 +41,8 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use crate::storage_api::object_store::{
|
||||
HTTPPreconditions, ListOperations, NamespaceLocking, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
|
||||
get_lock_acquire_timeout, read_config_preserve_empty_with_metadata_opts,
|
||||
HTTPPreconditions, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
|
||||
read_config_preserve_empty_with_metadata_opts,
|
||||
};
|
||||
|
||||
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{IAM_CONFIG_ROOT_PREFIX}/iam"));
|
||||
@@ -86,29 +85,6 @@ fn get_group_info_path(group: &str) -> String {
|
||||
path_join_buf(&[&IAM_CONFIG_GROUPS_PREFIX, group, IAM_GROUP_MEMBERS_FILE])
|
||||
}
|
||||
|
||||
fn unchanged_status_precondition(
|
||||
previous: &UserIdentity,
|
||||
item: &UserIdentity,
|
||||
etag: Option<String>,
|
||||
) -> Result<(HTTPPreconditions, bool)> {
|
||||
if previous.credentials.status != item.credentials.status
|
||||
|| previous.credentials.is_temp()
|
||||
|| previous.credentials.is_service_account()
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
let tag = etag
|
||||
.filter(|tag| !tag.is_empty() && tag != "*")
|
||||
.ok_or_else(|| Error::other("missing IAM identity revision"))?;
|
||||
Ok((
|
||||
HTTPPreconditions {
|
||||
if_match: Some(tag),
|
||||
..Default::default()
|
||||
},
|
||||
previous.credentials.secret_key != item.credentials.secret_key,
|
||||
))
|
||||
}
|
||||
|
||||
fn get_policy_doc_path(name: &str) -> String {
|
||||
path_join_buf(&[&IAM_CONFIG_POLICIES_PREFIX, name, IAM_POLICY_FILE])
|
||||
}
|
||||
@@ -232,53 +208,6 @@ pub struct ObjectStore {
|
||||
object_api: Arc<IamStore>,
|
||||
}
|
||||
|
||||
struct ObjectGroupEdit {
|
||||
api: Arc<IamStore>,
|
||||
path: String,
|
||||
policy_path: String,
|
||||
guard: WriteCommitGuard,
|
||||
}
|
||||
|
||||
impl ObjectGroupEdit {
|
||||
fn fenced(&self, mut opts: IamObjectOptions) -> Result<IamObjectOptions> {
|
||||
if self.guard.is_lock_lost() {
|
||||
return Err(Error::other("group transaction lock lost"));
|
||||
}
|
||||
if let Some(signal) = self.guard.lock_lost_signal() {
|
||||
opts.add_namespace_lock_lost_signal(signal);
|
||||
}
|
||||
Ok(opts)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl GroupEdit for ObjectGroupEdit {
|
||||
async fn save(&self, group: &GroupInfo) -> Result<()> {
|
||||
let data = ObjectStore::prepare_data_for_storage(&serde_json::to_vec(group)?)?;
|
||||
let opts = self.fenced(IamObjectOptions {
|
||||
max_parity: true,
|
||||
..Default::default()
|
||||
})?;
|
||||
save_iam_config_with_opts(self.api.clone(), &self.path, data, &opts).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_empty(&self) -> Result<()> {
|
||||
for path in [&self.policy_path, &self.path] {
|
||||
let opts = self.fenced(IamObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
..Default::default()
|
||||
})?;
|
||||
match self.api.delete_object(ObjectStore::BUCKET_NAME, path, opts).await {
|
||||
Ok(_) | Err(IamStorageError::FileNotFound | IamStorageError::ObjectNotFound(_, _)) => (),
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectStore {
|
||||
const BUCKET_NAME: &'static str = ".rustfs.sys";
|
||||
|
||||
@@ -968,34 +897,6 @@ impl Store for ObjectStore {
|
||||
e
|
||||
})
|
||||
}
|
||||
async fn save_regular_user_preserving_status(&self, name: &str, item: &UserIdentity) -> Result<bool> {
|
||||
let path = get_user_identity_path(name, UserType::Reg);
|
||||
let existing =
|
||||
read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), &path, &LoadMode::Locked.read_opts()).await;
|
||||
let (conditions, rotated) = match existing.map_err(Error::from) {
|
||||
Ok((data, metadata)) => {
|
||||
let previous: UserIdentity = Self::decode_iam_config_strict(&data)?;
|
||||
unchanged_status_precondition(&previous, item, metadata.etag)?
|
||||
}
|
||||
Err(err) if is_err_config_not_found(&err) => (
|
||||
HTTPPreconditions {
|
||||
if_none_match: Some("*".into()),
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let opts = IamObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(conditions),
|
||||
..Default::default()
|
||||
};
|
||||
let data = Self::prepare_data_for_storage(&serde_json::to_vec(item)?)?;
|
||||
save_iam_config_with_opts(self.object_api.clone(), &path, data, &opts).await?;
|
||||
Ok(rotated)
|
||||
}
|
||||
|
||||
async fn delete_user_identity(&self, name: &str, user_type: UserType) -> Result<()> {
|
||||
self.delete_iam_config(get_user_identity_path(name, user_type))
|
||||
.await
|
||||
@@ -1052,28 +953,6 @@ impl Store for ObjectStore {
|
||||
Ok(u.credentials.secret_key)
|
||||
}
|
||||
|
||||
async fn begin_group_edit(&self, name: &str) -> Result<(Box<dyn GroupEdit>, Option<GroupInfo>)> {
|
||||
let path = get_group_info_path(name);
|
||||
// Take the transaction lock before any object locks. A distinct key lets ordinary
|
||||
// config reads, writes and deletes keep taking their own namespace locks.
|
||||
let lock = self.object_api.new_ns_lock(Self::BUCKET_NAME, &format!("{path}.txn")).await?;
|
||||
let guard = WriteCommitGuard::acquire(&lock, get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
let group = match self.load_iam_config_strict(&path).await {
|
||||
Ok(gi) => Some(gi),
|
||||
Err(Error::ConfigNotFound) => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let edit = ObjectGroupEdit {
|
||||
api: self.object_api.clone(),
|
||||
path,
|
||||
policy_path: get_mapped_policy_path(name, UserType::Reg, true),
|
||||
guard,
|
||||
};
|
||||
Ok((Box::new(edit), group))
|
||||
}
|
||||
|
||||
async fn save_group_info(&self, name: &str, item: GroupInfo) -> Result<()> {
|
||||
self.save_iam_config(item, get_group_info_path(name)).await
|
||||
}
|
||||
@@ -1624,78 +1503,6 @@ mod tests {
|
||||
assert!(matches!(err, Error::NoSuchUser(name) if name == "missing-sts"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restricted_user_upsert_requires_unchanged_status_and_exact_revision() {
|
||||
let old = super::UserIdentity::from(Credentials {
|
||||
status: "off".into(),
|
||||
secret_key: "old-fixture".into(),
|
||||
..Default::default()
|
||||
});
|
||||
let mut new = old.clone();
|
||||
new.credentials.secret_key = "new-fixture".into();
|
||||
let (predicate, rotated) =
|
||||
super::unchanged_status_precondition(&old, &new, Some("rev1".into())).expect("status remains disabled");
|
||||
assert!(rotated);
|
||||
assert_eq!(predicate.if_match.as_deref(), Some("rev1"));
|
||||
assert!(super::unchanged_status_precondition(&old, &new, None).is_err());
|
||||
new.credentials.status = "on".into();
|
||||
assert!(matches!(
|
||||
super::unchanged_status_precondition(&old, &new, Some("rev1".into())),
|
||||
Err(crate::error::Error::IAMActionNotAllowed)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn restricted_user_upsert_checks_durable_status_before_replacing_credentials() {
|
||||
use crate::store::{Store, UserType};
|
||||
let _credentials = test_cred();
|
||||
let temp = tempfile::tempdir().expect("IAM fixture directory");
|
||||
let env = Box::pin(
|
||||
rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(temp.path())
|
||||
.init_bucket_metadata(false)
|
||||
.build(),
|
||||
)
|
||||
.await;
|
||||
let store = ObjectStore::new(env.ecstore);
|
||||
let name = "status-fixture";
|
||||
let mut entry = super::UserIdentity::from(Credentials {
|
||||
access_key: name.into(),
|
||||
status: "off".into(),
|
||||
secret_key: "first-fixture".into(),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(
|
||||
!store
|
||||
.save_regular_user_preserving_status(name, &entry)
|
||||
.await
|
||||
.expect("create absent account")
|
||||
);
|
||||
entry.credentials.status = "on".into();
|
||||
entry.credentials.secret_key = "second-fixture".into();
|
||||
assert!(matches!(
|
||||
store.save_regular_user_preserving_status(name, &entry).await,
|
||||
Err(Error::IAMActionNotAllowed)
|
||||
));
|
||||
let persisted = store
|
||||
.load_user_identity(name, UserType::Reg)
|
||||
.await
|
||||
.expect("reload denied write");
|
||||
assert_eq!(persisted.credentials.status, "off");
|
||||
assert_eq!(persisted.credentials.secret_key, "first-fixture");
|
||||
entry.credentials.status = "off".into();
|
||||
assert!(
|
||||
store
|
||||
.save_regular_user_preserving_status(name, &entry)
|
||||
.await
|
||||
.expect("rotate while disabled")
|
||||
);
|
||||
let persisted = store.load_user_identity(name, UserType::Reg).await.expect("reload rotation");
|
||||
assert_eq!(persisted.credentials.status, "off");
|
||||
assert_eq!(persisted.credentials.secret_key, "second-fixture");
|
||||
}
|
||||
|
||||
fn test_cred() -> Credentials {
|
||||
if let Some(cred) = crate::root_credentials::credentials() {
|
||||
return cred;
|
||||
|
||||
+65
-425
@@ -76,8 +76,6 @@ pub const STATUS_ENABLED: &str = "enabled";
|
||||
pub const STATUS_DISABLED: &str = "disabled";
|
||||
|
||||
pub const POLICYNAME: &str = "policy";
|
||||
/// Signed subset of the OIDC policy selection granted independently of IAM mappings.
|
||||
pub const OIDC_EXPLICIT_POLICIES_CLAIM: &str = "x-rustfs-internal-oidc-explicit-policies";
|
||||
pub const PLUGIN_ATTEMPT_CREDENTIAL_CLAIM: &str = "rustfs:plugin-attempt-v1";
|
||||
pub const SESSION_POLICY_NAME: &str = "sessionPolicy";
|
||||
pub const SESSION_POLICY_NAME_EXTRACTED: &str = "sessionPolicy-extracted";
|
||||
@@ -1297,22 +1295,6 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<OffsetDateTime> {
|
||||
self.upsert_regular_user(access_key, args, updated_at, true).await
|
||||
}
|
||||
|
||||
/// Create a regular user, or update its credentials without changing the stored account status.
|
||||
pub async fn create_user_preserving_status(&self, access_key: &str, args: &AddOrUpdateUserReq) -> Result<OffsetDateTime> {
|
||||
self.upsert_regular_user(access_key, args, OffsetDateTime::now_utc(), false)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn upsert_regular_user(
|
||||
&self,
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
allow_status_change: bool,
|
||||
) -> Result<OffsetDateTime> {
|
||||
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
|
||||
let iam = self.clone();
|
||||
@@ -1320,10 +1302,7 @@ impl<T: Store> IamSys<T> {
|
||||
let args = args.clone();
|
||||
#[cfg(test)]
|
||||
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
|
||||
let operation = async move {
|
||||
iam.create_user_at_inner(&access_key, &args, updated_at, allow_status_change)
|
||||
.await
|
||||
};
|
||||
let operation = async move { iam.create_user_at_inner(&access_key, &args, updated_at).await };
|
||||
#[cfg(test)]
|
||||
let task = runtime.spawn(async move {
|
||||
if let Some(probe) = notification_probe {
|
||||
@@ -1341,7 +1320,6 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
allow_status_change: bool,
|
||||
) -> Result<OffsetDateTime> {
|
||||
if !is_access_key_valid(access_key) {
|
||||
return Err(IamError::InvalidAccessKeyLength);
|
||||
@@ -1357,7 +1335,7 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
let (updated_at, secret_changed, parent_guard) = self
|
||||
.store
|
||||
.add_user_at_with_secret_change(access_key, args, updated_at, allow_status_change)
|
||||
.add_user_at_with_secret_change(access_key, args, updated_at)
|
||||
.await?;
|
||||
if secret_changed {
|
||||
self.revoke_sts_sessions_for_parent(access_key).await?;
|
||||
@@ -1692,10 +1670,16 @@ impl<T: Store> IamSys<T> {
|
||||
parent_user: &str,
|
||||
groups: &Option<Vec<String>>,
|
||||
) -> Result<Option<HashMap<String, Value>>> {
|
||||
let Some((resolved, _)) = self.verified_federated_policy(claims, parent_user, groups).await? else {
|
||||
let Some(policy_names) = self.verified_federated_policy_names(claims, parent_user, groups).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let (resolved, _) = self.store.merge_policies(&policy_names.join(",")).await;
|
||||
let resolved_names = MappedPolicy::new(&resolved).to_slice();
|
||||
if policy_names.iter().any(|policy_name| !resolved_names.contains(policy_name)) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
|
||||
let boundary = match claims.get(SESSION_POLICY_NAME) {
|
||||
Some(Value::String(encoded)) => {
|
||||
decode_session_policy(encoded)?;
|
||||
@@ -1712,9 +1696,6 @@ impl<T: Store> IamSys<T> {
|
||||
if claims.get(OIDC_VIRTUAL_PARENT_CLAIM).and_then(Value::as_str) == Some(parent_user) {
|
||||
verified_claims.insert(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), Value::String(parent_user.to_string()));
|
||||
}
|
||||
if let Some(explicit) = claims.get(OIDC_EXPLICIT_POLICIES_CLAIM) {
|
||||
verified_claims.insert(OIDC_EXPLICIT_POLICIES_CLAIM.to_owned(), explicit.clone());
|
||||
}
|
||||
if let Some(boundary) = boundary {
|
||||
verified_claims.insert(FEDERATED_POLICY_BOUNDARY_CLAIM.to_string(), Value::String(boundary));
|
||||
}
|
||||
@@ -1896,24 +1877,20 @@ impl<T: Store> IamSys<T> {
|
||||
Ok(policy_names)
|
||||
}
|
||||
|
||||
async fn resolve_oidc_policy(&self, names: &[String]) -> Result<(String, Policy)> {
|
||||
if names.is_empty() || names.iter().any(|name| !is_safe_claim_policy_name(name)) {
|
||||
fn signed_oidc_policy_names(claims: &HashMap<String, Value>, parent_user: &str) -> Result<Vec<String>> {
|
||||
let policy_names = Self::safe_claim_policy_names(claims, parent_user);
|
||||
if policy_names.is_empty() {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
let (resolved, policy) = self.store.merge_policies(&names.join(",")).await;
|
||||
let resolved_names = MappedPolicy::new(&resolved).to_slice();
|
||||
if names.iter().any(|name| !resolved_names.contains(name)) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
Ok((resolved, policy))
|
||||
Ok(policy_names)
|
||||
}
|
||||
|
||||
async fn verified_federated_policy(
|
||||
async fn verified_federated_policy_names(
|
||||
&self,
|
||||
claims: &HashMap<String, Value>,
|
||||
parent_user: &str,
|
||||
groups: &Option<Vec<String>>,
|
||||
) -> Result<Option<(String, Policy)>> {
|
||||
) -> Result<Option<Vec<String>>> {
|
||||
if !is_rustfs_oidc_claims(claims) {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -1921,72 +1898,12 @@ impl<T: Store> IamSys<T> {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
|
||||
let explicit = claims.get(OIDC_EXPLICIT_POLICIES_CLAIM);
|
||||
if explicit.is_none() {
|
||||
// Earlier grouped credentials cannot distinguish an explicit Deny from a detached grant.
|
||||
if claims.contains_key("groups") || groups.as_ref().is_some_and(|groups| !groups.is_empty()) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
if !is_verified_federated_session(claims, parent_user) && !has_verified_federated_policy(claims) {
|
||||
let names = self.oidc_policy_names(claims, parent_user, groups).await?;
|
||||
return self.resolve_oidc_policy(&names).await.map(Some);
|
||||
}
|
||||
}
|
||||
let issued: Vec<String> = claims
|
||||
.get(POLICYNAME)
|
||||
.and_then(Value::as_str)
|
||||
.ok_or(IamError::InvalidArgument)?
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.map(str::to_owned)
|
||||
.collect();
|
||||
if issued.iter().any(|name| !is_safe_claim_policy_name(name)) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
let Some(explicit) = explicit else {
|
||||
return self.resolve_oidc_policy(&issued).await.map(Some);
|
||||
let policy_names = if is_verified_federated_session(claims, parent_user) {
|
||||
Self::signed_oidc_policy_names(claims, parent_user)?
|
||||
} else {
|
||||
self.oidc_policy_names(claims, parent_user, groups).await?
|
||||
};
|
||||
if !is_verified_federated_session(claims, parent_user) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
let explicit: HashSet<&str> = explicit
|
||||
.as_array()
|
||||
.ok_or(IamError::InvalidArgument)?
|
||||
.iter()
|
||||
.map(|v| {
|
||||
v.as_str()
|
||||
.filter(|name| issued.iter().any(|n| n == name))
|
||||
.ok_or(IamError::InvalidArgument)
|
||||
})
|
||||
.collect::<Result<_>>()?;
|
||||
// Use the issuer's signed group list, including on replicated and derived credentials.
|
||||
let signed_groups: Option<Vec<String>> = claims
|
||||
.get("groups")
|
||||
.map(serde::Deserialize::deserialize)
|
||||
.transpose()
|
||||
.map_err(|_| IamError::InvalidArgument)?;
|
||||
if signed_groups.is_none() && groups.as_ref().is_some_and(|groups| !groups.is_empty()) {
|
||||
return Err(IamError::InvalidArgument);
|
||||
}
|
||||
let current = self.store.sts_policy_db_get(parent_user, &signed_groups).await?;
|
||||
let active: Vec<_> = issued
|
||||
.iter()
|
||||
.filter(|name| explicit.contains(name.as_str()) || current.contains(name))
|
||||
.cloned()
|
||||
.collect();
|
||||
let (resolved, mut policy) = self.resolve_oidc_policy(&active).await?;
|
||||
// Later attachments can restrict the credential, but cannot widen its issuance ceiling.
|
||||
let new: Vec<_> = current.into_iter().filter(|name| !issued.contains(name)).collect();
|
||||
if !new.is_empty() {
|
||||
let (_, restrictions) = self.resolve_oidc_policy(&new).await?;
|
||||
policy.statements.extend(
|
||||
restrictions
|
||||
.statements
|
||||
.into_iter()
|
||||
.filter(|s| s.effect == rustfs_policy::policy::Effect::Deny),
|
||||
);
|
||||
}
|
||||
Ok(Some((resolved, policy)))
|
||||
Ok(Some(policy_names))
|
||||
}
|
||||
|
||||
/// Compatibility wrapper for service-account authorization entry points.
|
||||
@@ -2016,9 +1933,6 @@ impl<T: Store> IamSys<T> {
|
||||
let state = Self::policy_plugin_state().await;
|
||||
if marked {
|
||||
if args.is_owner
|
||||
// Plugin-attempt sessions are confined to their allowlisted work;
|
||||
// they cannot use a default permission to manage the parent identity.
|
||||
|| self_service_deny_check(args)
|
||||
|| args.claims.get(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM) != Some(&Value::Bool(true))
|
||||
|| !matches!(state, PolicyPluginState::Disabled)
|
||||
{
|
||||
@@ -2167,10 +2081,10 @@ impl<T: Store> IamSys<T> {
|
||||
session_policy,
|
||||
} => {
|
||||
let session_ok = evaluate_prepared_session_policy(session_policy, args).await;
|
||||
if session_ok == Some(false) && is_rustfs_oidc_claims(args.claims) {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
if let Some(ok) = session_ok {
|
||||
return Ok(ok && (*is_owner || combined_policy.is_allowed(args).await));
|
||||
}
|
||||
session_ok != Some(false) && (*is_owner || derived_policy_allowed(combined_policy, args).await?)
|
||||
*is_owner || combined_policy.is_allowed(args).await
|
||||
}
|
||||
PreparedIamMode::ServiceAccount {
|
||||
is_owner,
|
||||
@@ -2191,18 +2105,12 @@ impl<T: Store> IamSys<T> {
|
||||
true
|
||||
};
|
||||
|
||||
if !boundary_allowed {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
let parent_allowed =
|
||||
*bypass_parent_policy || *is_owner || derived_policy_allowed(combined_policy, &parent_args).await?;
|
||||
(*bypass_parent_policy || *is_owner || combined_policy.is_allowed(&parent_args).await) && boundary_allowed;
|
||||
match mode {
|
||||
PreparedServicePolicyMode::Inherited => parent_allowed,
|
||||
PreparedServicePolicyMode::SessionBound => {
|
||||
let session_ok = evaluate_prepared_session_policy(session_policy, args).await;
|
||||
if session_ok == Some(false) && is_rustfs_oidc_claims(args.claims) {
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
if let Some(ok) = session_ok {
|
||||
return Ok(ok && parent_allowed);
|
||||
}
|
||||
@@ -2222,24 +2130,9 @@ impl<T: Store> IamSys<T> {
|
||||
};
|
||||
|
||||
if policies.is_empty() {
|
||||
// The policy lookup also returns an empty set for an inactive user.
|
||||
// Only a live identity gets the default for its own credential changes.
|
||||
let mode = if self_service_deny_check(args)
|
||||
&& self
|
||||
.store
|
||||
.get_user(args.account)
|
||||
.await
|
||||
.is_some_and(|u| u.credentials.is_valid())
|
||||
{
|
||||
PreparedIamMode::Regular {
|
||||
combined_policy: Policy::default(),
|
||||
}
|
||||
} else {
|
||||
PreparedIamMode::Deny
|
||||
};
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2266,19 +2159,24 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_sts_auth(&self, args: &Args<'_>, parent_user: &str) -> PreparedIamAuth {
|
||||
let federated_policy = match self.verified_federated_policy(args.claims, parent_user, args.groups).await {
|
||||
Ok(policy) => policy,
|
||||
let federated_policy_names = match self
|
||||
.verified_federated_policy_names(args.claims, parent_user, args.groups)
|
||||
.await
|
||||
{
|
||||
Ok(policy_names) => policy_names,
|
||||
Err(_) => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
};
|
||||
let federated_session = federated_policy_names.is_some();
|
||||
let verified_federated_session = is_verified_federated_session(args.claims, parent_user);
|
||||
let is_owner = !is_rustfs_oidc_claims(args.claims) && crate::is_root_access_key(parent_user);
|
||||
let role_arn = args.get_role_arn();
|
||||
|
||||
let (effective_groups, groups_source, mut policy_names) = if federated_policy.is_some() {
|
||||
let (effective_groups, groups_source, policies) = if federated_session {
|
||||
(args.groups.clone(), "oidc_claims", Vec::new())
|
||||
} else if is_owner {
|
||||
(None, "owner", Vec::new())
|
||||
@@ -2310,27 +2208,21 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
},
|
||||
};
|
||||
let p = match self.policy_db_get(parent_user, &effective_groups).await {
|
||||
Ok(policies) => policies,
|
||||
Err(_) if self_service_deny_check(args) => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
Err(_) => Vec::new(),
|
||||
};
|
||||
let p = self.policy_db_get(parent_user, &effective_groups).await.unwrap_or_default();
|
||||
(effective_groups, groups_source, p)
|
||||
};
|
||||
|
||||
let mut policy_names = if let Some(policy_names) = federated_policy_names {
|
||||
policy_names
|
||||
} else {
|
||||
policies
|
||||
};
|
||||
if !is_owner && policy_names.is_empty() && !is_rustfs_oidc_claims(args.claims) {
|
||||
policy_names = Self::safe_claim_policy_names(args.claims, parent_user);
|
||||
}
|
||||
|
||||
let mut combined_policy = Policy::default();
|
||||
if let Some((_, policy)) = federated_policy {
|
||||
combined_policy = policy;
|
||||
} else if !is_owner {
|
||||
if !is_owner {
|
||||
if policy_names.is_empty() {
|
||||
if args.deny_only {
|
||||
combined_policy = Policy::default();
|
||||
@@ -2349,7 +2241,7 @@ impl<T: Store> IamSys<T> {
|
||||
requested_policies = %requested_policies,
|
||||
"prepare_sts_auth: no STS policy names resolved"
|
||||
);
|
||||
if args.deny_only && !self_service_deny_check(args) {
|
||||
if args.deny_only && !verified_federated_session {
|
||||
combined_policy = Policy::default();
|
||||
} else {
|
||||
return PreparedIamAuth {
|
||||
@@ -2363,7 +2255,7 @@ impl<T: Store> IamSys<T> {
|
||||
.iter()
|
||||
.any(|policy_name| !resolved_policy_names.iter().any(|resolved| resolved == policy_name));
|
||||
if has_unresolved_policy_names {
|
||||
if self_service_deny_check(args) {
|
||||
if verified_federated_session {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
@@ -2440,14 +2332,14 @@ impl<T: Store> IamSys<T> {
|
||||
if is_rustfs_oidc_claims(args.claims) && !has_verified_federated_policy {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
let federated_boundary = if has_verified_federated_policy {
|
||||
if !is_valid_verified_federated_policy(args.claims, parent_user) {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
match federated_policy_boundary(args.claims) {
|
||||
@@ -2455,7 +2347,7 @@ impl<T: Store> IamSys<T> {
|
||||
Err(_) => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -2474,8 +2366,10 @@ impl<T: Store> IamSys<T> {
|
||||
let is_owner = !is_rustfs_oidc_claims(args.claims) && crate::is_root_access_key(parent_user);
|
||||
let role_arn = args.get_role_arn();
|
||||
|
||||
let svc_policies = if is_owner || bypass_parent_policy || has_verified_federated_policy {
|
||||
let svc_policies = if is_owner || bypass_parent_policy {
|
||||
Vec::new()
|
||||
} else if has_verified_federated_policy {
|
||||
Self::safe_claim_policy_names(args.claims, parent_user)
|
||||
} else if role_arn.is_some() {
|
||||
let Ok(arn) = ARN::parse(role_arn.unwrap_or_default()) else {
|
||||
tracing::warn!(
|
||||
@@ -2499,7 +2393,7 @@ impl<T: Store> IamSys<T> {
|
||||
policies
|
||||
};
|
||||
|
||||
if !is_owner && !bypass_parent_policy && !has_verified_federated_policy && svc_policies.is_empty() {
|
||||
if !is_owner && !bypass_parent_policy && svc_policies.is_empty() {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
@@ -2508,17 +2402,12 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
let combined_policy = if is_owner || bypass_parent_policy {
|
||||
Policy::default()
|
||||
} else if has_verified_federated_policy {
|
||||
let Ok(Some((_, policy))) = self.verified_federated_policy(args.claims, parent_user, args.groups).await else {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::TerminalDeny,
|
||||
};
|
||||
};
|
||||
policy
|
||||
} else {
|
||||
let (a, c) = self.store.merge_policies(&svc_policies.join(",")).await;
|
||||
if a.is_empty() {
|
||||
let resolved = MappedPolicy::new(&a).to_slice();
|
||||
if a.is_empty()
|
||||
|| has_verified_federated_policy && svc_policies.iter().any(|policy_name| !resolved.contains(policy_name))
|
||||
{
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
@@ -2589,26 +2478,8 @@ async fn prepared_session_policy_needs_existing_object_tag_for_args(policy: &Pre
|
||||
}
|
||||
}
|
||||
|
||||
fn self_service_deny_check(args: &Args<'_>) -> bool {
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
args.deny_only
|
||||
&& matches!(
|
||||
args.action,
|
||||
Action::AdminAction(
|
||||
AdminAction::ChangeOwnPasswordAdminAction
|
||||
| AdminAction::CreateUserAdminAction
|
||||
| AdminAction::ManageOwnMFAAdminAction
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
fn prepare_session_policy(args: &Args<'_>, empty_is_none: bool) -> PreparedSessionPolicy {
|
||||
let Some(policy_str) = extract_session_policy_text(args.claims) else {
|
||||
if self_service_deny_check(args)
|
||||
&& (args.claims.contains_key(SESSION_POLICY_NAME) || args.claims.contains_key(SESSION_POLICY_NAME_EXTRACTED))
|
||||
{
|
||||
return PreparedSessionPolicy::DenyAll;
|
||||
}
|
||||
return PreparedSessionPolicy::None;
|
||||
};
|
||||
|
||||
@@ -2681,7 +2552,6 @@ pub fn remove_verified_federated_policy(claims: &mut HashMap<String, Value>) ->
|
||||
let removed = claims.remove(VERIFIED_FEDERATED_POLICY_CLAIM).is_some();
|
||||
claims.remove(FEDERATED_POLICY_BOUNDARY_CLAIM);
|
||||
claims.remove(OIDC_VIRTUAL_PARENT_CLAIM);
|
||||
claims.remove(OIDC_EXPLICIT_POLICIES_CLAIM);
|
||||
removed
|
||||
}
|
||||
|
||||
@@ -2690,10 +2560,6 @@ fn merge_derived_service_account_claims(
|
||||
source_claims: &HashMap<String, Value>,
|
||||
verified_federated_policy: Option<HashMap<String, Value>>,
|
||||
) -> Result<()> {
|
||||
if verified_federated_policy.is_some() {
|
||||
// Only the parent's signed groups may participate in federated policy resolution.
|
||||
target_claims.remove("groups");
|
||||
}
|
||||
for (key, value) in source_claims {
|
||||
if key != "exp" {
|
||||
target_claims.insert(key.clone(), value.clone());
|
||||
@@ -2760,24 +2626,6 @@ fn extract_session_policy_text(claims: &HashMap<String, Value>) -> Option<String
|
||||
String::from_utf8(bytes).ok()
|
||||
}
|
||||
|
||||
async fn derived_policy_allowed(policy: &Policy, args: &Args<'_>) -> Result<bool> {
|
||||
let allowed = policy.is_allowed(args).await;
|
||||
// A bucket policy may supplement a missing Allow, but not override a live OIDC Deny.
|
||||
if !allowed
|
||||
&& is_rustfs_oidc_claims(args.claims)
|
||||
&& (args.deny_only
|
||||
|| !policy
|
||||
.is_allowed(&Args {
|
||||
deny_only: true,
|
||||
..args.clone()
|
||||
})
|
||||
.await)
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
Ok(allowed)
|
||||
}
|
||||
|
||||
async fn evaluate_prepared_session_policy(policy: &PreparedSessionPolicy, args: &Args<'_>) -> Option<bool> {
|
||||
match policy {
|
||||
PreparedSessionPolicy::None => None,
|
||||
@@ -2864,7 +2712,7 @@ mod tests {
|
||||
use crate::cache::{Cache, CacheEntity};
|
||||
use crate::error::Error;
|
||||
use crate::manager::get_default_policies;
|
||||
use crate::store::{GroupEdit, GroupInfo, MappedPolicy, Store, UserType};
|
||||
use crate::store::{GroupInfo, MappedPolicy, Store, UserType};
|
||||
use rustfs_credentials::{Credentials, init_global_action_credentials};
|
||||
use rustfs_policy::auth::{UserIdentity, get_new_credentials_with_metadata};
|
||||
use rustfs_policy::policy::Args;
|
||||
@@ -3600,30 +3448,10 @@ mod tests {
|
||||
derived_save_started: Arc<tokio::sync::Notify>,
|
||||
release_derived_save: Arc<tokio::sync::Notify>,
|
||||
block_group_save: Arc<std::sync::atomic::AtomicBool>,
|
||||
group_locks: Arc<Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>>,
|
||||
saved_groups: Arc<Mutex<HashMap<String, GroupInfo>>>,
|
||||
group_save_started: Arc<tokio::sync::Notify>,
|
||||
group_save_release: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
struct MockGroupEdit {
|
||||
store: StsTestMockStore,
|
||||
name: String,
|
||||
_guard: tokio::sync::OwnedMutexGuard<()>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl GroupEdit for MockGroupEdit {
|
||||
async fn save(&self, group: &GroupInfo) -> Result<()> {
|
||||
self.store.save_group_info(&self.name, group.clone()).await
|
||||
}
|
||||
|
||||
async fn delete_empty(&self) -> Result<()> {
|
||||
self.store.delete_mapped_policy(&self.name, UserType::Reg, true).await?;
|
||||
self.store.delete_group_info(&self.name).await
|
||||
}
|
||||
}
|
||||
|
||||
impl StsTestMockStore {
|
||||
fn new(empty_policies: bool) -> Self {
|
||||
Self {
|
||||
@@ -3644,12 +3472,6 @@ mod tests {
|
||||
derived_save_started: Arc::new(tokio::sync::Notify::new()),
|
||||
release_derived_save: Arc::new(tokio::sync::Notify::new()),
|
||||
block_group_save: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
group_locks: Default::default(),
|
||||
saved_groups: Arc::new(Mutex::new(if empty_policies {
|
||||
HashMap::new()
|
||||
} else {
|
||||
HashMap::from([("testgroup".to_owned(), GroupInfo::new(vec!["sts-fallback-test-parent".to_owned()]))])
|
||||
})),
|
||||
group_save_started: Arc::new(tokio::sync::Notify::new()),
|
||||
group_save_release: Arc::new(tokio::sync::Notify::new()),
|
||||
}
|
||||
@@ -3792,36 +3614,15 @@ mod tests {
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
async fn begin_group_edit(&self, name: &str) -> Result<(Box<dyn GroupEdit>, Option<GroupInfo>)> {
|
||||
let lock = self
|
||||
.group_locks
|
||||
.lock()
|
||||
.expect("group locks")
|
||||
.entry(name.to_owned())
|
||||
.or_default()
|
||||
.clone();
|
||||
let guard = lock.lock_owned().await;
|
||||
Ok((
|
||||
Box::new(MockGroupEdit {
|
||||
store: self.clone(),
|
||||
name: name.to_owned(),
|
||||
_guard: guard,
|
||||
}),
|
||||
self.saved_groups.lock().expect("saved groups").get(name).cloned(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn save_group_info(&self, name: &str, item: GroupInfo) -> Result<()> {
|
||||
async fn save_group_info(&self, _name: &str, _item: GroupInfo) -> Result<()> {
|
||||
if self.block_group_save.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
self.group_save_started.notify_one();
|
||||
self.group_save_release.notified().await;
|
||||
}
|
||||
self.saved_groups.lock().expect("saved groups").insert(name.to_owned(), item);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_group_info(&self, name: &str) -> Result<()> {
|
||||
self.saved_groups.lock().expect("saved groups").remove(name);
|
||||
async fn delete_group_info(&self, _name: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -3874,14 +3675,11 @@ mod tests {
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
async fn delete_mapped_policy(&self, name: &str, user_type: UserType, is_group: bool) -> Result<()> {
|
||||
async fn delete_mapped_policy(&self, name: &str, user_type: UserType, _is_group: bool) -> Result<()> {
|
||||
self.deleted_mapped_policies
|
||||
.lock()
|
||||
.expect("deleted_mapped_policies mutex poisoned")
|
||||
.push((name.to_string(), user_type));
|
||||
if is_group {
|
||||
return Ok(());
|
||||
}
|
||||
Err(Error::InvalidArgument)
|
||||
}
|
||||
|
||||
@@ -3990,7 +3788,9 @@ mod tests {
|
||||
users.insert(PARENT_USER.to_string(), parent_identity);
|
||||
users.extend(saved_users);
|
||||
|
||||
let groups = self.saved_groups.lock().expect("saved groups").clone();
|
||||
let group = GroupInfo::new(vec![PARENT_USER.to_string()]);
|
||||
let mut groups = HashMap::new();
|
||||
groups.insert(GROUP_NAME.to_string(), group);
|
||||
|
||||
let group_policy = MappedPolicy::new("readwrite");
|
||||
let mut group_policies = HashMap::new();
|
||||
@@ -4023,82 +3823,6 @@ mod tests {
|
||||
IamSys::new(cache)
|
||||
}
|
||||
|
||||
async fn shared_group_caches() -> (StsTestMockStore, Arc<IamCache<StsTestMockStore>>, Arc<IamCache<StsTestMockStore>>) {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let (first, second) = temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], async {
|
||||
(
|
||||
IamCache::new(store.clone()).await.expect("first cache"),
|
||||
IamCache::new(store.clone()).await.expect("second cache"),
|
||||
)
|
||||
})
|
||||
.await;
|
||||
(store, first, second)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn group_edit_status_and_removal_survive_shared_backend_reload() {
|
||||
let (store, first, second) = shared_group_caches().await;
|
||||
let group = "testgroup";
|
||||
let member = "sts-fallback-test-parent";
|
||||
second
|
||||
.add_users_to_group("unrelated", vec![member.to_owned()])
|
||||
.await
|
||||
.expect("seed unrelated membership");
|
||||
store.block_group_save.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let mut removal = Box::pin(first.remove_users_from_group(group, vec![member.to_owned()]));
|
||||
assert!(futures::poll!(&mut removal).is_pending());
|
||||
store.block_group_save.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||
let mut status = Box::pin(second.set_group_status(group, false));
|
||||
assert!(futures::poll!(&mut status).is_pending(), "status must wait for the other writer");
|
||||
store.group_save_release.notify_one();
|
||||
removal.await.expect("commit removal");
|
||||
status.await.expect("commit status");
|
||||
|
||||
store.load_all(&first.cache).await.expect("reload durable groups");
|
||||
for node in [&first, &second] {
|
||||
let state = node.cache.snapshot();
|
||||
assert!(state.groups[group].members.is_empty());
|
||||
assert_eq!(state.groups[group].status, "disabled");
|
||||
assert_eq!(state.user_group_memberships[member], HashSet::from(["unrelated".to_owned()]));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn group_edit_delete_orders_emptiness_check_with_add() {
|
||||
let (store, first, second) = shared_group_caches().await;
|
||||
let group = "testgroup";
|
||||
let member = "sts-fallback-test-parent";
|
||||
second
|
||||
.remove_users_from_group(group, vec![member.to_owned()])
|
||||
.await
|
||||
.expect("empty group");
|
||||
store.block_group_save.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let mut add = Box::pin(first.add_users_to_group(group, vec![member.to_owned()]));
|
||||
assert!(futures::poll!(&mut add).is_pending());
|
||||
let mut delete = Box::pin(second.remove_users_from_group(group, vec![]));
|
||||
assert!(futures::poll!(&mut delete).is_pending(), "delete must wait before checking emptiness");
|
||||
store.group_save_release.notify_one();
|
||||
add.await.expect("restore membership");
|
||||
assert!(matches!(delete.await, Err(Error::GroupNotEmpty)));
|
||||
assert!(store.deleted_mapped_policies.lock().expect("policy deletions").is_empty());
|
||||
store.block_group_save.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||
second.set_group_status(group, false).await.expect("refresh from peer add");
|
||||
assert!(second.cache.snapshot().user_group_memberships[member].contains(group));
|
||||
first
|
||||
.remove_users_from_group(group, vec![member.to_owned()])
|
||||
.await
|
||||
.expect("empty group");
|
||||
second
|
||||
.remove_users_from_group(group, vec![])
|
||||
.await
|
||||
.expect("delete empty group");
|
||||
assert!(matches!(first.set_group_status(group, true).await, Err(Error::NoSuchGroup(_))));
|
||||
assert!(!second.cache.snapshot().user_group_memberships.contains_key(member));
|
||||
assert!(!store.saved_groups.lock().expect("durable groups").contains_key(group));
|
||||
}
|
||||
|
||||
async fn assert_group_write_during_reload_is_published(remove: bool) {
|
||||
let iam_sys = Arc::new(temp_env::async_with_vars([("RUSTFS_SKIP_BACKGROUND_TASK", Some("1"))], test_iam_sys()).await);
|
||||
let member = "sts-fallback-test-parent";
|
||||
@@ -6058,15 +5782,13 @@ mod tests {
|
||||
("sub".to_string(), Value::String("subject-123".to_string())),
|
||||
("parent".to_string(), Value::String(parent_user.to_string())),
|
||||
(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), Value::String(parent_user.to_string())),
|
||||
(POLICYNAME.to_string(), Value::String("readwrite,readonly".to_string())),
|
||||
(OIDC_EXPLICIT_POLICIES_CLAIM.to_string(), serde_json::json!(["readonly"])),
|
||||
("groups".to_string(), serde_json::json!(["testgroup"])),
|
||||
(POLICYNAME.to_string(), Value::String("readwrite".to_string())),
|
||||
]);
|
||||
let conditions = HashMap::new();
|
||||
let sts_args = Args {
|
||||
account: "OIDCGROUPCALLER01",
|
||||
groups: &groups,
|
||||
action: Action::S3Action(S3Action::PutObjectAction),
|
||||
action: Action::S3Action(S3Action::GetObjectAction),
|
||||
bucket: "bucket",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
@@ -6101,10 +5823,7 @@ mod tests {
|
||||
.await
|
||||
.expect("group-authorized OIDC caller should create a service account");
|
||||
assert_eq!(replication_claims.get(VERIFIED_FEDERATED_POLICY_CLAIM), Some(&Value::Bool(true)));
|
||||
assert_eq!(
|
||||
replication_claims.get(OIDC_EXPLICIT_POLICIES_CLAIM),
|
||||
claims.get(OIDC_EXPLICIT_POLICIES_CLAIM)
|
||||
);
|
||||
assert_eq!(replication_claims.get(POLICYNAME), Some(&Value::String("readwrite".to_string())));
|
||||
|
||||
let service_claims = iam_sys
|
||||
.get_claims_for_svc_acc(&service_account.access_key)
|
||||
@@ -6120,83 +5839,6 @@ mod tests {
|
||||
iam_sys.is_allowed(&service_args).await,
|
||||
"OIDC service account must resolve the policy names selected at STS issuance"
|
||||
);
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let deny_read = Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::bucket/*"}]}"#).expect("parse Deny");
|
||||
iam_sys
|
||||
.store
|
||||
.cache
|
||||
.add_or_update_policy_doc("deny-read", &PolicyDoc::new(deny_read), now);
|
||||
for (mapping, enabled, read, write) in [
|
||||
("", true, true, false),
|
||||
("readwrite", false, true, false),
|
||||
("readwrite,deny-read", true, false, true),
|
||||
("readwrite,missing-deny", true, false, false),
|
||||
] {
|
||||
if mapping.is_empty() {
|
||||
iam_sys.store.cache.delete_group_policy("testgroup", now);
|
||||
} else {
|
||||
iam_sys
|
||||
.store
|
||||
.cache
|
||||
.add_or_update_group_policy("testgroup", &MappedPolicy::new(mapping), now);
|
||||
}
|
||||
let mut group = GroupInfo::new(Vec::new());
|
||||
if !enabled {
|
||||
group.status = STATUS_DISABLED.to_owned();
|
||||
}
|
||||
iam_sys.store.cache.add_or_update_group("testgroup", &group, now);
|
||||
for (action, expected) in [(S3Action::GetObjectAction, read), (S3Action::PutObjectAction, write)] {
|
||||
let sts = Args {
|
||||
action: Action::S3Action(action),
|
||||
..sts_args.clone()
|
||||
};
|
||||
let service = Args {
|
||||
action: sts.action,
|
||||
..service_args.clone()
|
||||
};
|
||||
let prepared = iam_sys.prepare_sts_auth(&sts, parent_user).await;
|
||||
assert_eq!(
|
||||
iam_sys.eval_prepared(&prepared, &sts).await,
|
||||
expected,
|
||||
"STS: {mapping}, enabled={enabled}"
|
||||
);
|
||||
assert_eq!(
|
||||
iam_sys.is_allowed(&service).await,
|
||||
expected,
|
||||
"service account: {mapping}, enabled={enabled}"
|
||||
);
|
||||
if mapping == "readwrite,deny-read" && action == S3Action::GetObjectAction {
|
||||
let child = iam_sys.prepare_service_account_auth(&service, parent_user).await;
|
||||
for (prepared, args) in [(&prepared, &sts), (&child, &service)] {
|
||||
assert!(
|
||||
matches!(iam_sys.try_eval_prepared(prepared, args).await, Err(Error::IAMActionNotAllowed)),
|
||||
"a newly mapped Deny must stop bucket policy fallback"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
iam_sys.store.cache.delete_group_policy("testgroup", now);
|
||||
for (source, account) in [(&claims, sts_args.account), (&service_claims, service_args.account)] {
|
||||
let mut legacy = source.clone();
|
||||
legacy.remove(OIDC_EXPLICIT_POLICIES_CLAIM);
|
||||
let args = Args {
|
||||
claims: &legacy,
|
||||
account,
|
||||
deny_only: true,
|
||||
..sts_args.clone()
|
||||
};
|
||||
let prepared = if account == sts_args.account {
|
||||
iam_sys.prepare_sts_auth(&args, parent_user).await
|
||||
} else {
|
||||
iam_sys.prepare_service_account_auth(&args, parent_user).await
|
||||
};
|
||||
assert!(
|
||||
!iam_sys.eval_prepared(&prepared, &args).await,
|
||||
"ambiguous legacy grouped credentials must be reissued"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -6224,8 +5866,6 @@ mod tests {
|
||||
("parent".to_string(), Value::String(parent_user.to_string())),
|
||||
(OIDC_VIRTUAL_PARENT_CLAIM.to_string(), Value::String(parent_user.to_string())),
|
||||
(POLICYNAME.to_string(), Value::String("readonly,deny-delete".to_string())),
|
||||
(OIDC_EXPLICIT_POLICIES_CLAIM.to_string(), serde_json::json!(["readonly", "deny-delete"])),
|
||||
("groups".to_string(), serde_json::json!(["testgroup"])),
|
||||
]);
|
||||
let groups = Some(vec!["testgroup".to_string()]);
|
||||
let conditions = HashMap::new();
|
||||
@@ -6430,7 +6070,7 @@ mod tests {
|
||||
|
||||
let prepared = iam_sys.prepare_sts_auth(&args, parent_user).await;
|
||||
|
||||
assert!(matches!(prepared.mode, PreparedIamMode::TerminalDeny));
|
||||
assert!(matches!(prepared.mode, PreparedIamMode::Deny));
|
||||
assert!(!iam_sys.eval_prepared(&prepared, &args).await);
|
||||
}
|
||||
|
||||
|
||||
@@ -33,18 +33,23 @@ default = []
|
||||
diagnostic-internode-http2-ping = []
|
||||
diagnostic-read-version-caller = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"rustfs-s3-ops/hotpath",
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-s3-ops/hotpath-alloc",
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-s3-ops/hotpath-cpu",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
metrics = { workspace = true }
|
||||
rustfs-s3-ops = { workspace = true }
|
||||
num_cpus = { workspace = true }
|
||||
|
||||
@@ -59,7 +59,6 @@ use std::sync::{
|
||||
LazyLock, Mutex,
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Global switch for detailed per-stage PUT metrics (path label, stage durations).
|
||||
/// When `false`, `record_put_object_path` and `record_put_object_stage_duration`
|
||||
@@ -450,125 +449,6 @@ pub const PUT_RENAME_QUORUM_FANOUT_STATE_PANIC: &str = "panic";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST: &str = "quorum_first";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL: &str = "quorum_tail";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR: &str = "error";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_ORDINARY: &str = "ordinary";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE: &str = "quota_fence";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_INCARNATION: &str = "incarnation";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_MULTIPART: &str = "multipart";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE: &str = "scanner_lease";
|
||||
pub const ECSTORE_RENAME_RPC_KIND_POOL_META_CAS: &str = "pool_meta_cas";
|
||||
pub const ECSTORE_RENAME_RPC_KINDS: [&str; 6] = [
|
||||
ECSTORE_RENAME_RPC_KIND_ORDINARY,
|
||||
ECSTORE_RENAME_RPC_KIND_QUOTA_FENCE,
|
||||
ECSTORE_RENAME_RPC_KIND_INCARNATION,
|
||||
ECSTORE_RENAME_RPC_KIND_MULTIPART,
|
||||
ECSTORE_RENAME_RPC_KIND_SCANNER_LEASE,
|
||||
ECSTORE_RENAME_RPC_KIND_POOL_META_CAS,
|
||||
];
|
||||
pub const ECSTORE_RENAME_PEER_WIDTHS: [&str; 18] = [
|
||||
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "gt16",
|
||||
];
|
||||
pub const ECSTORE_RENAME_COVERAGE: [&str; 3] = ["none", "partial", "full"];
|
||||
pub const ECSTORE_RENAME_BATCH_EXECUTION: [&str; 10] = [
|
||||
"planned_groups",
|
||||
"planned_items",
|
||||
"stream_groups",
|
||||
"stream_items",
|
||||
"completed_items",
|
||||
"fallback_groups",
|
||||
"fallback_items",
|
||||
"not_dispatched_items",
|
||||
"may_have_published_items",
|
||||
"terminal_missing_items",
|
||||
];
|
||||
pub const ECSTORE_RENAME_ADMISSION: [&str; 10] = [
|
||||
"unary_attempts",
|
||||
"unary_items",
|
||||
"unary_wait_ns_total",
|
||||
"unary_wait_ns_max",
|
||||
"unary_timeouts",
|
||||
"batch_attempts",
|
||||
"batch_items",
|
||||
"batch_wait_ns_total",
|
||||
"batch_wait_ns_max",
|
||||
"batch_timeouts",
|
||||
];
|
||||
pub const ECSTORE_RENAME_QUARANTINE: [&str; 12] = [
|
||||
"unary_events",
|
||||
"unary_items",
|
||||
"unary_hold_ns_total",
|
||||
"unary_hold_ns_max",
|
||||
"unary_released_items",
|
||||
"unary_no_runtime_items",
|
||||
"batch_events",
|
||||
"batch_items",
|
||||
"batch_hold_ns_total",
|
||||
"batch_hold_ns_max",
|
||||
"batch_released_items",
|
||||
"batch_no_runtime_items",
|
||||
];
|
||||
pub const ECSTORE_RENAME_BATCH_TIMING: [&str; 4] = [
|
||||
"client_stream_to_first",
|
||||
"client_first_to_last",
|
||||
"server_dispatch_to_first",
|
||||
"server_first_to_last",
|
||||
];
|
||||
|
||||
static ECSTORE_RENAME_RPC_COUNTS: [AtomicU64; 6] = [const { AtomicU64::new(0) }; 6];
|
||||
static ECSTORE_RENAME_REMOTE_WIDTH_COUNTS: [AtomicU64; 18] = [const { AtomicU64::new(0) }; 18];
|
||||
static ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS: [AtomicU64; 18] = [const { AtomicU64::new(0) }; 18];
|
||||
static ECSTORE_RENAME_COVERAGE_COUNTS: [AtomicU64; 3] = [const { AtomicU64::new(0) }; 3];
|
||||
static ECSTORE_RENAME_BATCH_EXECUTION_COUNTS: [AtomicU64; 10] = [const { AtomicU64::new(0) }; 10];
|
||||
static ECSTORE_RENAME_ADMISSION_COUNTS: [AtomicU64; 10] = [const { AtomicU64::new(0) }; 10];
|
||||
static ECSTORE_RENAME_QUARANTINE_COUNTS: [AtomicU64; 12] = [const { AtomicU64::new(0) }; 12];
|
||||
static ECSTORE_RENAME_BATCH_TIMING_COUNTS: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4];
|
||||
static ECSTORE_RENAME_BATCH_TIMING_TOTAL_NS: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4];
|
||||
static ECSTORE_RENAME_BATCH_TIMING_MAX_NS: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4];
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct EcstoreRenameDiagnosticsSnapshot {
|
||||
pub rpc_counts: [u64; 6],
|
||||
pub remote_width_counts: [u64; 18],
|
||||
pub batchable_width_counts: [u64; 18],
|
||||
pub coverage_counts: [u64; 3],
|
||||
pub batch_execution_counts: [u64; 10],
|
||||
pub admission_counts: [u64; 10],
|
||||
pub quarantine_counts: [u64; 12],
|
||||
pub batch_timing_counts: [u64; 4],
|
||||
pub batch_timing_total_ns: [u64; 4],
|
||||
pub batch_timing_max_ns: [u64; 4],
|
||||
}
|
||||
|
||||
impl EcstoreRenameDiagnosticsSnapshot {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.rpc_counts
|
||||
.iter()
|
||||
.chain(self.remote_width_counts.iter())
|
||||
.chain(self.batchable_width_counts.iter())
|
||||
.chain(self.coverage_counts.iter())
|
||||
.chain(self.batch_execution_counts.iter())
|
||||
.chain(self.admission_counts.iter())
|
||||
.chain(self.quarantine_counts.iter())
|
||||
.chain(self.batch_timing_counts.iter())
|
||||
.chain(self.batch_timing_total_ns.iter())
|
||||
.chain(self.batch_timing_max_ns.iter())
|
||||
.all(|count| *count == 0)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot_ecstore_rename_diagnostics() -> EcstoreRenameDiagnosticsSnapshot {
|
||||
EcstoreRenameDiagnosticsSnapshot {
|
||||
rpc_counts: std::array::from_fn(|index| ECSTORE_RENAME_RPC_COUNTS[index].load(Ordering::Relaxed)),
|
||||
remote_width_counts: std::array::from_fn(|index| ECSTORE_RENAME_REMOTE_WIDTH_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batchable_width_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS[index].load(Ordering::Relaxed)),
|
||||
coverage_counts: std::array::from_fn(|index| ECSTORE_RENAME_COVERAGE_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batch_execution_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_EXECUTION_COUNTS[index].load(Ordering::Relaxed)),
|
||||
admission_counts: std::array::from_fn(|index| ECSTORE_RENAME_ADMISSION_COUNTS[index].load(Ordering::Relaxed)),
|
||||
quarantine_counts: std::array::from_fn(|index| ECSTORE_RENAME_QUARANTINE_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batch_timing_counts: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_TIMING_COUNTS[index].load(Ordering::Relaxed)),
|
||||
batch_timing_total_ns: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_TIMING_TOTAL_NS[index].load(Ordering::Relaxed)),
|
||||
batch_timing_max_ns: std::array::from_fn(|index| ECSTORE_RENAME_BATCH_TIMING_MAX_NS[index].load(Ordering::Relaxed)),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn get_stage_metrics_enabled() -> bool {
|
||||
@@ -2783,169 +2663,6 @@ pub fn record_put_rename_quorum_wait_fanout(
|
||||
}
|
||||
}
|
||||
|
||||
/// Records a rename RPC after its canonical request digest has been attached.
|
||||
///
|
||||
/// This uses the PUT-stage diagnostics switch because it is temporary hot-path
|
||||
/// evidence, but the metric is ECStore-scoped: scanner and pool metadata
|
||||
/// renames are not S3 PutObject operations.
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_remote_rpc(kind: &'static str) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let Some(index) = ECSTORE_RENAME_RPC_KINDS.iter().position(|candidate| *candidate == kind) else {
|
||||
return;
|
||||
};
|
||||
ECSTORE_RENAME_RPC_COUNTS[index].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn rename_peer_width_index(width: usize) -> usize {
|
||||
width.min(17)
|
||||
}
|
||||
|
||||
/// Records the per-peer opportunity for a future body-bound rename batch.
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_candidate_peer(remote_disks: usize, batchable_disks: usize) {
|
||||
if !put_stage_metrics_enabled() || remote_disks == 0 {
|
||||
return;
|
||||
}
|
||||
ECSTORE_RENAME_REMOTE_WIDTH_COUNTS[rename_peer_width_index(remote_disks)].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_BATCHABLE_WIDTH_COUNTS[rename_peer_width_index(batchable_disks)].fetch_add(1, Ordering::Relaxed);
|
||||
let coverage_index = if batchable_disks == 0 {
|
||||
0
|
||||
} else if batchable_disks == remote_disks {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
};
|
||||
ECSTORE_RENAME_COVERAGE_COUNTS[coverage_index].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn record_ecstore_rename_batch_execution(index: usize, count: usize) {
|
||||
if put_stage_metrics_enabled() {
|
||||
let count = u64::try_from(count).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_BATCH_EXECUTION_COUNTS[index].fetch_add(count, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_planned(items: usize) {
|
||||
record_ecstore_rename_batch_execution(0, 1);
|
||||
record_ecstore_rename_batch_execution(1, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_stream_started(items: usize) {
|
||||
record_ecstore_rename_batch_execution(2, 1);
|
||||
record_ecstore_rename_batch_execution(3, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_item_completed() {
|
||||
record_ecstore_rename_batch_execution(4, 1);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_fallback(items: usize) {
|
||||
record_ecstore_rename_batch_execution(5, 1);
|
||||
record_ecstore_rename_batch_execution(6, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_not_dispatched(items: usize) {
|
||||
record_ecstore_rename_batch_execution(7, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_may_have_published(items: usize) {
|
||||
record_ecstore_rename_batch_execution(8, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_terminal_missing(items: usize) {
|
||||
record_ecstore_rename_batch_execution(9, items);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_admission(batch: bool, items: usize, wait: Duration, timed_out: bool) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let base = usize::from(batch) * 5;
|
||||
let items = u64::try_from(items).unwrap_or(u64::MAX);
|
||||
let wait_ns = u64::try_from(wait.as_nanos()).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base + 1].fetch_add(items, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base + 2].fetch_add(wait_ns, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base + 3].fetch_max(wait_ns, Ordering::Relaxed);
|
||||
if timed_out {
|
||||
ECSTORE_RENAME_ADMISSION_COUNTS[base + 4].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_quarantine(batch: bool, items: usize, hold: Duration) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let base = usize::from(batch) * 6;
|
||||
let items = u64::try_from(items).unwrap_or(u64::MAX);
|
||||
let hold_ns = u64::try_from(hold.as_nanos()).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 1].fetch_add(items, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 2].fetch_add(hold_ns, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 3].fetch_max(hold_ns, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_quarantine_released(batch: bool, items: usize) {
|
||||
if put_stage_metrics_enabled() {
|
||||
let base = usize::from(batch) * 6;
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 4].fetch_add(u64::try_from(items).unwrap_or(u64::MAX), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_quarantine_no_runtime(batch: bool, items: usize) {
|
||||
if put_stage_metrics_enabled() {
|
||||
let base = usize::from(batch) * 6;
|
||||
ECSTORE_RENAME_QUARANTINE_COUNTS[base + 5].fetch_add(u64::try_from(items).unwrap_or(u64::MAX), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn record_ecstore_rename_batch_timing(index: usize, duration: Duration) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let duration_ns = u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX);
|
||||
ECSTORE_RENAME_BATCH_TIMING_COUNTS[index].fetch_add(1, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_BATCH_TIMING_TOTAL_NS[index].fetch_add(duration_ns, Ordering::Relaxed);
|
||||
ECSTORE_RENAME_BATCH_TIMING_MAX_NS[index].fetch_max(duration_ns, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_client_stream_to_first(duration: Duration) {
|
||||
record_ecstore_rename_batch_timing(0, duration);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_client_first_to_last(duration: Duration) {
|
||||
record_ecstore_rename_batch_timing(1, duration);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_server_dispatch_to_first(duration: Duration) {
|
||||
record_ecstore_rename_batch_timing(2, duration);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_ecstore_rename_batch_server_first_to_last(duration: Duration) {
|
||||
record_ecstore_rename_batch_timing(3, duration);
|
||||
}
|
||||
|
||||
/// Record generic internal operation stage duration (non-PUT paths).
|
||||
/// Use this for metacache walks, listing, lifecycle, and other background
|
||||
/// operations that are NOT part of the PUT object hot path.
|
||||
@@ -4868,85 +4585,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ecstore_rename_batch_candidate_metrics_are_gated_and_bounded() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
set_put_stage_metrics_enabled(false);
|
||||
let disabled_before = snapshot_ecstore_rename_diagnostics();
|
||||
record_ecstore_rename_remote_rpc(ECSTORE_RENAME_RPC_KIND_ORDINARY);
|
||||
record_ecstore_rename_batch_candidate_peer(4, 4);
|
||||
record_ecstore_rename_admission(false, 1, Duration::from_nanos(10), false);
|
||||
record_ecstore_rename_quarantine(true, 4, Duration::from_nanos(30));
|
||||
record_ecstore_rename_batch_client_stream_to_first(Duration::from_nanos(10));
|
||||
assert_eq!(snapshot_ecstore_rename_diagnostics(), disabled_before);
|
||||
|
||||
set_put_stage_metrics_enabled(true);
|
||||
let before = snapshot_ecstore_rename_diagnostics();
|
||||
for kind in ECSTORE_RENAME_RPC_KINDS {
|
||||
record_ecstore_rename_remote_rpc(kind);
|
||||
}
|
||||
record_ecstore_rename_batch_candidate_peer(4, 4);
|
||||
record_ecstore_rename_batch_candidate_peer(4, 3);
|
||||
record_ecstore_rename_batch_candidate_peer(4, 0);
|
||||
record_ecstore_rename_batch_candidate_peer(20, 20);
|
||||
record_ecstore_rename_batch_planned(4);
|
||||
record_ecstore_rename_batch_stream_started(4);
|
||||
record_ecstore_rename_batch_item_completed();
|
||||
record_ecstore_rename_batch_fallback(3);
|
||||
record_ecstore_rename_batch_not_dispatched(2);
|
||||
record_ecstore_rename_batch_may_have_published(4);
|
||||
record_ecstore_rename_batch_terminal_missing(1);
|
||||
record_ecstore_rename_admission(false, 1, Duration::from_nanos(10), false);
|
||||
record_ecstore_rename_admission(true, 4, Duration::from_nanos(20), true);
|
||||
record_ecstore_rename_quarantine(true, 4, Duration::from_nanos(30));
|
||||
record_ecstore_rename_quarantine_released(true, 3);
|
||||
record_ecstore_rename_quarantine_no_runtime(true, 1);
|
||||
record_ecstore_rename_batch_client_stream_to_first(Duration::from_nanos(10));
|
||||
record_ecstore_rename_batch_client_first_to_last(Duration::from_nanos(20));
|
||||
record_ecstore_rename_batch_server_dispatch_to_first(Duration::from_nanos(30));
|
||||
record_ecstore_rename_batch_server_first_to_last(Duration::from_nanos(40));
|
||||
set_put_stage_metrics_enabled(false);
|
||||
|
||||
let after = snapshot_ecstore_rename_diagnostics();
|
||||
for index in 0..ECSTORE_RENAME_RPC_KINDS.len() {
|
||||
assert_eq!(after.rpc_counts[index] - before.rpc_counts[index], 1);
|
||||
}
|
||||
assert_eq!(after.remote_width_counts[4] - before.remote_width_counts[4], 3);
|
||||
assert_eq!(after.remote_width_counts[17] - before.remote_width_counts[17], 1);
|
||||
assert_eq!(after.batchable_width_counts[0] - before.batchable_width_counts[0], 1);
|
||||
assert_eq!(after.batchable_width_counts[3] - before.batchable_width_counts[3], 1);
|
||||
assert_eq!(after.batchable_width_counts[4] - before.batchable_width_counts[4], 1);
|
||||
assert_eq!(after.batchable_width_counts[17] - before.batchable_width_counts[17], 1);
|
||||
assert_eq!(after.coverage_counts[0] - before.coverage_counts[0], 1);
|
||||
assert_eq!(after.coverage_counts[1] - before.coverage_counts[1], 1);
|
||||
assert_eq!(after.coverage_counts[2] - before.coverage_counts[2], 2);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 10, _>(|index| {
|
||||
after.batch_execution_counts[index] - before.batch_execution_counts[index]
|
||||
}),
|
||||
[1, 4, 1, 4, 1, 1, 3, 2, 4, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 10, _>(|index| after.admission_counts[index] - before.admission_counts[index]),
|
||||
[1, 1, 10, 10, 0, 1, 4, 20, 20, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 12, _>(|index| after.quarantine_counts[index] - before.quarantine_counts[index]),
|
||||
[0, 0, 0, 0, 0, 0, 1, 4, 30, 30, 3, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 4, _>(|index| after.batch_timing_counts[index] - before.batch_timing_counts[index]),
|
||||
[1, 1, 1, 1]
|
||||
);
|
||||
assert_eq!(
|
||||
std::array::from_fn::<_, 4, _>(|index| { after.batch_timing_total_ns[index] - before.batch_timing_total_ns[index] }),
|
||||
[10, 20, 30, 40]
|
||||
);
|
||||
for (actual, expected) in after.batch_timing_max_ns.into_iter().zip([10, 20, 30, 40]) {
|
||||
assert!(actual >= expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_object_diagnostic_buckets() {
|
||||
assert_eq!(put_object_size_bucket(0), "unknown");
|
||||
|
||||
@@ -58,6 +58,8 @@ zeroize = { workspace = true, features = ["derive"] }
|
||||
|
||||
# Configuration and storage
|
||||
url = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
|
||||
# Caching
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
|
||||
|
||||
@@ -65,7 +65,7 @@ use aws_sdk_kms::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_kms::types::{DataKeySpec, KeySpec as AwsKeySpec, KeyState as AwsKeyState, KeyUsageType, Tag};
|
||||
use aws_smithy_runtime_api::client::orchestrator::HttpResponse;
|
||||
use aws_smithy_types::Blob;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, empty_key_page, list_keys_page_size};
|
||||
@@ -234,12 +234,12 @@ fn map_key_usage(usage: Option<&KeyUsageType>) -> KeyUsage {
|
||||
}
|
||||
}
|
||||
|
||||
fn to_zoned(value: Option<&aws_smithy_types::DateTime>) -> Option<ZonedDateTime> {
|
||||
fn to_zoned(value: Option<&aws_smithy_types::DateTime>) -> Option<Zoned> {
|
||||
let value = value?;
|
||||
let nanos = i32::try_from(value.subsec_nanos()).ok()?;
|
||||
jiff::Timestamp::new(value.secs(), nanos)
|
||||
.ok()
|
||||
.map(|timestamp| timestamp.to_zoned_datetime(jiff::tz::TimeZone::UTC))
|
||||
.map(|timestamp| timestamp.to_zoned(jiff::tz::TimeZone::UTC))
|
||||
}
|
||||
|
||||
/// `None` for an empty context so the request omits the field entirely rather
|
||||
@@ -593,7 +593,7 @@ impl KmsBackend for AwsKmsBackend {
|
||||
encrypted_key: ciphertext.into_inner(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: request.encryption_context,
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
// AWS addresses backing keys internally and never exposes a version.
|
||||
master_key_version: None,
|
||||
// AWS binds the context into the blob itself; this field describes
|
||||
@@ -876,7 +876,7 @@ impl KmsBackend for AwsKmsBackend {
|
||||
/// report what AWS has already done: a key that is gone counts as removed,
|
||||
/// a key that left `PendingDeletion` counts as a state change, and a key
|
||||
/// still pending is never expired by RustFS regardless of the deadline.
|
||||
async fn remove_expired_key(&self, key_id: &str, _now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, key_id: &str, _now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
match self.describe(key_id).await {
|
||||
Ok(metadata) => Ok(match metadata.key_state {
|
||||
KeyState::PendingDeletion => ExpiredKeyRemoval::NotExpired,
|
||||
@@ -1492,7 +1492,7 @@ mod tests {
|
||||
let (http_client, backend) = scripted_backend(vec![ok_event(key_metadata_json("test-key", "PendingDeletion"))]);
|
||||
assert_eq!(
|
||||
backend
|
||||
.remove_expired_key("test-key", &ZonedDateTime::now())
|
||||
.remove_expired_key("test-key", &Zoned::now())
|
||||
.await
|
||||
.expect("observation should succeed"),
|
||||
ExpiredKeyRemoval::NotExpired
|
||||
@@ -1502,7 +1502,7 @@ mod tests {
|
||||
let (_http, backend) = scripted_backend(vec![error_event(400, "NotFoundException", "key does not exist")]);
|
||||
assert_eq!(
|
||||
backend
|
||||
.remove_expired_key("test-key", &ZonedDateTime::now())
|
||||
.remove_expired_key("test-key", &Zoned::now())
|
||||
.await
|
||||
.expect("a key AWS already removed is a completed removal"),
|
||||
ExpiredKeyRemoval::Removed
|
||||
@@ -1511,7 +1511,7 @@ mod tests {
|
||||
let (_http, backend) = scripted_backend(vec![ok_event(key_metadata_json("test-key", "Enabled"))]);
|
||||
assert_eq!(
|
||||
backend
|
||||
.remove_expired_key("test-key", &ZonedDateTime::now())
|
||||
.remove_expired_key("test-key", &Zoned::now())
|
||||
.await
|
||||
.expect("observation should succeed"),
|
||||
ExpiredKeyRemoval::StateChanged
|
||||
|
||||
@@ -35,7 +35,7 @@ use aes_gcm::{
|
||||
use argon2::{Algorithm, Argon2, Params, Version};
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use rand::RngExt;
|
||||
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -582,14 +582,14 @@ struct StoredMasterKey {
|
||||
description: Option<String>,
|
||||
metadata: HashMap<String, String>,
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
created_at: ZonedDateTime,
|
||||
created_at: Zoned,
|
||||
#[serde(with = "crate::time_serde::option_zoned")]
|
||||
rotated_at: Option<ZonedDateTime>,
|
||||
rotated_at: Option<Zoned>,
|
||||
created_by: Option<String>,
|
||||
/// Scheduled deletion deadline; absent on records written before deadline
|
||||
/// persistence landed, so it must stay optional for backward compatibility.
|
||||
#[serde(default, with = "crate::time_serde::option_zoned")]
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
deletion_date: Option<Zoned>,
|
||||
/// Encrypted key material (32 bytes encoded in base64 for AES-256)
|
||||
encrypted_key_material: String,
|
||||
/// Nonce used for encryption
|
||||
@@ -685,10 +685,10 @@ impl<'de> Deserialize<'de> for StoredMasterKey {
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] ZonedDateTime);
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] Zoned);
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct OptionalZonedValue(#[serde(with = "crate::time_serde::option_zoned")] Option<ZonedDateTime>);
|
||||
struct OptionalZonedValue(#[serde(with = "crate::time_serde::option_zoned")] Option<Zoned>);
|
||||
|
||||
struct StoredMasterKeyVisitor;
|
||||
|
||||
@@ -1556,7 +1556,7 @@ impl LocalKmsClient {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding,
|
||||
};
|
||||
@@ -1596,7 +1596,7 @@ impl LocalKmsClient {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
// Local rotation is rejected, so the key has a single material version.
|
||||
master_key_version: None,
|
||||
context_binding,
|
||||
@@ -1860,7 +1860,7 @@ impl LocalKmsClient {
|
||||
let mut master_key = self.load_master_key(key_id).await?;
|
||||
ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::ScheduleDeletion)?;
|
||||
master_key.status = KeyStatus::PendingDeletion;
|
||||
master_key.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(pending_window_days as u64 * 86400));
|
||||
master_key.deletion_date = Some(Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400));
|
||||
|
||||
// Preserve the existing key material (see enable_key): scheduling deletion must not
|
||||
// regenerate the master key, or cancelling the deletion later would recover a key that
|
||||
@@ -1985,7 +1985,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: request.key_usage,
|
||||
description: request.description,
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
@@ -2113,7 +2113,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
Ok((_stored, key_material)) => {
|
||||
let mut tombstone = master_key.clone();
|
||||
tombstone.status = KeyStatus::Deleted;
|
||||
tombstone.deletion_date = Some(ZonedDateTime::now());
|
||||
tombstone.deletion_date = Some(Zoned::now());
|
||||
self.client.save_master_key(&tombstone, &key_material).await?;
|
||||
}
|
||||
Err(error) => {
|
||||
@@ -2136,7 +2136,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
key_usage: master_key.usage,
|
||||
key_state: KeyState::PendingDeletion, // AWS KMS compatibility
|
||||
creation_date: master_key.created_at,
|
||||
deletion_date: Some(ZonedDateTime::now()),
|
||||
deletion_date: Some(Zoned::now()),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
origin: "AWS_KMS".to_string(),
|
||||
tags: master_key.metadata,
|
||||
@@ -2162,7 +2162,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
)));
|
||||
}
|
||||
|
||||
let deletion_date = ZonedDateTime::now() + Duration::from_secs(days as u64 * 86400);
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
master_key.status = KeyStatus::PendingDeletion;
|
||||
master_key.deletion_date = Some(deletion_date.clone());
|
||||
|
||||
@@ -2317,7 +2317,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.with_update_key_metadata(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
// The per-key write lock serializes this against a concurrent
|
||||
// cancellation, closing the check-then-remove race.
|
||||
let _write_guard = self.client.lock_key_for_write(key_id).await;
|
||||
@@ -3088,12 +3088,12 @@ mod tests {
|
||||
description: Option<String>,
|
||||
metadata: HashMap<String, String>,
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
created_at: ZonedDateTime,
|
||||
created_at: Zoned,
|
||||
#[serde(with = "crate::time_serde::option_zoned")]
|
||||
rotated_at: Option<ZonedDateTime>,
|
||||
rotated_at: Option<Zoned>,
|
||||
created_by: Option<String>,
|
||||
#[serde(default, with = "crate::time_serde::option_zoned")]
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
deletion_date: Option<Zoned>,
|
||||
encrypted_key_material: String,
|
||||
nonce: Vec<u8>,
|
||||
#[serde(default)]
|
||||
@@ -4284,7 +4284,7 @@ mod tests {
|
||||
let (_stored, key_material) = client.decode_stored_key(&key_id).await.expect("decode stored key");
|
||||
let mut tombstone = client.load_master_key(&key_id).await.expect("load key");
|
||||
tombstone.status = KeyStatus::Deleted;
|
||||
tombstone.deletion_date = Some(ZonedDateTime::now());
|
||||
tombstone.deletion_date = Some(Zoned::now());
|
||||
client
|
||||
.save_master_key(&tombstone, &key_material)
|
||||
.await
|
||||
@@ -4292,7 +4292,7 @@ mod tests {
|
||||
|
||||
// The sweep primitive completes the crashed removal...
|
||||
let outcome = backend
|
||||
.remove_expired_key(&key_id, &ZonedDateTime::now())
|
||||
.remove_expired_key(&key_id, &Zoned::now())
|
||||
.await
|
||||
.expect("tombstone completion");
|
||||
assert_eq!(outcome, crate::backends::ExpiredKeyRemoval::Removed);
|
||||
@@ -4300,7 +4300,7 @@ mod tests {
|
||||
|
||||
// ...and stays idempotent once the key is gone.
|
||||
let outcome = backend
|
||||
.remove_expired_key(&key_id, &ZonedDateTime::now())
|
||||
.remove_expired_key(&key_id, &Zoned::now())
|
||||
.await
|
||||
.expect("repeat removal");
|
||||
assert_eq!(outcome, crate::backends::ExpiredKeyRemoval::Removed);
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -608,7 +608,7 @@ pub trait KmsBackend: Send + Sync {
|
||||
///
|
||||
/// The default rejects the operation for backends without deletion
|
||||
/// support.
|
||||
async fn remove_expired_key(&self, _key_id: &str, _now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, _key_id: &str, _now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
Err(KmsError::unsupported_capability("backend without deletion support", "remove_expired_key"))
|
||||
}
|
||||
|
||||
@@ -934,7 +934,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn default_remove_expired_key_is_unsupported() {
|
||||
let error = MinimalBackend
|
||||
.remove_expired_key("any-key", &jiff::ZonedDateTime::now())
|
||||
.remove_expired_key("any-key", &jiff::Zoned::now())
|
||||
.await
|
||||
.expect_err("backends without deletion support must reject expired-key removal");
|
||||
assert!(
|
||||
|
||||
@@ -34,7 +34,7 @@ use aes_gcm::{
|
||||
aead::{Aead, KeyInit, Payload},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use rand::RngExt;
|
||||
use std::collections::HashMap;
|
||||
use tracing::debug;
|
||||
@@ -65,7 +65,7 @@ pub struct StaticKmsBackend {
|
||||
/// across a restart and differs between nodes of one cluster, because there
|
||||
/// is no creation event to anchor it to; callers must treat it as "when this
|
||||
/// node loaded the key", not as the key's birth date.
|
||||
created_at: ZonedDateTime,
|
||||
created_at: Zoned,
|
||||
}
|
||||
|
||||
impl StaticKmsBackend {
|
||||
@@ -89,7 +89,7 @@ impl StaticKmsBackend {
|
||||
Ok(Self {
|
||||
key_id: static_config.key_id.clone(),
|
||||
key: key.into(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -149,7 +149,7 @@ impl StaticKmsBackend {
|
||||
encrypted_key: encrypted,
|
||||
nonce: nonce_bytes.to_vec(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
// The static backend has a single fixed key with no rotation.
|
||||
master_key_version: None,
|
||||
// The context is already bound as AAD by the Static cipher path
|
||||
@@ -200,7 +200,7 @@ impl StaticKmsBackend {
|
||||
encrypted_key: encrypted,
|
||||
nonce: nonce_bytes.to_vec(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
// The static backend has a single fixed key with no rotation.
|
||||
master_key_version: None,
|
||||
// The context is already bound as AAD by the Static cipher path
|
||||
|
||||
@@ -35,7 +35,7 @@ use crate::policy::{self, AttemptError, OpClass, OwnWrite, OwnWrites, RetryPolic
|
||||
use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
@@ -90,7 +90,7 @@ struct VaultKeyData {
|
||||
/// Key usage type
|
||||
usage: KeyUsage,
|
||||
/// Key creation timestamp
|
||||
created_at: ZonedDateTime,
|
||||
created_at: Zoned,
|
||||
/// Key status
|
||||
status: KeyStatus,
|
||||
/// Key version
|
||||
@@ -104,7 +104,7 @@ struct VaultKeyData {
|
||||
/// Scheduled deletion deadline; absent on records written before deadline
|
||||
/// persistence landed, so it must stay optional for backward compatibility.
|
||||
#[serde(default)]
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
deletion_date: Option<Zoned>,
|
||||
/// When the key's material last became current through a rotation.
|
||||
///
|
||||
/// Written by [`VaultKmsClient::rotate_key`] as part of the same
|
||||
@@ -117,7 +117,7 @@ struct VaultKeyData {
|
||||
/// See [`crate::deletion_worker`] for why collapsing the two is safe for the
|
||||
/// rotation-age gauge.
|
||||
#[serde(default)]
|
||||
rotated_at: Option<ZonedDateTime>,
|
||||
rotated_at: Option<Zoned>,
|
||||
/// Encrypted key material (base64 encoded)
|
||||
encrypted_key_material: String,
|
||||
/// Version that pre-versioning envelopes (no `master_key_version`) resolve to.
|
||||
@@ -381,7 +381,7 @@ struct VaultKeyVersionRecord {
|
||||
/// Encrypted key material (base64 encoded)
|
||||
encrypted_key_material: String,
|
||||
/// When this version's material was created
|
||||
created_at: ZonedDateTime,
|
||||
created_at: Zoned,
|
||||
}
|
||||
|
||||
/// Sub-path (under each key path) reserved for immutable version records.
|
||||
@@ -1305,7 +1305,7 @@ impl VaultKmsClient {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: Some(key_data.version),
|
||||
context_binding,
|
||||
};
|
||||
@@ -1347,7 +1347,7 @@ impl VaultKmsClient {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: Some(key_data.version),
|
||||
context_binding,
|
||||
};
|
||||
@@ -1718,7 +1718,7 @@ impl VaultKmsClient {
|
||||
let key_data = VaultKeyData {
|
||||
algorithm: algorithm.to_string(),
|
||||
usage: attributes.map_or(KeyUsage::EncryptDecrypt, |request| request.key_usage.clone()),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
status: KeyStatus::Active,
|
||||
version: 1,
|
||||
description: attributes.and_then(|request| request.description.clone()),
|
||||
@@ -2003,7 +2003,7 @@ impl VaultKmsClient {
|
||||
let record = VaultKeyVersionRecord {
|
||||
version: new_version,
|
||||
encrypted_key_material: new_material.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
};
|
||||
if !self.try_create_key_version_record(key_id, &record).await? {
|
||||
// A record for the next version already exists: an interrupted rotation
|
||||
@@ -2027,7 +2027,7 @@ impl VaultKmsClient {
|
||||
// the recorded time never runs ahead of the current version.
|
||||
key_data.version = new_version;
|
||||
key_data.encrypted_key_material = new_material;
|
||||
key_data.rotated_at = Some(ZonedDateTime::now());
|
||||
key_data.rotated_at = Some(Zoned::now());
|
||||
// Fresh material, fresh AES-GCM nonce budget: the wrap ceiling is per
|
||||
// key material version, so the counter restarts with the same commit
|
||||
// that makes the new material current.
|
||||
@@ -2114,7 +2114,7 @@ impl VaultKmsBackend {
|
||||
/// record whose key material is missing: persisting it would cement the
|
||||
/// empty-material state under a fresh document version — a damaged key
|
||||
/// must go through an explicit repair operation, not a lifecycle update.
|
||||
async fn mark_key_pending_deletion(&self, key_id: &str, deletion_date: &ZonedDateTime) -> Result<()> {
|
||||
async fn mark_key_pending_deletion(&self, key_id: &str, deletion_date: &Zoned) -> Result<()> {
|
||||
self.client
|
||||
.update_key_data_with_cas(key_id, |key_data| {
|
||||
ensure_key_status_permits(key_id, &key_data.status, StateGatedOperation::ScheduleDeletion)?;
|
||||
@@ -2144,7 +2144,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: request.key_usage,
|
||||
description: request.description,
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "VAULT".to_string(),
|
||||
key_manager: "VAULT".to_string(),
|
||||
@@ -2287,7 +2287,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
.await?;
|
||||
self.client.delete_key(key_id).await?;
|
||||
key_metadata.key_state = KeyState::PendingDeletion;
|
||||
key_metadata.deletion_date = Some(ZonedDateTime::now());
|
||||
key_metadata.deletion_date = Some(Zoned::now());
|
||||
|
||||
// No deletion date: the key is permanently gone, not scheduled.
|
||||
None
|
||||
@@ -2306,7 +2306,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
)));
|
||||
}
|
||||
|
||||
let deletion_date = ZonedDateTime::now() + Duration::from_secs(days as u64 * 86400);
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
self.mark_key_pending_deletion(key_id, &deletion_date).await?;
|
||||
key_metadata.key_state = KeyState::PendingDeletion;
|
||||
key_metadata.deletion_date = Some(deletion_date.clone());
|
||||
@@ -2415,7 +2415,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
.with_production_supported(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
// Tombstone under check-and-set with per-attempt re-validation: a
|
||||
// cancellation landing between the read and the write makes the write
|
||||
// conflict, and the re-read then observes the cancelled state and
|
||||
@@ -2513,7 +2513,7 @@ mod tests {
|
||||
VaultKeyData {
|
||||
algorithm: "AES_256".to_string(),
|
||||
usage: KeyUsage::EncryptDecrypt,
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
status: KeyStatus::Active,
|
||||
version: 1,
|
||||
description: None,
|
||||
@@ -3197,7 +3197,7 @@ mod tests {
|
||||
let key_data = VaultKeyData {
|
||||
algorithm: "AES_256".to_string(),
|
||||
usage: KeyUsage::EncryptDecrypt,
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
status: KeyStatus::Active,
|
||||
version: 1,
|
||||
description: None,
|
||||
@@ -3235,8 +3235,8 @@ mod tests {
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.wrap_budget_reserved = 7_000_000;
|
||||
key_data.baseline_version = Some(2);
|
||||
key_data.rotated_at = Some(ZonedDateTime::now());
|
||||
key_data.deletion_date = Some(ZonedDateTime::now());
|
||||
key_data.rotated_at = Some(Zoned::now());
|
||||
key_data.deletion_date = Some(Zoned::now());
|
||||
key_data.description = Some("described".to_string());
|
||||
|
||||
let value = serde_json::to_value(&key_data).expect("serialize key data");
|
||||
@@ -3683,11 +3683,11 @@ mod tests {
|
||||
/// of deadline persistence turns this test red.
|
||||
#[test]
|
||||
fn vault_key_data_deletion_date_round_trips_and_stays_backward_compatible() {
|
||||
let deadline = ZonedDateTime::now() + Duration::from_secs(7 * 86400);
|
||||
let deadline = Zoned::now() + Duration::from_secs(7 * 86400);
|
||||
let key_data = VaultKeyData {
|
||||
algorithm: "AES_256".to_string(),
|
||||
usage: KeyUsage::EncryptDecrypt,
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
status: KeyStatus::PendingDeletion,
|
||||
version: 1,
|
||||
description: None,
|
||||
@@ -3703,7 +3703,7 @@ mod tests {
|
||||
let mut value = serde_json::to_value(&key_data).expect("serialize");
|
||||
let restored: VaultKeyData = serde_json::from_value(value.clone()).expect("round trip");
|
||||
assert_eq!(
|
||||
restored.deletion_date.as_ref().map(ZonedDateTime::timestamp),
|
||||
restored.deletion_date.as_ref().map(Zoned::timestamp),
|
||||
Some(deadline.timestamp()),
|
||||
"deletion deadline must survive the KV2 round trip"
|
||||
);
|
||||
@@ -4055,10 +4055,10 @@ mod tests {
|
||||
/// cancellation and destroyed the key.
|
||||
#[tokio::test]
|
||||
async fn wired_expired_key_sweep_yields_to_concurrent_cancellation() {
|
||||
let now = ZonedDateTime::now() + Duration::from_secs(3600);
|
||||
let now = Zoned::now() + Duration::from_secs(3600);
|
||||
let mut pending = healthy_key_data();
|
||||
pending.status = KeyStatus::PendingDeletion;
|
||||
pending.deletion_date = Some(ZonedDateTime::now());
|
||||
pending.deletion_date = Some(Zoned::now());
|
||||
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
ScriptedResponse::ok(kv2_metadata_read_data(1)),
|
||||
@@ -4103,7 +4103,7 @@ mod tests {
|
||||
async fn wired_cancel_deletion_after_sweep_tombstone_fails_closed() {
|
||||
let mut pending = healthy_key_data();
|
||||
pending.status = KeyStatus::PendingDeletion;
|
||||
pending.deletion_date = Some(ZonedDateTime::now());
|
||||
pending.deletion_date = Some(Zoned::now());
|
||||
let mut tombstoned = healthy_key_data();
|
||||
tombstoned.status = KeyStatus::Deleted;
|
||||
|
||||
@@ -4192,7 +4192,7 @@ mod tests {
|
||||
async fn wired_immediate_deletion_of_a_pending_key_destroys_it() {
|
||||
let mut pending = healthy_key_data();
|
||||
pending.status = KeyStatus::PendingDeletion;
|
||||
pending.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(7 * 86400));
|
||||
pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
|
||||
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// describe_key: key info plus stored metadata.
|
||||
@@ -4328,7 +4328,7 @@ mod tests {
|
||||
async fn wired_schedule_deletion_regates_after_conflict() {
|
||||
let mut already_pending = healthy_key_data();
|
||||
already_pending.status = KeyStatus::PendingDeletion;
|
||||
already_pending.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(7 * 86400));
|
||||
already_pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
|
||||
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
|
||||
@@ -4747,7 +4747,7 @@ mod tests {
|
||||
let record_v2 = VaultKeyVersionRecord {
|
||||
version: 2,
|
||||
encrypted_key_material: base64_simd::STANDARD.encode_to_string(material_v2),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
};
|
||||
// A well-formed envelope wrapped under version 2 — under a reverted
|
||||
// guard this decrypt would *succeed*, which is exactly the masked
|
||||
@@ -4763,7 +4763,7 @@ mod tests {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: Some(2),
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -4843,7 +4843,7 @@ mod tests {
|
||||
let record_v2 = VaultKeyVersionRecord {
|
||||
version: 2,
|
||||
encrypted_key_material: adopted_material.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
};
|
||||
|
||||
let (vault, client) = scripted_client(vec![
|
||||
@@ -5042,7 +5042,7 @@ mod tests {
|
||||
/// Reverting either half turns this test red.
|
||||
#[tokio::test]
|
||||
async fn wired_rotate_persists_rotation_time_and_describe_reports_it() {
|
||||
let created_at = ZonedDateTime::now() - Duration::from_secs(365 * 86400);
|
||||
let created_at = Zoned::now() - Duration::from_secs(365 * 86400);
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.created_at = created_at.clone();
|
||||
key_data.version = 2;
|
||||
@@ -5069,7 +5069,7 @@ mod tests {
|
||||
let persisted: VaultKeyData =
|
||||
serde_json::from_value(committed["data"].clone()).expect("the committed record must deserialize");
|
||||
assert_eq!(
|
||||
persisted.rotated_at.as_ref().map(ZonedDateTime::timestamp),
|
||||
persisted.rotated_at.as_ref().map(Zoned::timestamp),
|
||||
Some(reported.timestamp()),
|
||||
"the rotation must persist the time it reports: {committed}"
|
||||
);
|
||||
@@ -5091,12 +5091,12 @@ mod tests {
|
||||
.await
|
||||
.expect("describe the rotated key");
|
||||
assert_eq!(
|
||||
described.rotated_at.as_ref().map(ZonedDateTime::timestamp),
|
||||
described.rotated_at.as_ref().map(Zoned::timestamp),
|
||||
Some(reported.timestamp()),
|
||||
"describe must report the persisted rotation time"
|
||||
);
|
||||
assert_ne!(
|
||||
described.rotated_at.as_ref().map(ZonedDateTime::timestamp),
|
||||
described.rotated_at.as_ref().map(Zoned::timestamp),
|
||||
Some(created_at.timestamp()),
|
||||
"a rotated key must not be aged from its creation"
|
||||
);
|
||||
@@ -5109,7 +5109,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn wired_describe_key_invents_no_rotation_time_for_legacy_records() {
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.created_at = ZonedDateTime::now() - Duration::from_secs(365 * 86400);
|
||||
key_data.created_at = Zoned::now() - Duration::from_secs(365 * 86400);
|
||||
key_data.version = 4;
|
||||
key_data.baseline_version = Some(1);
|
||||
|
||||
@@ -5143,7 +5143,7 @@ mod tests {
|
||||
/// rest of their contents intact.
|
||||
#[test]
|
||||
fn vault_key_data_rotated_at_round_trips_and_stays_backward_compatible() {
|
||||
let rotated_at = ZonedDateTime::now();
|
||||
let rotated_at = Zoned::now();
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.rotated_at = Some(rotated_at.clone());
|
||||
key_data.baseline_version = Some(1);
|
||||
@@ -5153,7 +5153,7 @@ mod tests {
|
||||
let mut value = serde_json::to_value(&key_data).expect("serialize");
|
||||
let restored: VaultKeyData = serde_json::from_value(value.clone()).expect("round trip");
|
||||
assert_eq!(
|
||||
restored.rotated_at.as_ref().map(ZonedDateTime::timestamp),
|
||||
restored.rotated_at.as_ref().map(Zoned::timestamp),
|
||||
Some(rotated_at.timestamp()),
|
||||
"the rotation time must survive the KV2 round trip"
|
||||
);
|
||||
@@ -5190,7 +5190,7 @@ mod tests {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -5250,7 +5250,7 @@ mod tests {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -5302,7 +5302,7 @@ mod tests {
|
||||
encrypted_key,
|
||||
nonce,
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
|
||||
@@ -33,7 +33,7 @@ use crate::policy::{self, AttemptError, OpClass, OwnWrite, OwnWrites, RetryPolic
|
||||
use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use moka::future::Cache;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
@@ -141,8 +141,8 @@ struct TransitKeyMetadata {
|
||||
description: Option<String>,
|
||||
tags: HashMap<String, String>,
|
||||
key_state: KeyState,
|
||||
created_at: ZonedDateTime,
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
created_at: Zoned,
|
||||
deletion_date: Option<Zoned>,
|
||||
origin: String,
|
||||
created_by: Option<String>,
|
||||
current_version: u32,
|
||||
@@ -160,8 +160,8 @@ struct TransitKeyMetadataPersisted {
|
||||
description: Option<String>,
|
||||
tags: HashMap<String, String>,
|
||||
key_state: KeyState,
|
||||
created_at: ZonedDateTime,
|
||||
deletion_date: Option<ZonedDateTime>,
|
||||
created_at: Zoned,
|
||||
deletion_date: Option<Zoned>,
|
||||
origin: String,
|
||||
created_by: Option<String>,
|
||||
current_version: u32,
|
||||
@@ -336,7 +336,7 @@ impl TransitKeyMetadata {
|
||||
description: request.description.clone(),
|
||||
tags: request.tags.clone(),
|
||||
key_state: KeyState::Enabled,
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: request.origin.clone().unwrap_or_else(|| "VAULT_TRANSIT".to_string()),
|
||||
created_by: None,
|
||||
@@ -360,7 +360,7 @@ impl TransitKeyMetadata {
|
||||
description: None,
|
||||
tags: HashMap::new(),
|
||||
key_state: KeyState::Enabled,
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "VAULT_TRANSIT".to_string(),
|
||||
created_by: None,
|
||||
@@ -1055,7 +1055,7 @@ impl VaultTransitKmsClient {
|
||||
encrypted_key: encrypted_key.into_bytes(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
// Transit ciphertext already self-describes its key version
|
||||
// ("vault:vN:..."), so the envelope never carries one.
|
||||
master_key_version: None,
|
||||
@@ -1100,7 +1100,7 @@ impl VaultTransitKmsClient {
|
||||
encrypted_key: encrypted.into_bytes(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: request.encryption_context.clone(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
// Transit ciphertext already self-describes its key version
|
||||
// ("vault:vN:..."), so the envelope never carries one.
|
||||
master_key_version: None,
|
||||
@@ -1543,7 +1543,7 @@ impl VaultTransitKmsClient {
|
||||
pending_window_days: u32,
|
||||
_context: Option<&OperationContext>,
|
||||
) -> Result<()> {
|
||||
let deletion_date = ZonedDateTime::now() + Duration::from_secs(pending_window_days as u64 * 86400);
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(pending_window_days as u64 * 86400);
|
||||
self.mutate_key_metadata(key_id, |metadata| {
|
||||
ensure_key_state_permits(key_id, &metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
|
||||
metadata.key_state = KeyState::PendingDeletion;
|
||||
@@ -1592,7 +1592,7 @@ impl VaultTransitKmsClient {
|
||||
description: metadata.description,
|
||||
metadata: metadata.tags,
|
||||
created_at: metadata.created_at,
|
||||
rotated_at: Some(ZonedDateTime::now()),
|
||||
rotated_at: Some(Zoned::now()),
|
||||
created_by: metadata.created_by,
|
||||
deletion_date: None,
|
||||
})
|
||||
@@ -1810,7 +1810,7 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
// Transit key is destroyed leaves a key that is already unusable.
|
||||
// Re-checked against every fresh snapshot: a concurrent
|
||||
// cancellation that commits first fails this removal.
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
self.client
|
||||
.mutate_key_metadata(&key_id, |metadata| match metadata.key_state {
|
||||
KeyState::PendingDeletion | KeyState::Unavailable => {
|
||||
@@ -1844,7 +1844,7 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
)));
|
||||
}
|
||||
|
||||
let scheduled = ZonedDateTime::now() + Duration::from_secs(days as u64 * 86400);
|
||||
let scheduled = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
self.client
|
||||
.mutate_key_metadata(&key_id, |metadata| {
|
||||
// Re-run the gate against every fresh snapshot: the check
|
||||
@@ -1934,7 +1934,7 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
.with_production_supported(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &ZonedDateTime) -> Result<ExpiredKeyRemoval> {
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
// The transit key's existence anchors "already removed": once it is
|
||||
// gone only stale scheduling metadata can remain, so clean that up.
|
||||
match self.client.read_transit_key(key_id).await {
|
||||
@@ -3220,7 +3220,7 @@ mod tests {
|
||||
encrypted_key: b"vault:v1:AAAA".to_vec(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: Some(1),
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -3297,7 +3297,7 @@ mod tests {
|
||||
async fn wired_immediate_deletion_of_a_pending_key_destroys_it() {
|
||||
let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
pending.key_state = KeyState::PendingDeletion;
|
||||
pending.deletion_date = Some(ZonedDateTime::now() + Duration::from_secs(7 * 86400));
|
||||
pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// Current key metadata for the response.
|
||||
ScriptedResponse::ok(transit_key_read_data("wired-key")),
|
||||
@@ -3354,7 +3354,7 @@ mod tests {
|
||||
async fn wired_expired_sweep_backs_off_when_cancel_wins_the_cas_race() {
|
||||
let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
pending.key_state = KeyState::PendingDeletion;
|
||||
pending.deletion_date = Some(ZonedDateTime::now());
|
||||
pending.deletion_date = Some(Zoned::now());
|
||||
let cancelled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// The transit key still exists.
|
||||
@@ -3378,7 +3378,7 @@ mod tests {
|
||||
.await
|
||||
.expect("vault transit backend should build");
|
||||
|
||||
let now = ZonedDateTime::now() + Duration::from_secs(3600);
|
||||
let now = Zoned::now() + Duration::from_secs(3600);
|
||||
let outcome = backend
|
||||
.remove_expired_key("wired-key", &now)
|
||||
.await
|
||||
|
||||
@@ -75,7 +75,7 @@ use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::service::ObjectEncryptionService;
|
||||
use crate::types::{CreateKeyRequest, EncryptionAlgorithm, EncryptionMetadata, KeyUsage};
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
@@ -272,10 +272,10 @@ pub struct DrillRecoveryEvidence {
|
||||
pub struct DrillRpoEvidence {
|
||||
/// When the export sealed the bundle.
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub snapshot_completed_at: ZonedDateTime,
|
||||
pub snapshot_completed_at: Zoned,
|
||||
/// When the disaster struck.
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub disaster_at: ZonedDateTime,
|
||||
pub disaster_at: Zoned,
|
||||
/// The data-loss window: work committed inside it is not in the bundle.
|
||||
pub rpo_window_millis: u64,
|
||||
/// Keys created after the fence closed, which the bundle cannot contain.
|
||||
@@ -322,7 +322,7 @@ pub struct DrillEvidence {
|
||||
pub dataset: DrillDataset,
|
||||
/// When the run started.
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub started_at: ZonedDateTime,
|
||||
pub started_at: Zoned,
|
||||
/// Bundle evidence.
|
||||
pub bundle: DrillBundleEvidence,
|
||||
/// Recovery evidence.
|
||||
@@ -508,7 +508,7 @@ impl PhaseClock {
|
||||
/// still produces the record of *how* it failed.
|
||||
pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result<DrillEvidence> {
|
||||
request.validate()?;
|
||||
let started_at = ZonedDateTime::now();
|
||||
let started_at = Zoned::now();
|
||||
let mut clock = PhaseClock::new();
|
||||
let mut findings = Vec::new();
|
||||
|
||||
@@ -557,7 +557,7 @@ pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result<
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let snapshot_completed_at = ZonedDateTime::now();
|
||||
let snapshot_completed_at = Zoned::now();
|
||||
clock.finish(DrillPhase::Export);
|
||||
|
||||
// Past the fence: this key belongs to the window the bundle cannot
|
||||
@@ -576,7 +576,7 @@ pub async fn run_local_drill(kek: &BackupKek, request: &DrillRequest) -> Result<
|
||||
let bundle_digest_before = tree_digest(&bundle_dir).await?;
|
||||
|
||||
let destroyed_files = apply_disaster(&live_dir, request.disaster).await?;
|
||||
let disaster_at = ZonedDateTime::now();
|
||||
let disaster_at = Zoned::now();
|
||||
clock.finish(DrillPhase::Disaster);
|
||||
|
||||
// Recovery step one is always the same: preserve what survived, then
|
||||
@@ -980,7 +980,7 @@ fn walk_files(root: &Path) -> std::io::Result<Vec<PathBuf>> {
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn elapsed_millis(from: &ZonedDateTime, to: &ZonedDateTime) -> u64 {
|
||||
fn elapsed_millis(from: &Zoned, to: &Zoned) -> u64 {
|
||||
let nanos = to.timestamp().as_nanosecond() - from.timestamp().as_nanosecond();
|
||||
u64::try_from(nanos / 1_000_000).unwrap_or(0)
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ use aes_gcm::{
|
||||
Aes256Gcm, Key, Nonce,
|
||||
aead::{Aead, KeyInit, Payload},
|
||||
};
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use rand::RngExt;
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -454,7 +454,7 @@ async fn build_and_write_bundle(
|
||||
// Normalized to UTC so the stored spelling is host-independent: the
|
||||
// local zone's name (or a POSIX TZ string in minimal containers) has
|
||||
// no business inside a portable bundle.
|
||||
created_at: ZonedDateTime::now().with_time_zone(jiff::tz::TimeZone::UTC),
|
||||
created_at: Zoned::now().with_time_zone(jiff::tz::TimeZone::UTC),
|
||||
rustfs_version: request.rustfs_version.clone(),
|
||||
deployment_identity: request.deployment_identity.clone(),
|
||||
backend: BackupBackendKind::Local,
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
use crate::backup::capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
|
||||
use crate::backup::error::BackupError;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeSet;
|
||||
@@ -503,7 +503,7 @@ pub struct BackupManifest {
|
||||
pub backup_id: String,
|
||||
/// Creation time of the bundle.
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub created_at: ZonedDateTime,
|
||||
pub created_at: Zoned,
|
||||
/// RustFS version that produced the bundle.
|
||||
pub rustfs_version: String,
|
||||
/// Opaque identity of the producing deployment, used by restore preflight
|
||||
|
||||
@@ -239,7 +239,7 @@ impl KmsCache {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{KeyState, KeyUsage};
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use std::time::Duration;
|
||||
@@ -264,7 +264,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
@@ -342,7 +342,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: Some("Test key".to_string()),
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
@@ -376,7 +376,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: Some("TTL test key".to_string()),
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
@@ -406,7 +406,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::backends::{ExpiredKeyRemoval, KmsBackend};
|
||||
use crate::error::Result;
|
||||
use crate::types::{KeyInfo, KeyStatus, ListKeysRequest, OperationContext};
|
||||
use async_trait::async_trait;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -117,7 +117,7 @@ struct KeyCensus {
|
||||
}
|
||||
|
||||
impl KeyCensus {
|
||||
fn observe(&mut self, key: &KeyInfo, now: &ZonedDateTime) {
|
||||
fn observe(&mut self, key: &KeyInfo, now: &Zoned) {
|
||||
match key.status {
|
||||
KeyStatus::PendingDeletion => self.pending_deletion += 1,
|
||||
KeyStatus::Deleted => self.tombstones += 1,
|
||||
@@ -145,7 +145,7 @@ impl KeyCensus {
|
||||
|
||||
/// Seconds from `earlier` to `now`, clamped at zero so clock skew (or a
|
||||
/// timestamp persisted by a node running ahead) cannot produce a negative age.
|
||||
fn seconds_between(earlier: &ZonedDateTime, now: &ZonedDateTime) -> f64 {
|
||||
fn seconds_between(earlier: &Zoned, now: &Zoned) -> f64 {
|
||||
(now.timestamp().as_second() - earlier.timestamp().as_second()).max(0) as f64
|
||||
}
|
||||
|
||||
@@ -273,7 +273,7 @@ impl DeletionWorker {
|
||||
}
|
||||
_ = ticker.tick() => {}
|
||||
}
|
||||
let report = self.sweep(&ZonedDateTime::now()).await;
|
||||
let report = self.sweep(&Zoned::now()).await;
|
||||
if !report.removed.is_empty() || !report.blocked.is_empty() || report.failed > 0 || report.unreadable > 0 {
|
||||
info!(
|
||||
removed = ?report.removed,
|
||||
@@ -293,7 +293,7 @@ impl DeletionWorker {
|
||||
/// Also feeds the lifecycle gauges: every key it lists is counted into a
|
||||
/// [`KeyCensus`] on the way past, so the observability comes out of the
|
||||
/// pages the sweep already had to fetch.
|
||||
pub(crate) async fn sweep(&self, now: &ZonedDateTime) -> SweepReport {
|
||||
pub(crate) async fn sweep(&self, now: &Zoned) -> SweepReport {
|
||||
let mut report = SweepReport::default();
|
||||
let mut census = KeyCensus::default();
|
||||
let mut marker: Option<String> = None;
|
||||
@@ -359,7 +359,7 @@ impl DeletionWorker {
|
||||
}
|
||||
|
||||
/// Handle one key that is on its way out; reports whether it was removed.
|
||||
async fn process_key(&self, key_id: &str, now: &ZonedDateTime, report: &mut SweepReport) -> bool {
|
||||
async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) -> bool {
|
||||
// Never remove a key that live configuration still points at. The
|
||||
// default key check is built in; broader references (bucket
|
||||
// encryption settings, ...) come from the injected checker.
|
||||
@@ -464,8 +464,8 @@ mod tests {
|
||||
DeletionWorker::new(backend, None, None, "local")
|
||||
}
|
||||
|
||||
fn after_window() -> ZonedDateTime {
|
||||
ZonedDateTime::now() + Duration::from_secs(8 * 86400)
|
||||
fn after_window() -> Zoned {
|
||||
Zoned::now() + Duration::from_secs(8 * 86400)
|
||||
}
|
||||
|
||||
async fn assert_key_gone(backend: &LocalKmsBackend, key_id: &str) {
|
||||
@@ -488,7 +488,7 @@ mod tests {
|
||||
let worker = worker(backend.clone());
|
||||
|
||||
// Not yet due: nothing happens.
|
||||
let report = worker.sweep(&ZonedDateTime::now()).await;
|
||||
let report = worker.sweep(&Zoned::now()).await;
|
||||
assert!(report.removed.is_empty());
|
||||
assert_eq!(report.skipped, 1);
|
||||
assert_eq!(report.failed, 0);
|
||||
@@ -635,7 +635,7 @@ mod tests {
|
||||
let worker = DeletionWorker::new(backend.clone(), None, None, "local").with_audit_sink(Some(sink.clone()));
|
||||
|
||||
// A key that is not yet due was never at risk, so it produces no record.
|
||||
worker.sweep(&ZonedDateTime::now()).await;
|
||||
worker.sweep(&Zoned::now()).await;
|
||||
assert!(
|
||||
sink.records.lock().expect("sink lock").is_empty(),
|
||||
"a key the sweep declined to touch must not be audited as a deletion"
|
||||
@@ -780,7 +780,7 @@ mod tests {
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn key_info(key_id: &str, status: KeyStatus, created_at: ZonedDateTime, rotated_at: Option<ZonedDateTime>) -> KeyInfo {
|
||||
fn key_info(key_id: &str, status: KeyStatus, created_at: Zoned, rotated_at: Option<Zoned>) -> KeyInfo {
|
||||
KeyInfo {
|
||||
key_id: key_id.to_string(),
|
||||
description: None,
|
||||
@@ -801,7 +801,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn census_ages_from_rotation_and_ignores_departing_keys() {
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
let day = Duration::from_secs(86400);
|
||||
let mut census = KeyCensus::default();
|
||||
|
||||
@@ -833,7 +833,7 @@ mod tests {
|
||||
/// material never wraps again, so its consumed nonce budget is moot.
|
||||
#[test]
|
||||
fn census_takes_the_max_wrap_reservation_of_usable_keys_only() {
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
let mut census = KeyCensus::default();
|
||||
|
||||
census.observe(&key_info("uncounted", KeyStatus::Active, now.clone(), None), &now);
|
||||
@@ -859,7 +859,7 @@ mod tests {
|
||||
fn wrap_budget_gauge_is_aggregate_and_carries_no_key_label() {
|
||||
let (snapshot, ()) = record_metrics(|| {
|
||||
Box::pin(async {
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
let mut census = KeyCensus::default();
|
||||
let mut wrapped = key_info("wrapped-key-id", KeyStatus::Active, now.clone(), None);
|
||||
wrapped.wrap_budget_reserved = Some(2_000_000);
|
||||
@@ -897,7 +897,7 @@ mod tests {
|
||||
// Three days in, still inside the seven-day window: the sweep
|
||||
// observes the scheduled key instead of removing it.
|
||||
let report = worker(backend.clone())
|
||||
.sweep(&(ZonedDateTime::now() + Duration::from_secs(3 * 86400)))
|
||||
.sweep(&(Zoned::now() + Duration::from_secs(3 * 86400)))
|
||||
.await;
|
||||
assert_eq!(report.skipped, 1);
|
||||
assert!(report.removed.is_empty());
|
||||
@@ -950,7 +950,7 @@ mod tests {
|
||||
|
||||
// Well inside the seven-day window: every key is observed as
|
||||
// pending rather than removed.
|
||||
let report = worker(backend.clone()).sweep(&ZonedDateTime::now()).await;
|
||||
let report = worker(backend.clone()).sweep(&Zoned::now()).await;
|
||||
assert_eq!(report.skipped, total, "every scheduled key must be inspected");
|
||||
assert!(report.removed.is_empty());
|
||||
})
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary};
|
||||
use async_trait::async_trait;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use rand::Rng;
|
||||
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -66,7 +66,7 @@ pub struct DataKeyEnvelope {
|
||||
pub nonce: Vec<u8>,
|
||||
pub encryption_context: HashMap<String, String>,
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
pub created_at: ZonedDateTime,
|
||||
pub created_at: Zoned,
|
||||
/// KEK version that wrapped `encrypted_key`; `None` on pre-versioning envelopes.
|
||||
///
|
||||
/// Optional and omitted when `None` so envelopes from non-rotating backends stay
|
||||
@@ -205,7 +205,7 @@ impl<'de> Deserialize<'de> for DataKeyEnvelope {
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] ZonedDateTime);
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] Zoned);
|
||||
|
||||
struct DataKeyEnvelopeVisitor;
|
||||
|
||||
@@ -591,7 +591,7 @@ mod tests {
|
||||
encrypted_key: vec![1, 2, 3, 4],
|
||||
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
encryption_context: HashMap::from([("bucket".to_string(), "b".to_string())]),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -625,7 +625,7 @@ mod tests {
|
||||
encrypted_key: vec![1, 2, 3, 4],
|
||||
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: Some(CONTEXT_BINDING_AAD_V1),
|
||||
};
|
||||
@@ -676,7 +676,7 @@ mod tests {
|
||||
map.insert("bucket".to_string(), "test-bucket".to_string());
|
||||
map
|
||||
},
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -694,7 +694,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_data_key_envelope_backward_compatibility() {
|
||||
// Test deserialization with current ZonedDateTime format (with timezone annotation)
|
||||
// Test deserialization with current Zoned format (with timezone annotation)
|
||||
let envelope_json = r#"{
|
||||
"key_id": "test-key-id",
|
||||
"master_key_id": "master-key-id",
|
||||
@@ -812,7 +812,7 @@ mod tests {
|
||||
encrypted_key: vec![1, 2, 3, 4],
|
||||
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: None,
|
||||
context_binding: None,
|
||||
};
|
||||
@@ -832,7 +832,7 @@ mod tests {
|
||||
encrypted_key: vec![1, 2, 3, 4],
|
||||
nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: Some(7),
|
||||
context_binding: None,
|
||||
};
|
||||
|
||||
@@ -28,7 +28,7 @@ use crate::types::{
|
||||
MAX_PENDING_DELETION_WINDOW_DAYS, MIN_PENDING_DELETION_WINDOW_DAYS, OperationContext, RewrapDataKeyRequest,
|
||||
RewrapDataKeyResponse, RotationDueReason,
|
||||
};
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -335,7 +335,7 @@ impl KmsManager {
|
||||
let mut result = self.backend.list_keys(request).await;
|
||||
if let Ok(response) = result.as_mut() {
|
||||
let rotates = self.backend.capabilities().rotate;
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
for key in &mut response.keys {
|
||||
self.apply_rotation_readiness(key, rotates, &now);
|
||||
}
|
||||
@@ -351,7 +351,7 @@ impl KmsManager {
|
||||
/// about what "overdue" means, and so a backend that cannot rotate is never
|
||||
/// the one deciding whether it should. Nothing consults the verdict before
|
||||
/// encrypting or decrypting: a key reported as due keeps serving traffic.
|
||||
fn apply_rotation_readiness(&self, key: &mut KeyInfo, backend_rotates: bool, now: &ZonedDateTime) {
|
||||
fn apply_rotation_readiness(&self, key: &mut KeyInfo, backend_rotates: bool, now: &Zoned) {
|
||||
if !backend_rotates {
|
||||
// Reported, not silently omitted: an operator chasing an overdue key
|
||||
// needs to know the answer is "this backend cannot rotate at all",
|
||||
@@ -699,7 +699,7 @@ mod tests {
|
||||
use crate::error::KmsError;
|
||||
use crate::types::{KeyMetadata, KeySpec, KeyState, KeyStatus, KeyUsage};
|
||||
use async_trait::async_trait;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
@@ -764,7 +764,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: ZonedDateTime::now(),
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "RUSTFS_KMS".to_string(),
|
||||
key_manager: "RUSTFS".to_string(),
|
||||
@@ -1810,7 +1810,7 @@ mod tests {
|
||||
manager
|
||||
}
|
||||
|
||||
fn aged_key(rotated_at: Option<ZonedDateTime>, created_at: ZonedDateTime) -> KeyInfo {
|
||||
fn aged_key(rotated_at: Option<Zoned>, created_at: Zoned) -> KeyInfo {
|
||||
KeyInfo {
|
||||
key_id: "key-a".to_string(),
|
||||
description: None,
|
||||
@@ -1833,7 +1833,7 @@ mod tests {
|
||||
/// a backend that cannot rotate is never told to rotate.
|
||||
#[test]
|
||||
fn rotation_readiness_reports_but_never_asks_the_impossible() {
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
let long_ago = &now - jiff::Span::new().days(400);
|
||||
let recently = &now - jiff::Span::new().hours(1);
|
||||
let day = Duration::from_secs(86_400);
|
||||
@@ -1887,13 +1887,13 @@ mod tests {
|
||||
/// independently of the age policy and reported under its own reason.
|
||||
#[test]
|
||||
fn rotation_readiness_reports_an_exhausted_wrap_budget() {
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
let recently = &now - jiff::Span::new().hours(1);
|
||||
let long_ago = &now - jiff::Span::new().days(400);
|
||||
let day = Duration::from_secs(86_400);
|
||||
let budget = 2_000_000;
|
||||
|
||||
let with_wraps = |manager: &KmsManager, wraps: Option<u64>, rotated_at: Option<ZonedDateTime>| {
|
||||
let with_wraps = |manager: &KmsManager, wraps: Option<u64>, rotated_at: Option<Zoned>| {
|
||||
let mut key = aged_key(rotated_at, recently.clone());
|
||||
key.wrap_budget_reserved = wraps;
|
||||
manager.apply_rotation_readiness(&mut key, true, &now);
|
||||
|
||||
@@ -595,7 +595,7 @@ mod tests {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: jiff::ZonedDateTime::now(),
|
||||
creation_date: jiff::Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "test".to_string(),
|
||||
key_manager: "test".to_string(),
|
||||
|
||||
@@ -23,7 +23,7 @@ use crate::encryption::context_aad;
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::types::*;
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rand::random;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
@@ -543,7 +543,7 @@ impl ObjectEncryptionService {
|
||||
iv,
|
||||
tag: Some(tag),
|
||||
encryption_context: context,
|
||||
encrypted_at: ZonedDateTime::now(),
|
||||
encrypted_at: Zoned::now(),
|
||||
// Pinned to the bytes actually fed to the AEAD, so the projection
|
||||
// below can store them verbatim instead of re-deriving them.
|
||||
context_aad: Some(aad),
|
||||
@@ -702,7 +702,7 @@ impl ObjectEncryptionService {
|
||||
iv,
|
||||
tag: Some(tag),
|
||||
encryption_context: context,
|
||||
encrypted_at: ZonedDateTime::now(),
|
||||
encrypted_at: Zoned::now(),
|
||||
// Pinned to the bytes actually fed to the AEAD, so the projection
|
||||
// below can store them verbatim instead of re-deriving them.
|
||||
context_aad: Some(aad),
|
||||
@@ -953,7 +953,7 @@ impl ObjectEncryptionService {
|
||||
iv,
|
||||
tag,
|
||||
encryption_context,
|
||||
encrypted_at: ZonedDateTime::now(),
|
||||
encrypted_at: Zoned::now(),
|
||||
original_size: 0, // Not available from headers
|
||||
encrypted_data_key,
|
||||
context_aad,
|
||||
@@ -1070,7 +1070,7 @@ mod tests {
|
||||
iv: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
|
||||
tag: Some(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
|
||||
encryption_context: HashMap::from([("bucket".to_string(), "test-bucket".to_string())]),
|
||||
encrypted_at: ZonedDateTime::now(),
|
||||
encrypted_at: Zoned::now(),
|
||||
original_size: 100,
|
||||
encrypted_data_key: vec![1, 2, 3, 4],
|
||||
// A hand-built record with no sealed bytes to defer to.
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
use jiff::{Timestamp, ZonedDateTime, tz::TimeZone};
|
||||
use jiff::{Timestamp, Zoned, tz::TimeZone};
|
||||
use serde::{Deserialize, Deserializer, Serializer};
|
||||
|
||||
pub(crate) mod zoned {
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn serialize<S>(value: &ZonedDateTime, serializer: S) -> Result<S::Ok, S::Error>
|
||||
pub(crate) fn serialize<S>(value: &Zoned, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
serializer.serialize_str(&value.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<ZonedDateTime, D::Error>
|
||||
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Zoned, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
@@ -23,7 +23,7 @@ pub(crate) mod zoned {
|
||||
pub(crate) mod option_zoned {
|
||||
use super::*;
|
||||
|
||||
pub(crate) fn serialize<S>(value: &Option<ZonedDateTime>, serializer: S) -> Result<S::Ok, S::Error>
|
||||
pub(crate) fn serialize<S>(value: &Option<Zoned>, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
@@ -33,7 +33,7 @@ pub(crate) mod option_zoned {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Option<ZonedDateTime>, D::Error>
|
||||
pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Option<Zoned>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
@@ -44,15 +44,15 @@ pub(crate) mod option_zoned {
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_zoned_compat(value: &str) -> Result<ZonedDateTime, String> {
|
||||
if let Ok(zoned) = value.parse::<ZonedDateTime>() {
|
||||
fn parse_zoned_compat(value: &str) -> Result<Zoned, String> {
|
||||
if let Ok(zoned) = value.parse::<Zoned>() {
|
||||
return Ok(zoned);
|
||||
}
|
||||
|
||||
let timestamp = value
|
||||
.parse::<Timestamp>()
|
||||
.map_err(|err| format!("failed to parse legacy timestamp '{value}': {err}"))?;
|
||||
Ok(timestamp.to_zoned_datetime(TimeZone::UTC))
|
||||
Ok(timestamp.to_zoned(TimeZone::UTC))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
+13
-13
@@ -14,7 +14,7 @@
|
||||
|
||||
//! Core type definitions for KMS operations
|
||||
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use uuid::Uuid;
|
||||
@@ -37,7 +37,7 @@ pub struct DataKeyInfo {
|
||||
/// Associated metadata
|
||||
pub metadata: HashMap<String, String>,
|
||||
/// Key creation timestamp
|
||||
pub created_at: ZonedDateTime,
|
||||
pub created_at: Zoned,
|
||||
}
|
||||
|
||||
impl DataKeyInfo {
|
||||
@@ -61,7 +61,7 @@ impl DataKeyInfo {
|
||||
ciphertext,
|
||||
key_spec,
|
||||
metadata: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,14 +112,14 @@ pub struct MasterKeyInfo {
|
||||
/// Associated metadata
|
||||
pub metadata: HashMap<String, String>,
|
||||
/// Key creation timestamp
|
||||
pub created_at: ZonedDateTime,
|
||||
pub created_at: Zoned,
|
||||
/// Key last rotation timestamp
|
||||
pub rotated_at: Option<ZonedDateTime>,
|
||||
pub rotated_at: Option<Zoned>,
|
||||
/// Key creator/owner
|
||||
pub created_by: Option<String>,
|
||||
/// Scheduled deletion deadline while the key is pending deletion
|
||||
#[serde(default)]
|
||||
pub deletion_date: Option<ZonedDateTime>,
|
||||
pub deletion_date: Option<Zoned>,
|
||||
}
|
||||
|
||||
impl MasterKeyInfo {
|
||||
@@ -142,7 +142,7 @@ impl MasterKeyInfo {
|
||||
status: KeyStatus::Active,
|
||||
description: None,
|
||||
metadata: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
rotated_at: None,
|
||||
created_by,
|
||||
deletion_date: None,
|
||||
@@ -174,7 +174,7 @@ impl MasterKeyInfo {
|
||||
status: KeyStatus::Active,
|
||||
description,
|
||||
metadata: HashMap::new(),
|
||||
created_at: ZonedDateTime::now(),
|
||||
created_at: Zoned::now(),
|
||||
rotated_at: None,
|
||||
created_by,
|
||||
deletion_date: None,
|
||||
@@ -247,9 +247,9 @@ pub struct KeyInfo {
|
||||
/// Key tags
|
||||
pub tags: HashMap<String, String>,
|
||||
/// Key creation timestamp
|
||||
pub created_at: ZonedDateTime,
|
||||
pub created_at: Zoned,
|
||||
/// Key last rotation timestamp
|
||||
pub rotated_at: Option<ZonedDateTime>,
|
||||
pub rotated_at: Option<Zoned>,
|
||||
/// Key creator
|
||||
pub created_by: Option<String>,
|
||||
/// Whether the key has outlived the configured rotation age.
|
||||
@@ -690,7 +690,7 @@ pub struct EncryptionMetadata {
|
||||
/// Encryption context
|
||||
pub encryption_context: HashMap<String, String>,
|
||||
/// Timestamp when encrypted
|
||||
pub encrypted_at: ZonedDateTime,
|
||||
pub encrypted_at: Zoned,
|
||||
/// Size of original data
|
||||
pub original_size: u64,
|
||||
/// Encrypted data key
|
||||
@@ -789,9 +789,9 @@ pub struct KeyMetadata {
|
||||
/// Key description
|
||||
pub description: Option<String>,
|
||||
/// Key creation timestamp
|
||||
pub creation_date: ZonedDateTime,
|
||||
pub creation_date: Zoned,
|
||||
/// Key deletion timestamp
|
||||
pub deletion_date: Option<ZonedDateTime>,
|
||||
pub deletion_date: Option<Zoned>,
|
||||
/// Key origin
|
||||
pub origin: String,
|
||||
/// Key manager
|
||||
|
||||
@@ -39,7 +39,7 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use common::{TestKms, assert_key_not_found, assert_unsupported_capability, ctx};
|
||||
use jiff::ZonedDateTime;
|
||||
use jiff::Zoned;
|
||||
use rustfs_kms::backends::local::LocalKmsBackend;
|
||||
use rustfs_kms::backends::static_kms::StaticKmsBackend;
|
||||
use rustfs_kms::backends::{ExpiredKeyRemoval, KmsBackend};
|
||||
@@ -79,8 +79,8 @@ async fn schedule(backend: &LocalKmsBackend, key_id: &str, days: u32) {
|
||||
.expect("deletion should be scheduled");
|
||||
}
|
||||
|
||||
fn days_from_now(days: u64) -> ZonedDateTime {
|
||||
ZonedDateTime::now() + Duration::from_secs(days * 86_400)
|
||||
fn days_from_now(days: u64) -> Zoned {
|
||||
Zoned::now() + Duration::from_secs(days * 86_400)
|
||||
}
|
||||
|
||||
async fn exists(backend: &LocalKmsBackend, key_id: &str) -> bool {
|
||||
@@ -422,7 +422,7 @@ async fn a_backend_without_deletion_support_reports_the_capability_gap() {
|
||||
"the static backend must not advertise deletion scheduling"
|
||||
);
|
||||
assert_unsupported_capability(
|
||||
backend.remove_expired_key(common::STATIC_KEY_ID, &ZonedDateTime::now()).await,
|
||||
backend.remove_expired_key(common::STATIC_KEY_ID, &Zoned::now()).await,
|
||||
"remove_expired_key",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ hotpath-alloc = ["hotpath"]
|
||||
hotpath-cpu = ["hotpath"]
|
||||
|
||||
[dependencies]
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
flate2 = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -20,8 +20,7 @@ use crate::ingest::{IngestReport, SkipReason};
|
||||
use crate::model::{EventKind, LogEvent, LogLevel, ParseStats};
|
||||
use crate::redact;
|
||||
use crate::rules::{Finding, FindingsCollector, RuleEngine, Severity};
|
||||
use crate::time::Rfc3339;
|
||||
use jiff::{SignedDuration, ZonedDateTime};
|
||||
use chrono::{DateTime, Duration, FixedOffset, Utc};
|
||||
use regex::Regex;
|
||||
use serde::Serialize;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
@@ -36,9 +35,9 @@ const BUCKET_WIDTHS_MIN: [i64; 6] = [1, 5, 15, 60, 360, 1440];
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AnalyzeOptions {
|
||||
/// Keep events at/after this time. Timestamp-less events always pass.
|
||||
pub since: Option<ZonedDateTime>,
|
||||
pub since: Option<DateTime<FixedOffset>>,
|
||||
/// Keep events at/before this time. Timestamp-less events always pass.
|
||||
pub until: Option<ZonedDateTime>,
|
||||
pub until: Option<DateTime<FixedOffset>>,
|
||||
/// Drop events below this level; level-less events always pass.
|
||||
pub min_level: Option<LogLevel>,
|
||||
pub max_samples: usize,
|
||||
@@ -65,7 +64,7 @@ pub struct Summary {
|
||||
pub events_after_filter: u64,
|
||||
/// "ERROR" -> n, …, "no_level" -> n.
|
||||
pub level_counts: BTreeMap<String, u64>,
|
||||
pub time_range: Option<(ZonedDateTime, ZonedDateTime)>,
|
||||
pub time_range: Option<(DateTime<FixedOffset>, DateTime<FixedOffset>)>,
|
||||
pub nodes: Vec<String>,
|
||||
pub parse: ParseStats,
|
||||
pub files_parsed: u64,
|
||||
@@ -89,7 +88,7 @@ pub struct UnmatchedCluster {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct TimeBucket {
|
||||
/// Bucket start, normalized to UTC.
|
||||
pub start: ZonedDateTime,
|
||||
pub start: DateTime<Utc>,
|
||||
pub error: u64,
|
||||
pub warn: u64,
|
||||
pub other: u64,
|
||||
@@ -106,8 +105,8 @@ pub struct TimelineAnomaly {
|
||||
/// Involved node labels (node_ranges_disjoint); empty otherwise.
|
||||
pub nodes: Vec<String>,
|
||||
/// Gap boundaries in UTC (log_gap); null otherwise.
|
||||
pub gap_start: Option<ZonedDateTime>,
|
||||
pub gap_end: Option<ZonedDateTime>,
|
||||
pub gap_start: Option<DateTime<Utc>>,
|
||||
pub gap_end: Option<DateTime<Utc>>,
|
||||
/// log_gap only: a startup-class finding begins within 5min after the gap.
|
||||
pub restart_evidence: bool,
|
||||
}
|
||||
@@ -130,7 +129,7 @@ pub struct AnalysisReport {
|
||||
}
|
||||
|
||||
/// (min ts, max ts, timestamped event count) for one node label.
|
||||
type NodeRange = (ZonedDateTime, ZonedDateTime, u64);
|
||||
type NodeRange = (DateTime<FixedOffset>, DateTime<FixedOffset>, u64);
|
||||
|
||||
#[derive(Default)]
|
||||
struct BucketCounts {
|
||||
@@ -155,7 +154,7 @@ pub struct Analyzer {
|
||||
events_total: u64,
|
||||
events_after_filter: u64,
|
||||
level_counts: BTreeMap<String, u64>,
|
||||
time_range: Option<(ZonedDateTime, ZonedDateTime)>,
|
||||
time_range: Option<(DateTime<FixedOffset>, DateTime<FixedOffset>)>,
|
||||
nodes: BTreeSet<String>,
|
||||
offsets: BTreeSet<String>,
|
||||
/// node label -> (min ts, max ts, timestamped event count); feeds the
|
||||
@@ -190,9 +189,8 @@ impl Analyzer {
|
||||
pub fn observe(&mut self, mut event: LogEvent) {
|
||||
self.events_total += 1;
|
||||
|
||||
if let Some(ts) = event.timestamp.as_ref()
|
||||
&& (self.opts.since.as_ref().is_some_and(|since| ts < since)
|
||||
|| self.opts.until.as_ref().is_some_and(|until| ts > until))
|
||||
if let Some(ts) = event.timestamp
|
||||
&& (self.opts.since.is_some_and(|since| ts < since) || self.opts.until.is_some_and(|until| ts > until))
|
||||
{
|
||||
return;
|
||||
}
|
||||
@@ -219,26 +217,23 @@ impl Analyzer {
|
||||
self.nodes.insert(node.to_string());
|
||||
}
|
||||
|
||||
if let Some(ts) = event.timestamp.clone() {
|
||||
self.time_range = Some(match self.time_range.take() {
|
||||
None => (ts.clone(), ts.clone()),
|
||||
Some((first, last)) => (first.min(ts.clone()), last.max(ts.clone())),
|
||||
if let Some(ts) = event.timestamp {
|
||||
self.time_range = Some(match self.time_range {
|
||||
None => (ts, ts),
|
||||
Some((first, last)) => (first.min(ts), last.max(ts)),
|
||||
});
|
||||
self.offsets.insert(format_offset(&ts));
|
||||
if let Some(node) = event.node.as_deref() {
|
||||
self.node_ranges
|
||||
.entry(node.to_string())
|
||||
.and_modify(|(min, max, count)| {
|
||||
*min = min.clone().min(ts.clone());
|
||||
*max = max.clone().max(ts.clone());
|
||||
*min = (*min).min(ts);
|
||||
*max = (*max).max(ts);
|
||||
*count += 1;
|
||||
})
|
||||
.or_insert((ts.clone(), ts.clone(), 1));
|
||||
.or_insert((ts, ts, 1));
|
||||
}
|
||||
let bucket = self
|
||||
.minute_buckets
|
||||
.entry(ts.timestamp().as_second().div_euclid(60))
|
||||
.or_default();
|
||||
let bucket = self.minute_buckets.entry(ts.timestamp().div_euclid(60)).or_default();
|
||||
match event.level {
|
||||
Some(LogLevel::Error) => bucket.error += 1,
|
||||
Some(LogLevel::Warn) => bucket.warn += 1,
|
||||
@@ -361,8 +356,8 @@ impl Analyzer {
|
||||
}
|
||||
}
|
||||
|
||||
fn format_offset(ts: &ZonedDateTime) -> String {
|
||||
let seconds = crate::time::offset_seconds(ts);
|
||||
fn format_offset(ts: &DateTime<FixedOffset>) -> String {
|
||||
let seconds = ts.offset().local_minus_utc();
|
||||
if seconds == 0 {
|
||||
return "Z".to_string();
|
||||
}
|
||||
@@ -395,7 +390,7 @@ fn merge_timeline(minute_buckets: &BTreeMap<i64, BucketCounts>) -> (Vec<TimeBuck
|
||||
let buckets = merged
|
||||
.into_iter()
|
||||
.map(|(slot, counts)| TimeBucket {
|
||||
start: crate::time::utc_from_unix(slot * width * 60),
|
||||
start: DateTime::<Utc>::from_timestamp(slot * width * 60, 0).expect("valid timestamp"),
|
||||
error: counts.error,
|
||||
warn: counts.warn,
|
||||
other: counts.other,
|
||||
@@ -476,13 +471,13 @@ fn detect_timeline_anomalies(
|
||||
// span of at least max(15min, 3x display bucket width), then activity
|
||||
// again — the shape of a process restart or hang.
|
||||
let threshold = (3 * bucket_width_min).max(15);
|
||||
let startup_after_gap = |gap_end: &ZonedDateTime| {
|
||||
let startup_after_gap = |gap_end: DateTime<Utc>| {
|
||||
findings.iter().chain(low_confidence).find_map(|f| {
|
||||
if !STARTUP_RULE_IDS.contains(&f.rule_id.as_str()) {
|
||||
return None;
|
||||
}
|
||||
let first = f.first_seen.as_ref()?;
|
||||
(first >= gap_end && first <= gap_end + SignedDuration::from_mins(5)).then(|| f.rule_id.clone())
|
||||
let first = f.first_seen?.with_timezone(&Utc);
|
||||
(first >= gap_end && first <= gap_end + Duration::minutes(5)).then(|| f.rule_id.clone())
|
||||
})
|
||||
};
|
||||
let minutes: Vec<i64> = minute_buckets.keys().copied().collect();
|
||||
@@ -495,9 +490,9 @@ fn detect_timeline_anomalies(
|
||||
continue;
|
||||
}
|
||||
if run_len >= 3 && delta > threshold {
|
||||
let gap_start = crate::time::utc_from_unix((prev + 1) * 60);
|
||||
let gap_end = crate::time::utc_from_unix(next * 60);
|
||||
let restart = startup_after_gap(&gap_end);
|
||||
let gap_start = DateTime::<Utc>::from_timestamp((prev + 1) * 60, 0).expect("valid timestamp");
|
||||
let gap_end = DateTime::<Utc>::from_timestamp(next * 60, 0).expect("valid timestamp");
|
||||
let restart = startup_after_gap(gap_end);
|
||||
let span = human_duration_minutes(delta - 1);
|
||||
let message = match &restart {
|
||||
Some(rule_id) => format!(
|
||||
@@ -550,7 +545,7 @@ fn collapse_findings(findings: &mut [Finding]) {
|
||||
let index_of: HashMap<String, usize> = findings.iter().enumerate().map(|(i, f)| (f.rule_id.clone(), i)).collect();
|
||||
// Among several qualifying roots pick the earliest first_seen;
|
||||
// timestamp-less roots come last, rule id breaks remaining ties.
|
||||
let root_key = |f: &Finding| (f.first_seen.is_none(), f.first_seen.clone(), f.rule_id.clone());
|
||||
let root_key = |f: &Finding| (f.first_seen.is_none(), f.first_seen, f.rule_id.clone());
|
||||
|
||||
let mut chosen: Vec<Option<usize>> = vec![None; findings.len()];
|
||||
for (i, symptom) in findings.iter().enumerate() {
|
||||
@@ -560,10 +555,10 @@ fn collapse_findings(findings: &mut [Finding]) {
|
||||
continue;
|
||||
}
|
||||
let root = &findings[r];
|
||||
let qualifies = match (root.first_seen.as_ref(), symptom.first_seen.as_ref()) {
|
||||
let qualifies = match (root.first_seen, symptom.first_seen) {
|
||||
(Some(root_first), Some(symptom_first)) => {
|
||||
root_first <= symptom_first + SignedDuration::from_mins(5)
|
||||
&& root.last_seen.as_ref().unwrap_or(root_first) >= symptom_first - SignedDuration::from_mins(30)
|
||||
root_first <= symptom_first + Duration::minutes(5)
|
||||
&& root.last_seen.unwrap_or(root_first) >= symptom_first - Duration::minutes(30)
|
||||
}
|
||||
_ => true,
|
||||
};
|
||||
@@ -704,7 +699,7 @@ mod tests {
|
||||
|
||||
fn event(message: &str, level: Option<LogLevel>, ts: Option<&str>) -> LogEvent {
|
||||
LogEvent {
|
||||
timestamp: ts.map(|t| crate::time::parse_rfc3339(t).expect("ts")),
|
||||
timestamp: ts.map(|t| DateTime::parse_from_rfc3339(t).expect("ts")),
|
||||
level,
|
||||
target: Some("rustfs::server::http".to_string()),
|
||||
message: message.to_string(),
|
||||
@@ -820,8 +815,8 @@ mod tests {
|
||||
assert_eq!(gaps.len(), 1, "{:?}", report.timeline_anomalies);
|
||||
assert!(gaps[0].restart_evidence, "startup-fatal right after the gap: {:?}", gaps[0]);
|
||||
assert!(gaps[0].message.contains("重启证据"));
|
||||
assert_eq!(gaps[0].gap_start.as_ref().expect("start").to_rfc3339(), "2026-07-15T03:04:00+00:00");
|
||||
assert_eq!(gaps[0].gap_end.as_ref().expect("end").to_rfc3339(), "2026-07-15T03:44:00+00:00");
|
||||
assert_eq!(gaps[0].gap_start.expect("start").to_rfc3339(), "2026-07-15T03:04:00+00:00");
|
||||
assert_eq!(gaps[0].gap_end.expect("end").to_rfc3339(), "2026-07-15T03:44:00+00:00");
|
||||
|
||||
// Same hole but no startup finding afterwards: still a gap, no upgrade.
|
||||
let mut a = analyzer(AnalyzeOptions::default());
|
||||
@@ -1016,7 +1011,7 @@ mod tests {
|
||||
#[test]
|
||||
fn since_until_filters_timestamped_events_only() {
|
||||
let mut a = analyzer(AnalyzeOptions {
|
||||
since: Some(crate::time::parse_rfc3339("2026-07-15T03:00:00+08:00").expect("ts")),
|
||||
since: Some(DateTime::parse_from_rfc3339("2026-07-15T03:00:00+08:00").expect("ts")),
|
||||
..Default::default()
|
||||
});
|
||||
a.observe(event("Disk full", Some(LogLevel::Error), Some("2026-07-15T02:00:00+08:00")));
|
||||
|
||||
@@ -27,7 +27,6 @@ pub mod parse;
|
||||
mod redact;
|
||||
pub mod report;
|
||||
pub mod rules;
|
||||
pub(crate) mod time;
|
||||
|
||||
pub use analyze::{AnalysisReport, AnalyzeOptions, Analyzer, Summary, TimeBucket, TimelineAnomaly, UnmatchedCluster};
|
||||
pub use ingest::{IngestOptions, IngestReport, SkipReason, ingest_path, ingest_reader};
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
//! Unified event model shared by every stage of the analyzer.
|
||||
|
||||
use jiff::ZonedDateTime;
|
||||
use chrono::{DateTime, FixedOffset};
|
||||
use serde::{Deserialize, Serialize, Serializer};
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
@@ -102,7 +102,7 @@ pub enum EventKind {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct LogEvent {
|
||||
/// RFC-3339 timestamp with offset. None for panic/text lines that carry none.
|
||||
pub timestamp: Option<ZonedDateTime>,
|
||||
pub timestamp: Option<DateTime<FixedOffset>>,
|
||||
/// None when the line carries no recognizable level (text lines).
|
||||
pub level: Option<LogLevel>,
|
||||
/// tracing `target`, e.g. "rustfs::scanner::io". None for panic/text.
|
||||
|
||||
@@ -36,7 +36,7 @@ pub(crate) fn try_parse(line: &str, file: &Arc<str>, line_no: u64, node: &Option
|
||||
}
|
||||
|
||||
let timestamp = match map.remove("timestamp") {
|
||||
Some(Value::String(raw)) => match crate::time::parse_rfc3339(&raw) {
|
||||
Some(Value::String(raw)) => match chrono::DateTime::parse_from_rfc3339(&raw) {
|
||||
Ok(ts) => Some(ts),
|
||||
Err(_) => {
|
||||
map.insert("timestamp_raw".to_string(), Value::String(raw));
|
||||
|
||||
@@ -156,8 +156,8 @@ mod tests {
|
||||
assert_eq!(ev.level, Some(LogLevel::Error));
|
||||
assert_eq!(ev.target.as_deref(), Some("rustfs::scanner::io"));
|
||||
assert_eq!(ev.message, "Disk health check marked disk faulty");
|
||||
let ts = ev.timestamp.as_ref().expect("timestamp");
|
||||
assert_eq!(crate::time::offset_seconds(ts), 8 * 3600);
|
||||
let ts = ev.timestamp.expect("timestamp");
|
||||
assert_eq!(ts.offset().local_minus_utc(), 8 * 3600);
|
||||
assert_eq!(ev.field_str("reason"), Some("faulty_disk"));
|
||||
assert_eq!(ev.field_str("disk"), Some("/data/disk3"));
|
||||
assert!(ev.fields.contains_key("filename"));
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
|
||||
use super::{human_bytes, parse_percentage};
|
||||
use crate::analyze::AnalysisReport;
|
||||
use crate::time::Rfc3339;
|
||||
use std::io;
|
||||
|
||||
/// GFM parses a table row into cells before inline code, so a literal `|`
|
||||
@@ -34,7 +33,7 @@ pub(super) fn render(report: &AnalysisReport, w: &mut dyn io::Write) -> io::Resu
|
||||
writeln!(w)?;
|
||||
writeln!(w, "| 项 | 值 |")?;
|
||||
writeln!(w, "|---|---|")?;
|
||||
match report.summary.time_range.as_ref() {
|
||||
match report.summary.time_range {
|
||||
Some((first, last)) => writeln!(w, "| 时间范围 | {} → {} |", first.to_rfc3339(), last.to_rfc3339())?,
|
||||
None => writeln!(w, "| 时间范围 | (无带时间戳的事件) |")?,
|
||||
}
|
||||
@@ -105,7 +104,7 @@ pub(super) fn render(report: &AnalysisReport, w: &mut dyn io::Write) -> io::Resu
|
||||
finding.count
|
||||
)?;
|
||||
writeln!(w)?;
|
||||
if let (Some(first), Some(last)) = (finding.first_seen.as_ref(), finding.last_seen.as_ref()) {
|
||||
if let (Some(first), Some(last)) = (finding.first_seen, finding.last_seen) {
|
||||
writeln!(w, "- 首次: {} · 末次: {}", first.to_rfc3339(), last.to_rfc3339())?;
|
||||
}
|
||||
if !finding.caused.is_empty() {
|
||||
|
||||
@@ -74,13 +74,13 @@ mod tests {
|
||||
use crate::ingest::IngestReport;
|
||||
use crate::model::{EventKind, LogEvent, LogLevel, SourceRef};
|
||||
use crate::rules::{RuleEngine, seed_rule_set};
|
||||
use crate::time::parse_rfc3339;
|
||||
use chrono::DateTime;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn fixed_report() -> AnalysisReport {
|
||||
let mut analyzer = Analyzer::new(RuleEngine::new(seed_rule_set()), AnalyzeOptions::default());
|
||||
let mut ev = LogEvent {
|
||||
timestamp: Some(parse_rfc3339("2026-07-15T03:12:44+08:00").expect("ts")),
|
||||
timestamp: Some(DateTime::parse_from_rfc3339("2026-07-15T03:12:44+08:00").expect("ts")),
|
||||
level: Some(LogLevel::Error),
|
||||
target: Some("rustfs_ecstore".to_string() + "::erasure"),
|
||||
message: "erasure write quorum (required=8, achieved=5, failed=3)".to_string(),
|
||||
@@ -96,11 +96,11 @@ mod tests {
|
||||
.insert("required".to_string(), serde_json::Value::String("8".into()));
|
||||
analyzer.observe(ev.clone());
|
||||
analyzer.observe(LogEvent {
|
||||
timestamp: Some(parse_rfc3339("2026-07-15T03:20:44+08:00").expect("ts")),
|
||||
timestamp: Some(DateTime::parse_from_rfc3339("2026-07-15T03:20:44+08:00").expect("ts")),
|
||||
..ev
|
||||
});
|
||||
analyzer.observe(LogEvent {
|
||||
timestamp: Some(parse_rfc3339("2026-07-15T03:25:00+08:00").expect("ts")),
|
||||
timestamp: Some(DateTime::parse_from_rfc3339("2026-07-15T03:25:00+08:00").expect("ts")),
|
||||
level: Some(LogLevel::Error),
|
||||
target: None,
|
||||
message: "failed to sync /data/x9 after 5 retries".to_string(),
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
use super::{human_bytes, human_duration, parse_percentage};
|
||||
use crate::analyze::AnalysisReport;
|
||||
use crate::rules::Finding;
|
||||
use crate::time::Rfc3339;
|
||||
use std::io;
|
||||
|
||||
const HEAVY_RULE: &str = "════════════════════════════════════════";
|
||||
@@ -27,9 +26,9 @@ pub(super) fn render(report: &AnalysisReport, w: &mut dyn io::Write) -> io::Resu
|
||||
writeln!(w, "RustFS 日志诊断报告")?;
|
||||
writeln!(w, "{HEAVY_RULE}")?;
|
||||
|
||||
match report.summary.time_range.as_ref() {
|
||||
match report.summary.time_range {
|
||||
Some((first, last)) => {
|
||||
let span = human_duration(last.duration_since(first).as_secs());
|
||||
let span = human_duration(last.signed_duration_since(first).num_seconds());
|
||||
writeln!(w, "时间范围 {} → {} ({span})", first.to_rfc3339(), last.to_rfc3339())?;
|
||||
}
|
||||
None => writeln!(w, "时间范围 (无带时间戳的事件)")?,
|
||||
@@ -158,7 +157,7 @@ fn render_finding(finding: &Finding, counts: &std::collections::HashMap<&str, u6
|
||||
finding.count,
|
||||
nodes
|
||||
)?;
|
||||
if let (Some(first), Some(last)) = (finding.first_seen.as_ref(), finding.last_seen.as_ref()) {
|
||||
if let (Some(first), Some(last)) = (finding.first_seen, finding.last_seen) {
|
||||
writeln!(w, " 首次 {} 末次 {}", first.to_rfc3339(), last.to_rfc3339())?;
|
||||
}
|
||||
if !finding.caused.is_empty() {
|
||||
@@ -181,10 +180,7 @@ fn render_finding(finding: &Finding, counts: &std::collections::HashMap<&str, u6
|
||||
writeln!(w, " 证据: {field} = {}{overflow}", shown.join(", "))?;
|
||||
}
|
||||
for sample in &finding.samples {
|
||||
let ts = sample
|
||||
.timestamp
|
||||
.as_ref()
|
||||
.map_or_else(|| "(无时间戳)".to_string(), |t| t.to_rfc3339());
|
||||
let ts = sample.timestamp.map_or_else(|| "(无时间戳)".to_string(), |t| t.to_rfc3339());
|
||||
let node = sample.node.as_deref().unwrap_or("-");
|
||||
writeln!(w, " 样本: {ts} {node} {}:{}", sample.source.file, sample.source.line)?;
|
||||
writeln!(w, " {}", sample.message)?;
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
use super::model::{Rule, Severity};
|
||||
use crate::model::LogEvent;
|
||||
use jiff::ZonedDateTime;
|
||||
use chrono::{DateTime, FixedOffset};
|
||||
use serde::Serialize;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
@@ -44,8 +44,8 @@ pub struct Finding {
|
||||
pub suggestion: String,
|
||||
pub count: u64,
|
||||
/// Earliest / latest timestamps among hits that carry one.
|
||||
pub first_seen: Option<ZonedDateTime>,
|
||||
pub last_seen: Option<ZonedDateTime>,
|
||||
pub first_seen: Option<DateTime<FixedOffset>>,
|
||||
pub last_seen: Option<DateTime<FixedOffset>>,
|
||||
/// Node labels of hits; label-less events are recorded as "-".
|
||||
pub nodes: BTreeSet<String>,
|
||||
/// The earliest `max_samples` hit events (timestamp-less ones sort last).
|
||||
@@ -79,28 +79,28 @@ pub struct FindingsCollector {
|
||||
/// Sort key for sample selection: earliest first, timestamp-less last,
|
||||
/// (file, line) as the deterministic tiebreaker — line numbers alone are only
|
||||
/// unique within one file, so ties across inputs would be order-dependent.
|
||||
fn sample_key(event: &LogEvent) -> (bool, Option<ZonedDateTime>, &str, u64) {
|
||||
(event.timestamp.is_none(), event.timestamp.clone(), &event.source.file, event.source.line)
|
||||
fn sample_key(event: &LogEvent) -> (bool, Option<DateTime<FixedOffset>>, &str, u64) {
|
||||
(event.timestamp.is_none(), event.timestamp, &event.source.file, event.source.line)
|
||||
}
|
||||
|
||||
/// `ZonedDateTime` orders by instant, so two equal instants carrying
|
||||
/// `DateTime<FixedOffset>` orders by instant, so two equal instants carrying
|
||||
/// different offsets compare equal and `min`/`max` keep whichever arrived
|
||||
/// first — an input-order dependency that then leaks into the serialized
|
||||
/// RFC3339 offset. Break the tie on the offset itself to stay deterministic.
|
||||
fn earliest(a: ZonedDateTime, b: ZonedDateTime) -> ZonedDateTime {
|
||||
fn earliest(a: DateTime<FixedOffset>, b: DateTime<FixedOffset>) -> DateTime<FixedOffset> {
|
||||
match a.cmp(&b) {
|
||||
std::cmp::Ordering::Less => a,
|
||||
std::cmp::Ordering::Greater => b,
|
||||
std::cmp::Ordering::Equal if crate::time::offset_seconds(&a) <= crate::time::offset_seconds(&b) => a,
|
||||
std::cmp::Ordering::Equal if a.offset().local_minus_utc() <= b.offset().local_minus_utc() => a,
|
||||
std::cmp::Ordering::Equal => b,
|
||||
}
|
||||
}
|
||||
|
||||
fn latest(a: ZonedDateTime, b: ZonedDateTime) -> ZonedDateTime {
|
||||
fn latest(a: DateTime<FixedOffset>, b: DateTime<FixedOffset>) -> DateTime<FixedOffset> {
|
||||
match a.cmp(&b) {
|
||||
std::cmp::Ordering::Greater => a,
|
||||
std::cmp::Ordering::Less => b,
|
||||
std::cmp::Ordering::Equal if crate::time::offset_seconds(&a) >= crate::time::offset_seconds(&b) => a,
|
||||
std::cmp::Ordering::Equal if a.offset().local_minus_utc() >= b.offset().local_minus_utc() => a,
|
||||
std::cmp::Ordering::Equal => b,
|
||||
}
|
||||
}
|
||||
@@ -140,9 +140,9 @@ impl FindingsCollector {
|
||||
let finding = &mut acc.finding;
|
||||
finding.count += 1;
|
||||
|
||||
if let Some(ts) = event.timestamp.clone() {
|
||||
finding.first_seen = Some(finding.first_seen.take().map_or(ts.clone(), |cur| earliest(cur, ts.clone())));
|
||||
finding.last_seen = Some(finding.last_seen.take().map_or(ts.clone(), |cur| latest(cur, ts)));
|
||||
if let Some(ts) = event.timestamp {
|
||||
finding.first_seen = Some(finding.first_seen.map_or(ts, |cur| earliest(cur, ts)));
|
||||
finding.last_seen = Some(finding.last_seen.map_or(ts, |cur| latest(cur, ts)));
|
||||
}
|
||||
|
||||
finding.nodes.insert(event.node.as_deref().unwrap_or("-").to_string());
|
||||
|
||||
@@ -40,7 +40,6 @@ pub fn seed_rule_set() -> RuleSet {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::model::{EventKind, LogEvent, LogLevel, SourceRef};
|
||||
use crate::time::Rfc3339;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn event(message: &str, level: Option<LogLevel>, line: u64) -> LogEvent {
|
||||
@@ -61,7 +60,7 @@ mod tests {
|
||||
|
||||
fn event_at(message: &str, ts: &str, line: u64) -> LogEvent {
|
||||
LogEvent {
|
||||
timestamp: Some(crate::time::parse_rfc3339(ts).expect("ts")),
|
||||
timestamp: Some(chrono::DateTime::parse_from_rfc3339(ts).expect("ts")),
|
||||
..event(message, Some(LogLevel::Error), line)
|
||||
}
|
||||
}
|
||||
@@ -225,8 +224,8 @@ mod tests {
|
||||
};
|
||||
let f = &findings[0];
|
||||
assert_eq!(f.count, 5);
|
||||
assert_eq!(f.first_seen.as_ref().expect("first").to_rfc3339(), "2026-07-15T01:00:00+08:00");
|
||||
assert_eq!(f.last_seen.as_ref().expect("last").to_rfc3339(), "2026-07-15T04:00:00+08:00");
|
||||
assert_eq!(f.first_seen.expect("first").to_rfc3339(), "2026-07-15T01:00:00+08:00");
|
||||
assert_eq!(f.last_seen.expect("last").to_rfc3339(), "2026-07-15T04:00:00+08:00");
|
||||
let sample_messages: Vec<_> = f.samples.iter().map(|s| s.message.as_str()).collect();
|
||||
assert_eq!(sample_messages, vec!["quorum e2", "quorum e3", "quorum e1"]);
|
||||
}
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
use jiff::{Timestamp, ZonedDateTime, tz::TimeZone};
|
||||
|
||||
pub(crate) trait Rfc3339 {
|
||||
fn to_rfc3339(&self) -> String;
|
||||
fn format(&self, pattern: &str) -> String;
|
||||
}
|
||||
|
||||
impl Rfc3339 for ZonedDateTime {
|
||||
fn to_rfc3339(&self) -> String {
|
||||
format_rfc3339(self)
|
||||
}
|
||||
|
||||
fn format(&self, pattern: &str) -> String {
|
||||
self.strftime(pattern).to_string()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_rfc3339(value: &str) -> Result<ZonedDateTime, jiff::Error> {
|
||||
if let Some(prefix) = value.strip_suffix('Z') {
|
||||
let normalized = format!("{prefix}+00:00");
|
||||
return ZonedDateTime::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", normalized);
|
||||
}
|
||||
ZonedDateTime::strptime("%Y-%m-%dT%H:%M:%S%.f%:z", value)
|
||||
}
|
||||
|
||||
pub(crate) fn format_rfc3339(value: &ZonedDateTime) -> String {
|
||||
value.strftime("%Y-%m-%dT%H:%M:%S%.f%:z").to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn utc_from_unix(seconds: i64) -> ZonedDateTime {
|
||||
Timestamp::new(seconds, 0)
|
||||
.expect("unix timestamp must fit jiff")
|
||||
.to_zoned_datetime(TimeZone::UTC)
|
||||
}
|
||||
|
||||
pub(crate) fn offset_seconds(value: &ZonedDateTime) -> i32 {
|
||||
value.offset().seconds()
|
||||
}
|
||||
@@ -30,13 +30,12 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Kept as no-op compatibility features for dependents that forward the
|
||||
# workspace profiling feature set to every crate.
|
||||
hotpath = []
|
||||
hotpath-alloc = []
|
||||
hotpath-cpu = []
|
||||
hotpath = ["hotpath/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
humantime.workspace = true
|
||||
http.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
|
||||
@@ -359,7 +359,7 @@ where
|
||||
}
|
||||
|
||||
fn format_timestamp_rfc3339_millis(value: &Timestamp) -> String {
|
||||
let utc = value.to_zoned_datetime(TimeZone::UTC);
|
||||
let utc = value.to_zoned(TimeZone::UTC);
|
||||
format!(
|
||||
"{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:03}Z",
|
||||
utc.year(),
|
||||
|
||||
@@ -127,6 +127,7 @@ rustfs-security-governance = { workspace = true }
|
||||
rustfs-storage-api = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["ip"] }
|
||||
rustix = { workspace = true, features = ["fs", "stdio"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
flate2 = { workspace = true }
|
||||
glob = { workspace = true }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
|
||||
@@ -23,7 +23,8 @@ use crate::global::{
|
||||
METRIC_LOG_CLEANER_ACTIVE_FILE_SIZE_BYTES, METRIC_LOG_CLEANER_ROTATION_DURATION_SECONDS,
|
||||
METRIC_LOG_CLEANER_ROTATION_FAILURES_TOTAL, METRIC_LOG_CLEANER_ROTATION_TOTAL,
|
||||
};
|
||||
use jiff::{Timestamp, ZonedDateTime, tz::TimeZone};
|
||||
use chrono::{Local, TimeZone as _};
|
||||
use jiff::Zoned;
|
||||
use metrics::{counter, gauge, histogram};
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, Write};
|
||||
@@ -57,12 +58,8 @@ impl Rotation {
|
||||
Rotation::Minutely => now / 60 != last / 60,
|
||||
Rotation::Hourly => now / 3600 != last / 3600,
|
||||
Rotation::Daily => {
|
||||
let last_day = Timestamp::new(last, 0)
|
||||
.ok()
|
||||
.map(|ts| ts.to_zoned_datetime(TimeZone::system()).date());
|
||||
let now_day = Timestamp::new(now, 0)
|
||||
.ok()
|
||||
.map(|ts| ts.to_zoned_datetime(TimeZone::system()).date());
|
||||
let last_day = Local.timestamp_opt(last, 0).single().map(|ts| ts.date_naive());
|
||||
let now_day = Local.timestamp_opt(now, 0).single().map(|ts| ts.date_naive());
|
||||
|
||||
match (last_day, now_day) {
|
||||
(Some(last_day), Some(now_day)) => last_day != now_day,
|
||||
@@ -134,7 +131,7 @@ impl RollingAppender {
|
||||
match_mode,
|
||||
file: None,
|
||||
size: 0,
|
||||
last_roll_ts: ZonedDateTime::now().timestamp().as_second(),
|
||||
last_roll_ts: Zoned::now().timestamp().as_second(),
|
||||
rotation_retry_cooldown_until: None,
|
||||
roll_signal: None,
|
||||
};
|
||||
@@ -228,7 +225,7 @@ impl RollingAppender {
|
||||
|
||||
// 2. Time-based check
|
||||
// We check this after size check to avoid unnecessary time calls if size forces a roll.
|
||||
let now = ZonedDateTime::now().timestamp().as_second();
|
||||
let now = Zoned::now().timestamp().as_second();
|
||||
|
||||
self.rotation.check_should_roll(self.last_roll_ts, now)
|
||||
}
|
||||
@@ -252,7 +249,7 @@ impl RollingAppender {
|
||||
// 2. Generate archive name.
|
||||
// Format: YYYYMMDDHHMMSS.uuuuuu (Microsecond/Nanosecond precision)
|
||||
// We use jiff's strftime. "%Y%m%d%H%M%S%.6f" gives microsecond precision.
|
||||
let now = ZonedDateTime::now();
|
||||
let now = Zoned::now();
|
||||
let timestamp_str = now.strftime("%Y%m%d%H%M%S%.6f").to_string();
|
||||
|
||||
// Add a unique counter to prevent collisions in high-concurrency/fast-rotation scenarios.
|
||||
@@ -670,12 +667,11 @@ mod tests {
|
||||
/// The fixtures below use mid-January dates, which sit on no DST or
|
||||
/// offset transition in any zone, so every wall-clock time exists exactly once.
|
||||
fn local_ts(year: i32, month: u32, day: u32, hour: u32, min: u32, sec: u32) -> i64 {
|
||||
jiff::civil::DateTime::new(year as i16, month as i8, day as i8, hour as i8, min as i8, sec as i8, 0)
|
||||
.expect("fixture local time must be valid")
|
||||
.to_zoned_datetime(TimeZone::system())
|
||||
Local
|
||||
.with_ymd_and_hms(year, month, day, hour, min, sec)
|
||||
.single()
|
||||
.expect("fixture local time must exist exactly once")
|
||||
.timestamp()
|
||||
.as_second()
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -31,18 +31,21 @@ workspace = true
|
||||
[features]
|
||||
default = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"rustfs-config/hotpath",
|
||||
"rustfs-credentials/hotpath",
|
||||
"rustfs-crypto/hotpath",
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-config/hotpath-alloc",
|
||||
"rustfs-credentials/hotpath-alloc",
|
||||
"rustfs-crypto/hotpath-alloc",
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-config/hotpath-cpu",
|
||||
"rustfs-credentials/hotpath-cpu",
|
||||
"rustfs-crypto/hotpath-cpu",
|
||||
@@ -68,6 +71,7 @@ tracing.workspace = true
|
||||
moka = { workspace = true, features = ["future"] }
|
||||
async-trait.workspace = true
|
||||
futures.workspace = true
|
||||
hotpath.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
pollster.workspace = true
|
||||
|
||||
@@ -473,10 +473,6 @@ pub enum AdminAction {
|
||||
TLSCertificateStatusAdminAction,
|
||||
#[strum(serialize = "admin:CreateUser")]
|
||||
CreateUserAdminAction,
|
||||
#[strum(serialize = "admin:ChangeOwnPassword")]
|
||||
ChangeOwnPasswordAdminAction,
|
||||
#[strum(serialize = "admin:ManageOwnMFA")]
|
||||
ManageOwnMFAAdminAction,
|
||||
#[strum(serialize = "admin:DeleteUser")]
|
||||
DeleteUserAdminAction,
|
||||
#[strum(serialize = "admin:ListUsers")]
|
||||
@@ -702,8 +698,6 @@ impl AdminAction {
|
||||
| AdminAction::TLSCertificatePublishAdminAction
|
||||
| AdminAction::TLSCertificateStatusAdminAction
|
||||
| AdminAction::CreateUserAdminAction
|
||||
| AdminAction::ChangeOwnPasswordAdminAction
|
||||
| AdminAction::ManageOwnMFAAdminAction
|
||||
| AdminAction::DeleteUserAdminAction
|
||||
| AdminAction::ListUsersAdminAction
|
||||
| AdminAction::EnableUserAdminAction
|
||||
|
||||
@@ -505,45 +505,6 @@ pub struct RenameDataResponse {
|
||||
#[prost(bytes = "bytes", tag = "4")]
|
||||
pub rename_data_resp_bin: ::prost::bytes::Bytes,
|
||||
}
|
||||
/// Ordinary rename transport batch. Protected rename variants intentionally use
|
||||
/// their dedicated fail-closed RPCs and are never admitted here.
|
||||
#[allow(clippy::all)]
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct RenameDataBatchItem {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub item_index: u32,
|
||||
#[prost(string, tag = "2")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub file_info_msgpack: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[allow(clippy::all)]
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct RenameDataBatchRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub src_volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub src_path: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub dst_volume: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "4")]
|
||||
pub dst_path: ::prost::alloc::string::String,
|
||||
#[prost(message, repeated, tag = "5")]
|
||||
pub items: ::prost::alloc::vec::Vec<RenameDataBatchItem>,
|
||||
}
|
||||
#[allow(clippy::all)]
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
#[prost(skip_debug)]
|
||||
pub struct RenameDataBatchResult {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub item_index: u32,
|
||||
#[prost(bool, tag = "2")]
|
||||
pub success: bool,
|
||||
#[prost(message, optional, tag = "3")]
|
||||
pub error: ::core::option::Option<Error>,
|
||||
#[prost(bytes = "bytes", tag = "4")]
|
||||
pub rename_data_resp_msgpack: ::prost::bytes::Bytes,
|
||||
}
|
||||
/// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
|
||||
/// accepts an arbitrary volume or object path.
|
||||
#[allow(clippy::all)]
|
||||
@@ -2933,21 +2894,6 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "RenameData"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn rename_data_batch(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::RenameDataBatchRequest>,
|
||||
) -> std::result::Result<tonic::Response<tonic::codec::Streaming<super::RenameDataBatchResult>>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameDataBatch");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "RenameDataBatch"));
|
||||
self.inner.server_streaming(req, path, codec).await
|
||||
}
|
||||
/// The method itself requires source-preserving multipart publication on every peer.
|
||||
pub async fn rename_multipart_data(
|
||||
&mut self,
|
||||
@@ -4306,14 +4252,6 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::RenameDataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::RenameDataResponse>, tonic::Status>;
|
||||
/// Server streaming response type for the RenameDataBatch method.
|
||||
type RenameDataBatchStream: tonic::codegen::tokio_stream::Stream<Item = std::result::Result<super::RenameDataBatchResult, tonic::Status>>
|
||||
+ std::marker::Send
|
||||
+ 'static;
|
||||
async fn rename_data_batch(
|
||||
&self,
|
||||
request: tonic::Request<super::RenameDataBatchRequest>,
|
||||
) -> std::result::Result<tonic::Response<Self::RenameDataBatchStream>, tonic::Status>;
|
||||
/// The method itself requires source-preserving multipart publication on every peer.
|
||||
async fn rename_multipart_data(
|
||||
&self,
|
||||
@@ -5635,35 +5573,6 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/RenameDataBatch" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct RenameDataBatchSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::ServerStreamingService<super::RenameDataBatchRequest> for RenameDataBatchSvc<T> {
|
||||
type Response = super::RenameDataBatchResult;
|
||||
type ResponseStream = T::RenameDataBatchStream;
|
||||
type Future = BoxFuture<tonic::Response<Self::ResponseStream>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::RenameDataBatchRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::rename_data_batch(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = RenameDataBatchSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.server_streaming(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/RenameMultipartData" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct RenameMultipartDataSvc<T: NodeService>(pub Arc<T>);
|
||||
|
||||
@@ -67,18 +67,6 @@ impl std::fmt::Debug for proto_gen::node_service::RenameDataResponse {
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for proto_gen::node_service::RenameDataBatchResult {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("RenameDataBatchResult")
|
||||
.field("item_index", &self.item_index)
|
||||
.field("success", &self.success)
|
||||
.field("error", &self.error)
|
||||
.field("rename_data_resp_msgpack", &"[REDACTED]")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for proto_gen::node_service::RecoverMultipartSourceRequest {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
@@ -1183,23 +1171,6 @@ pub fn canonical_rename_data_request_body(
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_rename_data_batch_request_body(
|
||||
request: &proto_gen::node_service::RenameDataBatchRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-rename-data-batch-request-v1\0");
|
||||
body.push_str(&request.src_volume)?;
|
||||
body.push_str(&request.src_path)?;
|
||||
body.push_str(&request.dst_volume)?;
|
||||
body.push_str(&request.dst_path)?;
|
||||
body.push_count(request.items.len())?;
|
||||
for item in &request.items {
|
||||
body.push_u32(item.item_index);
|
||||
body.push_str(&item.disk)?;
|
||||
body.push_bytes(&item.file_info_msgpack)?;
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
/// The fixed-MPU recovery method authenticates every wire field in its own domain.
|
||||
pub fn canonical_recover_multipart_source_request_body(
|
||||
request: &proto_gen::node_service::RecoverMultipartSourceRequest,
|
||||
@@ -1719,10 +1690,9 @@ pub fn canonical_make_volumes_request_body(
|
||||
mod disk_mutation_canonical_tests {
|
||||
use super::proto_gen::node_service::{
|
||||
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
|
||||
DeleteVolumeRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, RenameDataBatchItem,
|
||||
RenameDataBatchRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, SettlePartTransactionRequest,
|
||||
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, UpdateMetadataRequest, WriteAllRequest,
|
||||
WriteMetadataRequest,
|
||||
DeleteVolumeRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, RenameDataRequest,
|
||||
RenameFileRequest, RenamePartRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest,
|
||||
SnapshotLeaseRenewRequest, SnapshotLeaseRequest, UpdateMetadataRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
};
|
||||
use super::*;
|
||||
|
||||
@@ -1874,51 +1844,6 @@ mod disk_mutation_canonical_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_batch_canonical_body_binds_every_field_and_item_order() {
|
||||
let baseline = RenameDataBatchRequest {
|
||||
src_volume: "src-vol".into(),
|
||||
src_path: "src-path".into(),
|
||||
dst_volume: "dst-vol".into(),
|
||||
dst_path: "dst-path".into(),
|
||||
items: vec![
|
||||
RenameDataBatchItem {
|
||||
item_index: 1,
|
||||
disk: "disk-a".into(),
|
||||
file_info_msgpack: vec![0x81, 0x01].into(),
|
||||
},
|
||||
RenameDataBatchItem {
|
||||
item_index: 2,
|
||||
disk: "disk-b".into(),
|
||||
file_info_msgpack: vec![0x81, 0x02].into(),
|
||||
},
|
||||
],
|
||||
};
|
||||
let mut bodies = vec![canonical_rename_data_batch_request_body(&baseline).expect("baseline batch body")];
|
||||
for mutate in [
|
||||
|r: &mut RenameDataBatchRequest| r.src_volume = "src-vol2".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.src_path = "src-path2".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.dst_volume = "dst-vol2".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.dst_path = "dst-path2".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.items[0].item_index = 3,
|
||||
|r: &mut RenameDataBatchRequest| r.items[0].disk = "disk-c".into(),
|
||||
|r: &mut RenameDataBatchRequest| r.items[0].file_info_msgpack = vec![0x81, 0x03].into(),
|
||||
|r: &mut RenameDataBatchRequest| r.items.swap(0, 1),
|
||||
|r: &mut RenameDataBatchRequest| {
|
||||
r.items.pop();
|
||||
},
|
||||
] {
|
||||
let mut changed = baseline.clone();
|
||||
mutate(&mut changed);
|
||||
bodies.push(canonical_rename_data_batch_request_body(&changed).expect("changed batch body"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
assert_ne!(
|
||||
canonical_rename_data_batch_request_body(&baseline).expect("batch body"),
|
||||
canonical_rename_data_request_body(&RenameDataRequest::default()).expect("unary body"),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_version_canonical_body_binds_every_field() {
|
||||
let baseline = DeleteVersionRequest {
|
||||
@@ -3730,7 +3655,7 @@ fn complete_pending_rebuild_on_cache_hit(dial_addr: &str, cache_key: &str) -> bo
|
||||
}
|
||||
|
||||
// Limits transport response futures; client RPC metrics separately cover unary body decoding.
|
||||
pub const INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT: usize = 64;
|
||||
const INTERNODE_RPC_CONCURRENCY_LIMIT: usize = 64;
|
||||
|
||||
// Quorum reads cancel surplus metadata RPCs in bursts. Keep bounded reset state
|
||||
// for their in-flight replies without spending the connection's protocol-error
|
||||
@@ -4184,7 +4109,7 @@ async fn dial_channel(
|
||||
rustfs_config::DEFAULT_INTERNODE_BULK_CONCURRENCY_LIMIT,
|
||||
))?
|
||||
} else {
|
||||
INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT
|
||||
INTERNODE_RPC_CONCURRENCY_LIMIT
|
||||
};
|
||||
let stream_window = checked_rpc_window(stream_window)?;
|
||||
let conn_window = checked_rpc_window(conn_window)?;
|
||||
|
||||
@@ -69,7 +69,6 @@ fn main() -> Result<(), AnyError> {
|
||||
// Their hand-written Debug implementations redact the complete payload.
|
||||
.skip_debug([
|
||||
".node_service.RenameDataResponse",
|
||||
".node_service.RenameDataBatchResult",
|
||||
".node_service.RecoverMultipartSourceRequest",
|
||||
".node_service.RecoverMultipartSourceResponse",
|
||||
])
|
||||
|
||||
@@ -345,29 +345,6 @@ message RenameDataResponse {
|
||||
bytes rename_data_resp_bin = 4;
|
||||
}
|
||||
|
||||
// Ordinary rename transport batch. Protected rename variants intentionally use
|
||||
// their dedicated fail-closed RPCs and are never admitted here.
|
||||
message RenameDataBatchItem {
|
||||
uint32 item_index = 1;
|
||||
string disk = 2;
|
||||
bytes file_info_msgpack = 3;
|
||||
}
|
||||
|
||||
message RenameDataBatchRequest {
|
||||
string src_volume = 1;
|
||||
string src_path = 2;
|
||||
string dst_volume = 3;
|
||||
string dst_path = 4;
|
||||
repeated RenameDataBatchItem items = 5;
|
||||
}
|
||||
|
||||
message RenameDataBatchResult {
|
||||
uint32 item_index = 1;
|
||||
bool success = 2;
|
||||
optional Error error = 3;
|
||||
bytes rename_data_resp_msgpack = 4;
|
||||
}
|
||||
|
||||
// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
|
||||
// accepts an arbitrary volume or object path.
|
||||
message PoolMetaNativeRecoveryRequest {
|
||||
@@ -1417,7 +1394,6 @@ service NodeService {
|
||||
rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
|
||||
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
|
||||
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameDataBatch(RenameDataBatchRequest) returns (stream RenameDataBatchResult) {}; // auth-policy: body-bound
|
||||
// The method itself requires source-preserving multipart publication on every peer.
|
||||
rpc RenameMultipartData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
|
||||
@@ -29,11 +29,12 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["rustfs-s3-types/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "rustfs-s3-types/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "rustfs-s3-types/hotpath-cpu"]
|
||||
hotpath = ["hotpath/hotpath", "rustfs-s3-types/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-s3-types/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-s3-types/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
rustfs-s3-types = { workspace = true }
|
||||
|
||||
[lib]
|
||||
|
||||
@@ -67,7 +67,6 @@ bytes = { workspace = true, features = ["serde"] }
|
||||
base64-simd.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
crc-fast.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
datafusion = { workspace = true, default-features = false, features = ["parquet", "recursive_protection", "sql"] }
|
||||
|
||||
@@ -54,7 +54,6 @@ use futures::pin_mut;
|
||||
use futures::{Stream, StreamExt, TryStreamExt, future::ready, stream};
|
||||
use futures_core::stream::BoxStream;
|
||||
use http::{HeaderMap, HeaderValue, header::HeaderName};
|
||||
use jiff::Timestamp;
|
||||
use memchr::{memchr, memmem};
|
||||
use rustfs_common::DEFAULT_DELIMITER;
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
@@ -757,22 +756,10 @@ fn snapshot_last_modified(snapshot: &SelectObjectSnapshot) -> Result<DateTime<Ut
|
||||
store: "EcObjectStore",
|
||||
source: std::io::Error::new(std::io::ErrorKind::InvalidData, "snapshot metadata has no modification time").into(),
|
||||
})?;
|
||||
let timestamp = Timestamp::new(
|
||||
mod_time.unix_timestamp(),
|
||||
i32::try_from(mod_time.nanosecond()).map_err(|_| o_Error::Generic {
|
||||
store: "EcObjectStore",
|
||||
source: std::io::Error::new(std::io::ErrorKind::InvalidData, "snapshot modification time is out of range").into(),
|
||||
})?,
|
||||
)
|
||||
.map_err(|_| o_Error::Generic {
|
||||
DateTime::<Utc>::from_timestamp(mod_time.unix_timestamp(), mod_time.nanosecond()).ok_or_else(|| o_Error::Generic {
|
||||
store: "EcObjectStore",
|
||||
source: std::io::Error::new(std::io::ErrorKind::InvalidData, "snapshot modification time is out of range").into(),
|
||||
})?;
|
||||
let system_time = std::time::SystemTime::try_from(timestamp).map_err(|_| o_Error::Generic {
|
||||
store: "EcObjectStore",
|
||||
source: std::io::Error::new(std::io::ErrorKind::InvalidData, "snapshot modification time is out of range").into(),
|
||||
})?;
|
||||
Ok(DateTime::<Utc>::from(system_time))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn scan_range_from_bounds(start: Option<i64>, end: Option<i64>, object_size: u64) -> Result<Option<SelectScanRange>> {
|
||||
|
||||
@@ -28,6 +28,7 @@ categories = ["web-programming", "development-tools", "data-structures"]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
metrics = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
|
||||
@@ -12,23 +12,23 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use jiff::Timestamp;
|
||||
use chrono::{DateTime, Utc};
|
||||
use std::sync::LazyLock;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// Global initialization time of the RustFS node.
|
||||
pub static GLOBAL_INIT_TIME: LazyLock<RwLock<Option<Timestamp>>> = LazyLock::new(|| RwLock::new(None));
|
||||
pub static GLOBAL_INIT_TIME: LazyLock<RwLock<Option<DateTime<Utc>>>> = LazyLock::new(|| RwLock::new(None));
|
||||
|
||||
/// Set the global RustFS initialization time to the current UTC time.
|
||||
pub async fn set_global_init_time_now() {
|
||||
let now = Timestamp::now();
|
||||
let now = Utc::now();
|
||||
*GLOBAL_INIT_TIME.write().await = Some(now);
|
||||
}
|
||||
|
||||
/// Get the global RustFS initialization time.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Option<Timestamp>` - The initialization time if set.
|
||||
pub async fn get_global_init_time() -> Option<Timestamp> {
|
||||
/// * `Option<DateTime<Utc>>` - The initialization time if set.
|
||||
pub async fn get_global_init_time() -> Option<DateTime<Utc>> {
|
||||
*GLOBAL_INIT_TIME.read().await
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ use crate::last_minute::{AccElem, LastMinuteLatency};
|
||||
use crate::usage_candidate::{
|
||||
ScannerUsageCandidateCount, ScannerUsageCandidateKind, ScannerUsageCandidateOutcome, USAGE_CANDIDATE_COUNTER_SLOTS,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use jiff::Timestamp;
|
||||
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -704,6 +705,27 @@ impl CurrentPathTracker {
|
||||
}
|
||||
}
|
||||
|
||||
fn chrono_to_jiff_timestamp(dt: DateTime<Utc>) -> Timestamp {
|
||||
let seconds = dt.timestamp();
|
||||
let nanoseconds = match i32::try_from(dt.timestamp_subsec_nanos()) {
|
||||
Ok(nanoseconds) => nanoseconds,
|
||||
Err(_) => {
|
||||
return if seconds < 0 { Timestamp::MIN } else { Timestamp::MAX };
|
||||
}
|
||||
};
|
||||
|
||||
match Timestamp::new(seconds, nanoseconds) {
|
||||
Ok(timestamp) => timestamp,
|
||||
Err(_) => {
|
||||
if seconds < 0 {
|
||||
Timestamp::MIN
|
||||
} else {
|
||||
Timestamp::MAX
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
|
||||
let duration = now.duration_since(earlier);
|
||||
if duration.is_negative() {
|
||||
@@ -1218,8 +1240,8 @@ impl ScannerUsageSaveResult {
|
||||
pub struct CurrentCycle {
|
||||
pub current: u64,
|
||||
pub next: u64,
|
||||
pub cycle_completed: Vec<Timestamp>,
|
||||
pub started: Timestamp,
|
||||
pub cycle_completed: Vec<DateTime<Utc>>,
|
||||
pub started: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
@@ -3128,12 +3150,12 @@ impl Metrics {
|
||||
*self.scanner_leader_lock_last_error.write().await = error.into();
|
||||
self.scanner_leader_lock_held.store(held, Ordering::Relaxed);
|
||||
self.scanner_leader_lock_last_update_unix_secs
|
||||
.store(Timestamp::now().as_second().max(0) as u64, Ordering::Relaxed);
|
||||
.store(Utc::now().timestamp().max(0) as u64, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scanner_cycle_end_time(&self) {
|
||||
self.last_scan_cycle_end_unix_secs
|
||||
.store(Timestamp::now().as_second().max(0) as u64, Ordering::Relaxed);
|
||||
.store(Utc::now().timestamp().max(0) as u64, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_complete(&self, success: bool, duration: Duration) {
|
||||
@@ -3722,8 +3744,8 @@ impl Metrics {
|
||||
.snapshot();
|
||||
let has_cycle = if let Some(cycle) = cycle.as_ref() {
|
||||
m.current_cycle = cycle.current;
|
||||
m.cycles_completed_at = cycle.cycle_completed.clone();
|
||||
m.current_started = cycle.started;
|
||||
m.cycles_completed_at = cycle.cycle_completed.iter().copied().map(chrono_to_jiff_timestamp).collect();
|
||||
m.current_started = chrono_to_jiff_timestamp(cycle.started);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
@@ -3761,7 +3783,7 @@ impl Metrics {
|
||||
};
|
||||
|
||||
if !has_cycle && let Some(init_time) = crate::init_time::get_global_init_time().await {
|
||||
m.current_started = init_time;
|
||||
m.current_started = chrono_to_jiff_timestamp(init_time);
|
||||
}
|
||||
|
||||
m.collected_at = Timestamp::now();
|
||||
@@ -5113,8 +5135,8 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn report_preserves_current_cycle_started_time() {
|
||||
let previous_init_time = *crate::init_time::GLOBAL_INIT_TIME.read().await;
|
||||
let init_time = Timestamp::now() - jiff::SignedDuration::from_hours(1);
|
||||
let cycle_started = Timestamp::now();
|
||||
let init_time = Utc::now() - chrono::Duration::hours(1);
|
||||
let cycle_started = Utc::now();
|
||||
*crate::init_time::GLOBAL_INIT_TIME.write().await = Some(init_time);
|
||||
|
||||
let metrics = Metrics::new();
|
||||
@@ -5129,7 +5151,7 @@ mod tests {
|
||||
let report = metrics.report().await;
|
||||
*crate::init_time::GLOBAL_INIT_TIME.write().await = previous_init_time;
|
||||
|
||||
assert_eq!(report.current_started, cycle_started);
|
||||
assert_eq!(report.current_started, chrono_to_jiff_timestamp(cycle_started));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -5696,7 +5718,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn scan_cycle_activity_and_cycle_state_publish_together() {
|
||||
let metrics = Metrics::new();
|
||||
let cycle_started = Timestamp::now() - jiff::SignedDuration::from_secs(5);
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let active_cycle = CurrentCycle {
|
||||
current: 12,
|
||||
next: 13,
|
||||
@@ -5716,7 +5738,7 @@ mod tests {
|
||||
let active = metrics.report().await;
|
||||
assert!(active.current_cycle_active);
|
||||
assert_eq!(active.current_cycle, 12);
|
||||
assert_eq!(active.current_started, cycle_started);
|
||||
assert_eq!(active.current_started, chrono_to_jiff_timestamp(cycle_started));
|
||||
|
||||
let idle_cycle = CurrentCycle {
|
||||
current: 0,
|
||||
@@ -5742,7 +5764,7 @@ mod tests {
|
||||
let cycle_ten = CurrentCycle {
|
||||
current: 10,
|
||||
next: 11,
|
||||
started: Timestamp::now() - jiff::SignedDuration::from_secs(10),
|
||||
started: Utc::now() - chrono::Duration::seconds(10),
|
||||
..Default::default()
|
||||
};
|
||||
let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await;
|
||||
@@ -5762,7 +5784,7 @@ mod tests {
|
||||
.start_scan_cycle_work_with_cycle(CurrentCycle {
|
||||
current: 11,
|
||||
next: 12,
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -90,7 +90,7 @@ thiserror = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
jiff = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
rmp-serde = { workspace = true }
|
||||
hmac = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
|
||||
@@ -41,7 +41,7 @@ use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed};
|
||||
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard};
|
||||
use crate::{ScannerConfigObjectDelete, ScannerObjectIO, ScannerObjectOptions};
|
||||
use bytes::Bytes;
|
||||
use jiff::Timestamp;
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_config::ScannerSpeed;
|
||||
#[cfg(test)]
|
||||
use rustfs_config::{
|
||||
@@ -1637,7 +1637,7 @@ fn get_cycle_scan_mode(current_cycle: u64, start: ScannerBitrotStart, policy: Sc
|
||||
ScannerBitrotStart::Anchor(anchored) => anchored,
|
||||
};
|
||||
|
||||
let elapsed = std::time::Duration::try_from(Timestamp::now().duration_since(started)).unwrap_or(Duration::ZERO);
|
||||
let elapsed = Utc::now().signed_duration_since(started).to_std().unwrap_or(Duration::ZERO);
|
||||
if elapsed >= bitrot_cycle {
|
||||
HealScanMode::Deep
|
||||
} else {
|
||||
@@ -1649,7 +1649,7 @@ fn background_heal_info_for_scan_start(
|
||||
mut info: BackgroundHealInfo,
|
||||
current_cycle: u64,
|
||||
scan_mode: HealScanMode,
|
||||
now: Timestamp,
|
||||
now: DateTime<Utc>,
|
||||
policy: ScannerBitrotPolicy,
|
||||
) -> Option<BackgroundHealInfo> {
|
||||
let reset_bitrot_start = scan_mode == HealScanMode::Deep && should_reset_bitrot_start(&info, current_cycle, now, policy);
|
||||
@@ -1666,7 +1666,12 @@ fn background_heal_info_for_scan_start(
|
||||
Some(info)
|
||||
}
|
||||
|
||||
fn should_reset_bitrot_start(info: &BackgroundHealInfo, current_cycle: u64, now: Timestamp, policy: ScannerBitrotPolicy) -> bool {
|
||||
fn should_reset_bitrot_start(
|
||||
info: &BackgroundHealInfo,
|
||||
current_cycle: u64,
|
||||
now: DateTime<Utc>,
|
||||
policy: ScannerBitrotPolicy,
|
||||
) -> bool {
|
||||
let Some(bitrot_start_time) = info.bitrot_start_time else {
|
||||
return true;
|
||||
};
|
||||
@@ -1687,7 +1692,10 @@ fn should_reset_bitrot_start(info: &BackgroundHealInfo, current_cycle: u64, now:
|
||||
return false;
|
||||
}
|
||||
|
||||
let elapsed = now.duration_since(bitrot_start_time).try_into().unwrap_or(Duration::ZERO);
|
||||
let elapsed = now
|
||||
.signed_duration_since(bitrot_start_time)
|
||||
.to_std()
|
||||
.unwrap_or(Duration::ZERO);
|
||||
elapsed >= bitrot_cycle
|
||||
}
|
||||
|
||||
@@ -1712,7 +1720,7 @@ fn background_heal_info_for_scan_result(
|
||||
background_heal_info_for_scan_complete(info, scan_mode)
|
||||
}
|
||||
|
||||
fn retain_recent_cycle_completions(cycle_completed: &mut Vec<Timestamp>) {
|
||||
fn retain_recent_cycle_completions(cycle_completed: &mut Vec<DateTime<Utc>>) {
|
||||
let keep = data_usage_update_dir_cycles() as usize;
|
||||
if cycle_completed.len() > keep {
|
||||
let drop_count = cycle_completed.len() - keep;
|
||||
@@ -1913,7 +1921,7 @@ where
|
||||
let usage_persist_timeout = data_usage_persist_timeout();
|
||||
cycle_info.current = cycle_info.next;
|
||||
let now = Instant::now();
|
||||
cycle_info.started = Timestamp::now();
|
||||
cycle_info.started = Utc::now();
|
||||
|
||||
let mut cycle_metrics_guard =
|
||||
ScannerCycleMetricsGuard::new(cycle_info.clone(), leader_epoch, cycle_budget.remote_scan_progress()).await;
|
||||
@@ -1947,7 +1955,7 @@ where
|
||||
// The persisted record wins when it carries a start; the in-memory record
|
||||
// covers a save that was skipped or lost since this process recorded it.
|
||||
let (mut background_heal_info, mut bitrot_start_source) = resolve_background_heal_info_for_cycle(&background_heal_read);
|
||||
let bitrot_start = scanner_bitrot_start_for_cycle(&background_heal_info, Timestamp::now());
|
||||
let bitrot_start = scanner_bitrot_start_for_cycle(&background_heal_info, Utc::now());
|
||||
if matches!(bitrot_start, ScannerBitrotStart::Anchor(_)) {
|
||||
bitrot_start_source = ScannerBitrotStartSource::Memory;
|
||||
}
|
||||
@@ -1969,7 +1977,7 @@ where
|
||||
background_heal_info.clone(),
|
||||
cycle_info.current,
|
||||
scan_mode,
|
||||
Timestamp::now(),
|
||||
Utc::now(),
|
||||
bitrot_policy,
|
||||
) {
|
||||
background_heal_info = new_heal_info.clone();
|
||||
@@ -1981,7 +1989,7 @@ where
|
||||
save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
|
||||
}
|
||||
record_scanner_bitrot_status(
|
||||
scanner_bitrot_start_for_cycle(&background_heal_info, Timestamp::now()),
|
||||
scanner_bitrot_start_for_cycle(&background_heal_info, Utc::now()),
|
||||
bitrot_start_source,
|
||||
deep_scan_supported,
|
||||
scan_mode,
|
||||
@@ -2809,7 +2817,7 @@ where
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
cycle_info.current = 0;
|
||||
cycle_info.cycle_completed.push(Timestamp::now());
|
||||
cycle_info.cycle_completed.push(Utc::now());
|
||||
global_metrics().clear_current_scan_mode();
|
||||
|
||||
retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
|
||||
|
||||
@@ -653,7 +653,7 @@ pub(super) fn record_scanner_cycle_recovery_retry(attempt: u32, cycle_interval:
|
||||
}
|
||||
|
||||
fn unix_now_secs() -> u64 {
|
||||
u64::try_from(Timestamp::now().as_second()).unwrap_or(0)
|
||||
u64::try_from(Utc::now().timestamp()).unwrap_or(0)
|
||||
}
|
||||
|
||||
pub(super) fn sha256_hex(parts: &[&[u8]]) -> String {
|
||||
|
||||
@@ -21,7 +21,7 @@ use std::sync::RwLock as StdRwLock;
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackgroundHealInfo {
|
||||
/// Bitrot scan start time
|
||||
pub bitrot_start_time: Option<Timestamp>,
|
||||
pub bitrot_start_time: Option<DateTime<Utc>>,
|
||||
/// Bitrot scan start cycle
|
||||
pub bitrot_start_cycle: u64,
|
||||
/// Current scan mode
|
||||
@@ -222,11 +222,11 @@ pub(super) enum ScannerBitrotStart {
|
||||
None,
|
||||
/// A Deep selection window opened at `cycle` / `started`, from the
|
||||
/// persisted or the remembered record.
|
||||
Window { cycle: u64, started: Timestamp },
|
||||
Window { cycle: u64, started: DateTime<Utc> },
|
||||
/// No Deep window is recorded anywhere; the cadence counts from the first
|
||||
/// cycle this process ran without one. Never persisted, so it opens no
|
||||
/// selection window and old readers see no new state.
|
||||
Anchor(Timestamp),
|
||||
Anchor(DateTime<Utc>),
|
||||
}
|
||||
|
||||
impl ScannerBitrotStart {
|
||||
@@ -245,7 +245,7 @@ impl ScannerBitrotStart {
|
||||
}
|
||||
}
|
||||
|
||||
fn started(self) -> Option<Timestamp> {
|
||||
fn started(self) -> Option<DateTime<Utc>> {
|
||||
match self {
|
||||
Self::None => None,
|
||||
Self::Window { started, .. } => Some(started),
|
||||
@@ -290,9 +290,9 @@ static SCANNER_BITROT_STATUS: StdLazyLock<StdRwLock<ScannerBitrotStatus>> =
|
||||
StdLazyLock::new(|| StdRwLock::new(ScannerBitrotStatus::default()));
|
||||
|
||||
/// When this process first ran a cycle without any recorded Deep window.
|
||||
static SCANNER_BITROT_ANCHOR: StdRwLock<Option<Timestamp>> = StdRwLock::new(None);
|
||||
static SCANNER_BITROT_ANCHOR: StdRwLock<Option<DateTime<Utc>>> = StdRwLock::new(None);
|
||||
|
||||
fn bitrot_cadence_anchor(now: Timestamp) -> Timestamp {
|
||||
fn bitrot_cadence_anchor(now: DateTime<Utc>) -> DateTime<Utc> {
|
||||
let mut anchor = SCANNER_BITROT_ANCHOR.write().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
*anchor.get_or_insert(now)
|
||||
}
|
||||
@@ -304,7 +304,7 @@ pub(super) fn reset_bitrot_cadence_anchor_for_test() {
|
||||
|
||||
/// The cadence reference for a cycle: the recorded Deep window start when the
|
||||
/// record carries one, otherwise this process's in-memory anchor.
|
||||
pub(super) fn scanner_bitrot_start_for_cycle(info: &BackgroundHealInfo, now: Timestamp) -> ScannerBitrotStart {
|
||||
pub(super) fn scanner_bitrot_start_for_cycle(info: &BackgroundHealInfo, now: DateTime<Utc>) -> ScannerBitrotStart {
|
||||
match info.bitrot_start_time {
|
||||
Some(started) => ScannerBitrotStart::Window {
|
||||
cycle: info.bitrot_start_cycle,
|
||||
@@ -368,7 +368,7 @@ pub(super) fn record_scanner_bitrot_status(
|
||||
bitrot_start_cycle: start.cycle(),
|
||||
bitrot_start_unix_secs: start
|
||||
.started()
|
||||
.and_then(|started| u64::try_from(started.as_second()).ok())
|
||||
.and_then(|started| u64::try_from(started.timestamp()).ok())
|
||||
.unwrap_or(0),
|
||||
current_scan_mode: scan_mode.as_str(),
|
||||
};
|
||||
|
||||
@@ -32,7 +32,6 @@ use crate::{
|
||||
ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader, init_bucket_metadata_sys_for_scanner_tests,
|
||||
init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
|
||||
};
|
||||
use jiff::Timestamp;
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
ScannerUsagePublicationClass, ScannerUsagePublicationOutcome, ScannerUsagePublicationStatus,
|
||||
};
|
||||
@@ -1459,8 +1458,8 @@ async fn test_mark_scan_cycle_idle_clears_published_cycle_state() {
|
||||
let mut cycle_info = CurrentCycle {
|
||||
current: 12,
|
||||
next: 13,
|
||||
cycle_completed: vec![Timestamp::now()],
|
||||
started: Timestamp::now(),
|
||||
cycle_completed: vec![Utc::now()],
|
||||
started: Utc::now(),
|
||||
};
|
||||
|
||||
global_metrics().set_current_scan_mode(HealScanMode::Deep);
|
||||
@@ -1513,7 +1512,7 @@ async fn scanner_cycle_metrics_guard_installs_budget_remote_progress() {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
|
||||
let cycle_started = Timestamp::now() - jiff::SignedDuration::from_secs(5);
|
||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||
let mut cycle_info = CurrentCycle {
|
||||
current: 0,
|
||||
next: 1,
|
||||
@@ -1524,8 +1523,11 @@ async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
|
||||
let setup_report = global_metrics().report().await;
|
||||
assert!(setup_report.current_cycle_active);
|
||||
assert_eq!(setup_report.current_cycle, 0);
|
||||
assert_eq!(setup_report.current_started.as_second(), cycle_started.as_second());
|
||||
assert_eq!(setup_report.current_started.subsec_nanosecond(), cycle_started.subsec_nanosecond());
|
||||
assert_eq!(setup_report.current_started.as_second(), cycle_started.timestamp());
|
||||
assert_eq!(
|
||||
setup_report.current_started.subsec_nanosecond(),
|
||||
i32::try_from(cycle_started.timestamp_subsec_nanos()).expect("chrono nanoseconds fit in i32")
|
||||
);
|
||||
|
||||
mark_scan_cycle_idle(&mut cycle_info, &mut guard).await;
|
||||
let idle_report = global_metrics().report().await;
|
||||
@@ -1540,7 +1542,7 @@ async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finaliz
|
||||
let mut cycle_info = CurrentCycle {
|
||||
current: 12,
|
||||
next: 13,
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone(), 1, Arc::new(RemoteScanProgress::default())).await;
|
||||
@@ -1566,7 +1568,7 @@ async fn scanner_cycle_metrics_guard_drop_clears_activity() {
|
||||
CurrentCycle {
|
||||
current: 12,
|
||||
next: 13,
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
..Default::default()
|
||||
},
|
||||
1,
|
||||
@@ -1939,7 +1941,7 @@ async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
|
||||
current: 12,
|
||||
next: 12,
|
||||
cycle_completed: vec![],
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
};
|
||||
let mut cycle_metrics_guard =
|
||||
ScannerCycleMetricsGuard::new(cycle_info.clone(), 1, Arc::new(RemoteScanProgress::default())).await;
|
||||
@@ -1976,7 +1978,7 @@ async fn scanner_cycle_recovers_to_newer_durable_cache_floor() {
|
||||
current: 12,
|
||||
next: 12,
|
||||
cycle_completed: vec![],
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
};
|
||||
let mut cycle_metrics_guard =
|
||||
ScannerCycleMetricsGuard::new(cycle_info.clone(), 1, Arc::new(RemoteScanProgress::default())).await;
|
||||
@@ -2017,7 +2019,7 @@ async fn scanner_cycle_rejects_invalid_cache_floor() {
|
||||
current: 12,
|
||||
next: 12,
|
||||
cycle_completed: vec![],
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
};
|
||||
let mut cycle_metrics_guard =
|
||||
ScannerCycleMetricsGuard::new(cycle_info.clone(), 1, Arc::new(RemoteScanProgress::default())).await;
|
||||
@@ -2066,7 +2068,7 @@ fn scanner_cycle_state_decodes_legacy_and_fenced_formats() {
|
||||
current: 12,
|
||||
next: 13,
|
||||
cycle_completed: vec![],
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
};
|
||||
let mut legacy = cycle.next.to_le_bytes().to_vec();
|
||||
legacy.extend(cycle.marshal().expect("legacy cycle state should encode"));
|
||||
@@ -2676,7 +2678,7 @@ async fn full_rescan_reset_preserves_valid_primary_when_marker_is_malformed() {
|
||||
#[serial]
|
||||
async fn full_rescan_reset_resumes_cleanup_pending_preserved_primary() {
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
||||
let completed_at = Timestamp::now();
|
||||
let completed_at = Utc::now();
|
||||
let primary = CurrentCycle {
|
||||
current: 3,
|
||||
next: 42,
|
||||
@@ -3307,8 +3309,8 @@ async fn scanner_usage_floor_recovers_rc3_empty_fences_and_preserves_cycle_numbe
|
||||
let mut cycle = CurrentCycle {
|
||||
current: 17_117,
|
||||
next: 17_118,
|
||||
cycle_completed: vec![Timestamp::now()],
|
||||
started: Timestamp::now(),
|
||||
cycle_completed: vec![Utc::now()],
|
||||
started: Utc::now(),
|
||||
};
|
||||
assert_eq!(
|
||||
prepare_cycle_for_usage_floor_bootstrap(&mut cycle, restart_floor, restart_state),
|
||||
@@ -4755,8 +4757,8 @@ fn missing_usage_floor_discards_unfenced_cycle_progress() {
|
||||
let mut cycle = CurrentCycle {
|
||||
current: 11,
|
||||
next: 12,
|
||||
cycle_completed: vec![Timestamp::now()],
|
||||
started: Timestamp::now(),
|
||||
cycle_completed: vec![Utc::now()],
|
||||
started: Utc::now(),
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
@@ -5075,7 +5077,7 @@ async fn test_finalize_partial_scan_cycle_reports_persist_failure() {
|
||||
current: 12,
|
||||
next: 12,
|
||||
cycle_completed: vec![],
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
};
|
||||
let mut cycle_metrics_guard =
|
||||
ScannerCycleMetricsGuard::new(cycle_info.clone(), 1, Arc::new(RemoteScanProgress::default())).await;
|
||||
@@ -5098,7 +5100,7 @@ async fn test_persist_scanner_cycle_state_reconciles_newer_winner() {
|
||||
current: 0,
|
||||
next: 12,
|
||||
cycle_completed: vec![],
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
};
|
||||
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut initial_revision, 1).await);
|
||||
|
||||
@@ -5135,7 +5137,7 @@ async fn test_persist_scanner_cycle_state_retries_after_stale_winner() {
|
||||
current: 0,
|
||||
next: 12,
|
||||
cycle_completed: vec![],
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
};
|
||||
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut initial_revision, 1).await);
|
||||
|
||||
@@ -5169,7 +5171,7 @@ async fn test_persist_scanner_cycle_state_stops_retry_after_leader_fence() {
|
||||
current: 0,
|
||||
next: 12,
|
||||
cycle_completed: vec![],
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
};
|
||||
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut revision, 1).await);
|
||||
|
||||
@@ -5209,7 +5211,7 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf
|
||||
current: 0,
|
||||
next: 12,
|
||||
cycle_completed: vec![],
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
};
|
||||
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
|
||||
seed_usage_snapshot_for_leadership_claim(&store).await;
|
||||
@@ -5314,15 +5316,15 @@ async fn recovered_usage_bootstrap_claim_conflicts_preserve_the_highest_cycle_nu
|
||||
cycle = CurrentCycle {
|
||||
current: 17_117,
|
||||
next: 17_118,
|
||||
cycle_completed: vec![Timestamp::now()],
|
||||
started: Timestamp::now(),
|
||||
cycle_completed: vec![Utc::now()],
|
||||
started: Utc::now(),
|
||||
};
|
||||
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
|
||||
let winner = CurrentCycle {
|
||||
current: winner_next.saturating_sub(1),
|
||||
next: winner_next,
|
||||
cycle_completed: vec![Timestamp::now()],
|
||||
started: Timestamp::now(),
|
||||
cycle_completed: vec![Utc::now()],
|
||||
started: Utc::now(),
|
||||
};
|
||||
store
|
||||
.interleaving_puts
|
||||
@@ -5723,8 +5725,8 @@ async fn scanner_usage_state_reset_publishes_fenced_bootstrap_marker() {
|
||||
let cycle = CurrentCycle {
|
||||
current: 41,
|
||||
next: 42,
|
||||
cycle_completed: vec![Timestamp::now()],
|
||||
started: Timestamp::now(),
|
||||
cycle_completed: vec![Utc::now()],
|
||||
started: Utc::now(),
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
@@ -6107,8 +6109,8 @@ async fn scanner_usage_state_reset_resumes_real_store_cleanup_boundaries_after_r
|
||||
let cycle = CurrentCycle {
|
||||
current: 12,
|
||||
next: 42,
|
||||
cycle_completed: vec![Timestamp::now()],
|
||||
started: Timestamp::now(),
|
||||
cycle_completed: vec![Utc::now()],
|
||||
started: Utc::now(),
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
@@ -6558,7 +6560,7 @@ async fn test_leadership_claim_confirms_commit_after_returned_error() {
|
||||
current: 0,
|
||||
next: 12,
|
||||
cycle_completed: vec![],
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
};
|
||||
let mut persisted_epoch = 0;
|
||||
seed_usage_snapshot_for_leadership_claim(&store).await;
|
||||
@@ -6626,7 +6628,7 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() {
|
||||
let mut revision = DataUsageCacheRevision::Missing;
|
||||
let mut cycle = CurrentCycle {
|
||||
next: 12,
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut persisted_epoch = 4;
|
||||
@@ -6672,7 +6674,7 @@ async fn cycle_budget_lease_takeover_rejects_old_generation() {
|
||||
current: 0,
|
||||
next: 12,
|
||||
cycle_completed: vec![],
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
};
|
||||
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
|
||||
seed_usage_snapshot_for_leadership_claim(&store).await;
|
||||
@@ -11369,7 +11371,7 @@ fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
|
||||
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
||||
let start = ScannerBitrotStart::Window {
|
||||
cycle: 0,
|
||||
started: Timestamp::now(),
|
||||
started: Utc::now(),
|
||||
};
|
||||
let mode = get_cycle_scan_mode(10, start, ScannerBitrotPolicy::new(bitrot_scan_cycle(), true, 1024));
|
||||
assert_eq!(mode, HealScanMode::Deep);
|
||||
@@ -11380,8 +11382,8 @@ fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
|
||||
#[serial]
|
||||
fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() {
|
||||
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
||||
let recent = Timestamp::now() - jiff::SignedDuration::from_mins(30);
|
||||
let old = Timestamp::now() - jiff::SignedDuration::from_hours(2);
|
||||
let recent = Utc::now() - chrono::Duration::minutes(30);
|
||||
let old = Utc::now() - chrono::Duration::hours(2);
|
||||
|
||||
let recent = ScannerBitrotStart::Window {
|
||||
cycle: 0,
|
||||
@@ -11410,7 +11412,7 @@ fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
|
||||
assert_eq!(
|
||||
get_cycle_scan_mode(
|
||||
1,
|
||||
ScannerBitrotStart::Anchor(Timestamp::now() - jiff::SignedDuration::from_hours((400) * 24)),
|
||||
ScannerBitrotStart::Anchor(Utc::now() - chrono::Duration::days(400)),
|
||||
ScannerBitrotPolicy::new(bitrot_scan_cycle(), true, 1)
|
||||
),
|
||||
HealScanMode::Normal
|
||||
@@ -11433,21 +11435,17 @@ fn scan_mode_without_recorded_bitrot_start_stays_normal_until_the_cycle_elapses(
|
||||
|
||||
// An in-memory anchor counts the cadence without opening a selection
|
||||
// window: Normal until the cycle elapses, Deep afterwards.
|
||||
let anchored = ScannerBitrotStart::Anchor(Timestamp::now());
|
||||
let anchored = ScannerBitrotStart::Anchor(Utc::now());
|
||||
let modes: Vec<_> = (1..=3).map(|cycle| get_cycle_scan_mode(cycle, anchored, policy)).collect();
|
||||
assert_eq!(modes, vec![HealScanMode::Normal; 3]);
|
||||
assert_eq!(
|
||||
get_cycle_scan_mode(
|
||||
4,
|
||||
ScannerBitrotStart::Anchor(Timestamp::now() - jiff::SignedDuration::from_hours(2)),
|
||||
policy
|
||||
),
|
||||
get_cycle_scan_mode(4, ScannerBitrotStart::Anchor(Utc::now() - chrono::Duration::hours(2)), policy),
|
||||
HealScanMode::Deep
|
||||
);
|
||||
|
||||
// A recorded Deep window keeps its selection window, and an elapsed
|
||||
// cadence still triggers the next window.
|
||||
let started = Timestamp::now();
|
||||
let started = Utc::now();
|
||||
let window = ScannerBitrotStart::Window { cycle: 1, started };
|
||||
assert_eq!(get_cycle_scan_mode(1, window, policy), HealScanMode::Deep);
|
||||
assert_eq!(
|
||||
@@ -11455,7 +11453,7 @@ fn scan_mode_without_recorded_bitrot_start_stays_normal_until_the_cycle_elapses(
|
||||
2048,
|
||||
ScannerBitrotStart::Window {
|
||||
cycle: 1,
|
||||
started: started - jiff::SignedDuration::from_hours(2),
|
||||
started: started - chrono::Duration::hours(2),
|
||||
},
|
||||
policy
|
||||
),
|
||||
@@ -11468,7 +11466,7 @@ fn scan_mode_without_recorded_bitrot_start_stays_normal_until_the_cycle_elapses(
|
||||
#[serial]
|
||||
fn bitrot_cadence_anchor_is_remembered_in_memory_without_a_persisted_write() {
|
||||
reset_bitrot_cadence_anchor_for_test();
|
||||
let now = Timestamp::now();
|
||||
let now = Utc::now();
|
||||
let policy = ScannerBitrotPolicy::new(Some(Duration::from_secs(3600)), true, 1024);
|
||||
|
||||
// A Normal cycle without a recorded window writes nothing to storage ...
|
||||
@@ -11481,7 +11479,7 @@ fn bitrot_cadence_anchor_is_remembered_in_memory_without_a_persisted_write() {
|
||||
// keeps that anchor for later cycles.
|
||||
let first = scanner_bitrot_start_for_cycle(&BackgroundHealInfo::default(), now);
|
||||
assert_eq!(first, ScannerBitrotStart::Anchor(now));
|
||||
let later = scanner_bitrot_start_for_cycle(&BackgroundHealInfo::default(), now + jiff::SignedDuration::from_hours(2));
|
||||
let later = scanner_bitrot_start_for_cycle(&BackgroundHealInfo::default(), now + chrono::Duration::hours(2));
|
||||
assert_eq!(later, ScannerBitrotStart::Anchor(now));
|
||||
assert_eq!(get_cycle_scan_mode(4, later, policy), HealScanMode::Normal);
|
||||
|
||||
@@ -11510,7 +11508,7 @@ fn bitrot_cadence_anchor_is_remembered_in_memory_without_a_persisted_write() {
|
||||
#[serial]
|
||||
fn bitrot_start_memory_record_outlives_a_skipped_persisted_save() {
|
||||
reset_bitrot_cadence_anchor_for_test();
|
||||
let now = Timestamp::now();
|
||||
let now = Utc::now();
|
||||
let policy = ScannerBitrotPolicy::new(Some(Duration::from_secs(3600)), true, 1024);
|
||||
let recorded = BackgroundHealInfo {
|
||||
bitrot_start_time: Some(now),
|
||||
@@ -11549,7 +11547,7 @@ fn bitrot_start_memory_record_outlives_a_skipped_persisted_save() {
|
||||
// A newer in-memory start (a Deep window opened after the last
|
||||
// successful save) wins over the stale persisted record.
|
||||
let newer = BackgroundHealInfo {
|
||||
bitrot_start_time: Some(now + jiff::SignedDuration::from_secs(1)),
|
||||
bitrot_start_time: Some(now + chrono::Duration::seconds(1)),
|
||||
bitrot_start_cycle: 9,
|
||||
current_scan_mode: HealScanMode::Deep,
|
||||
};
|
||||
@@ -11576,7 +11574,7 @@ fn bitrot_start_memory_record_outlives_a_skipped_persisted_save() {
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scan_mode_status_reports_bitrot_start_source_and_topology_support() {
|
||||
let now = Timestamp::now();
|
||||
let now = Utc::now();
|
||||
let info = BackgroundHealInfo {
|
||||
bitrot_start_time: Some(now),
|
||||
bitrot_start_cycle: 2,
|
||||
@@ -11597,7 +11595,7 @@ fn scan_mode_status_reports_bitrot_start_source_and_topology_support() {
|
||||
assert_eq!(status.bitrot_start_cycle, 2);
|
||||
assert_eq!(
|
||||
status.bitrot_start_unix_secs,
|
||||
u64::try_from(now.as_second()).expect("current time is after the epoch")
|
||||
u64::try_from(now.timestamp()).expect("current time is after the epoch")
|
||||
);
|
||||
assert_eq!(status.current_scan_mode, "normal");
|
||||
let encoded = serde_json::to_value(status).expect("bitrot status serializes");
|
||||
@@ -11623,13 +11621,13 @@ fn scan_mode_status_reports_bitrot_start_source_and_topology_support() {
|
||||
assert_eq!(status.bitrot_start_cycle, 0);
|
||||
assert_eq!(
|
||||
status.bitrot_start_unix_secs,
|
||||
u64::try_from(now.as_second()).expect("current time is after the epoch")
|
||||
u64::try_from(now.timestamp()).expect("current time is after the epoch")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_erasure_sd_does_not_enter_deep_scan_mode() {
|
||||
let started = Timestamp::now();
|
||||
let started = Utc::now();
|
||||
let cycle = Some(Duration::from_secs(3600));
|
||||
|
||||
let unsupported_policy = ScannerBitrotPolicy::new(cycle, false, 1024);
|
||||
@@ -11661,7 +11659,7 @@ fn test_erasure_sd_does_not_enter_deep_scan_mode() {
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_background_heal_info_for_scan_start_marks_deep_active() {
|
||||
let now = Timestamp::now();
|
||||
let now = Utc::now();
|
||||
let info = background_heal_info_for_scan_start(
|
||||
BackgroundHealInfo::default(),
|
||||
7,
|
||||
@@ -11693,7 +11691,7 @@ fn background_heal_read_failures_never_become_initializable_defaults() {
|
||||
#[serial]
|
||||
fn test_background_heal_info_for_scan_start_keeps_deep_window_start() {
|
||||
with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || {
|
||||
let started_at = Timestamp::now();
|
||||
let started_at = Utc::now();
|
||||
let info = BackgroundHealInfo {
|
||||
bitrot_start_time: Some(started_at),
|
||||
bitrot_start_cycle: 7,
|
||||
@@ -11704,7 +11702,7 @@ fn test_background_heal_info_for_scan_start_keeps_deep_window_start() {
|
||||
info,
|
||||
8,
|
||||
HealScanMode::Deep,
|
||||
Timestamp::now(),
|
||||
Utc::now(),
|
||||
ScannerBitrotPolicy::new(bitrot_scan_cycle(), true, 1024),
|
||||
)
|
||||
.expect("deep scan should mark active status");
|
||||
@@ -11717,7 +11715,7 @@ fn test_background_heal_info_for_scan_start_keeps_deep_window_start() {
|
||||
|
||||
#[test]
|
||||
fn test_background_heal_info_for_scan_complete_marks_deep_idle() {
|
||||
let started_at = Timestamp::now();
|
||||
let started_at = Utc::now();
|
||||
let info = BackgroundHealInfo {
|
||||
bitrot_start_time: Some(started_at),
|
||||
bitrot_start_cycle: 7,
|
||||
@@ -11735,7 +11733,7 @@ fn test_background_heal_info_for_scan_complete_marks_deep_idle() {
|
||||
#[test]
|
||||
fn test_background_heal_info_for_scan_complete_leaves_normal_scan_unchanged() {
|
||||
let info = BackgroundHealInfo {
|
||||
bitrot_start_time: Some(Timestamp::now()),
|
||||
bitrot_start_time: Some(Utc::now()),
|
||||
bitrot_start_cycle: 7,
|
||||
current_scan_mode: HealScanMode::Normal,
|
||||
};
|
||||
@@ -11746,7 +11744,7 @@ fn test_background_heal_info_for_scan_complete_leaves_normal_scan_unchanged() {
|
||||
#[test]
|
||||
fn test_background_heal_info_for_failed_scan_preserves_deep_mode() {
|
||||
let info = BackgroundHealInfo {
|
||||
bitrot_start_time: Some(Timestamp::now()),
|
||||
bitrot_start_time: Some(Utc::now()),
|
||||
bitrot_start_cycle: 7,
|
||||
current_scan_mode: HealScanMode::Deep,
|
||||
};
|
||||
@@ -11756,15 +11754,13 @@ fn test_background_heal_info_for_failed_scan_preserves_deep_mode() {
|
||||
|
||||
#[test]
|
||||
fn test_retain_recent_cycle_completions_keeps_last_entries() {
|
||||
let base = Timestamp::now();
|
||||
let base = Utc::now();
|
||||
let keep = data_usage_update_dir_cycles() as usize;
|
||||
let mut completed: Vec<_> = (0..keep + 2)
|
||||
.map(|i| base + jiff::SignedDuration::from_secs(i as i64))
|
||||
.collect();
|
||||
let mut completed: Vec<_> = (0..keep + 2).map(|i| base + chrono::Duration::seconds(i as i64)).collect();
|
||||
|
||||
retain_recent_cycle_completions(&mut completed);
|
||||
|
||||
assert_eq!(completed.len(), keep);
|
||||
assert_eq!(completed.first().copied(), Some(base + jiff::SignedDuration::from_secs(2)));
|
||||
assert_eq!(completed.last().copied(), Some(base + jiff::SignedDuration::from_secs((keep + 1) as i64)));
|
||||
assert_eq!(completed.first().copied(), Some(base + chrono::Duration::seconds(2)));
|
||||
assert_eq!(completed.last().copied(), Some(base + chrono::Duration::seconds((keep + 1) as i64)));
|
||||
}
|
||||
|
||||
@@ -1279,7 +1279,7 @@ fn recovery_retry_budget_restarts_after_the_daily_window() {
|
||||
}
|
||||
|
||||
fn unix_now_for_test() -> u64 {
|
||||
u64::try_from(jiff::Timestamp::now().as_second()).expect("current time fits")
|
||||
u64::try_from(Utc::now().timestamp()).expect("current time fits")
|
||||
}
|
||||
|
||||
/// Unit view of the automatic rebuild: the first load quarantines and leaves
|
||||
|
||||
@@ -27,11 +27,12 @@ documentation = "https://docs.rs/rustfs-signer/latest/rustfs_signer/"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
hotpath = ["rustfs-utils/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "rustfs-utils/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "rustfs-utils/hotpath-cpu"]
|
||||
hotpath = ["hotpath/hotpath", "rustfs-utils/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-utils/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-utils/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
tracing.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
http.workspace = true
|
||||
|
||||
@@ -30,11 +30,12 @@ workspace = true
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustls/fips"]
|
||||
hotpath = ["rustfs-common/hotpath", "rustfs-config/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "rustfs-common/hotpath-alloc", "rustfs-config/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "rustfs-common/hotpath-cpu", "rustfs-config/hotpath-cpu"]
|
||||
hotpath = ["hotpath/hotpath", "hotpath/tokio", "rustfs-common/hotpath", "rustfs-config/hotpath"]
|
||||
hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "rustfs-common/hotpath-alloc", "rustfs-config/hotpath-alloc"]
|
||||
hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-common/hotpath-cpu", "rustfs-config/hotpath-cpu"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
rustfs-config.workspace = true
|
||||
arc-swap.workspace = true
|
||||
|
||||
@@ -26,19 +26,6 @@ ignore = [
|
||||
# in-process RSA decryption oracles are removed.
|
||||
# owner: rustfs-maintainers review: 2026-07
|
||||
{ id = "RUSTSEC-2023-0071", reason = "rsa Marvin timing sidechannel; no fixed upstream version; tracked separately" },
|
||||
# `chrono` is retained only at compatibility boundaries while the project
|
||||
# migrates its internal time model to jiff. The advisory reports no safe
|
||||
# upgrade for any chrono release.
|
||||
# owner: rustfs-maintainers review: 2026-11
|
||||
{ id = "RUSTSEC-2026-0543", reason = "chrono is unmaintained; temporary exception during jiff migration" },
|
||||
# `chrono-tz` is retained by a narrow compatibility path during the same
|
||||
# migration and has no safe upstream upgrade.
|
||||
# owner: rustfs-maintainers review: 2026-11
|
||||
{ id = "RUSTSEC-2026-0555", reason = "chrono-tz is unmaintained; temporary exception during jiff migration" },
|
||||
# humantime wraps chrono and is retained transitively while the chrono
|
||||
# graph is being removed.
|
||||
# owner: rustfs-maintainers review: 2026-11
|
||||
{ id = "RUSTSEC-2026-0542", reason = "humantime is unmaintained through chrono; temporary exception during jiff migration" },
|
||||
]
|
||||
|
||||
[sources]
|
||||
|
||||
@@ -22,7 +22,7 @@ The outer gate uses the same HTTP/2 and `application/grpc` predicate as the hybr
|
||||
|---|---|---|---|
|
||||
| StorageReady | Storage/global config readiness publication plus runtime storage checks. | Yes. | Startup can mark the stage before later runtime readiness rechecks storage. Initial pool metadata reads must complete before object-store publication; see [pool metadata recovery](../operations/pool-metadata-recovery.md). |
|
||||
| IamReady | Inline IAM bootstrap or deferred IAM recovery publication. | Yes. | Deferred recovery can publish IAM readiness after HTTP has already started. |
|
||||
| Lock quorum | Per-set shared-lock quorum readiness. | Yes. | Startup uses the same read boundary as node readiness. Mutations retain their exclusive-lock requirements. Do not replace the distributed lock quorum check with node count or endpoint count. |
|
||||
| Lock quorum | Per-set write quorum readiness. | Yes. | Do not replace the distributed lock quorum check with node count or endpoint count. |
|
||||
| Peer health | `peer_health_ready` runtime status. | Only when `RUSTFS_HEALTH_PEER_READY_CHECK_ENABLE` is enabled. | The gate is disabled by default; unknown peer health degrades readiness only when enabled. |
|
||||
| Peer re-admission | A strict majority of the peers in each erasure set this node serves count all of its drives in that set as online (`PeerDriveAdmission` internode RPC). | Startup transition only, while `RUSTFS_STARTUP_PEER_READMISSION_ENABLE` is enabled (default on). | Bounded and fail-open: publishes `FullReady` with a WARN log when `RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS` elapses, immediately when older peers cannot answer and the majority is out of reach, or after 15 s of unreachable peers keeping it out of reach. Not part of steady-state `/health/ready`. Distinct from peer health, which is this node's view of its peers. |
|
||||
| KMS compatibility | KMS health compatibility readiness. | Only when `RUSTFS_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE` is enabled (default off; `crates/config/src/constants/health.rs`). | When enabled, `/health/ready` additionally requires the KMS service to be running if a global KMS manager exists. KMS startup fatality and health reporting remain separate from pure docs work. |
|
||||
|
||||
@@ -108,10 +108,6 @@ Claim values are used verbatim as policy names (after `claim_prefix`, if any). N
|
||||
|
||||
For first-contact testing only, `RUSTFS_IDENTITY_OPENID_ROLE_POLICY=consoleAdmin` grants every login full access; remove it before production.
|
||||
|
||||
RustFS rechecks IAM mappings for OIDC sessions and service accounts derived from them. Detaching a group policy or disabling the group removes grants that depended on that mapping. A policy granted independently by the provider still applies. Newly attached policies can deny requests immediately; sign in again to receive new Allow grants and create a new service account if needed.
|
||||
|
||||
When upgrading, update every serving node and replication peer before relying on group revocation. Older nodes evaluate the policy names recorded at sign-in. On updated nodes, grouped credentials issued before policy provenance was recorded require a new login and replacement service accounts.
|
||||
|
||||
### Creating access keys for a local user
|
||||
|
||||
In the Console, open **Users → select the local user → Access Keys → Create**. An unrestricted root Console session or a local or OIDC administrator with the built-in `consoleAdmin` policy can create a key for an enabled local user. The key uses the target user's permissions and any narrower session policy supplied at creation.
|
||||
|
||||
@@ -62,8 +62,6 @@ The native CAS, undo and recovery RPC methods require all participating storage
|
||||
|
||||
A runtime metadata or identity read failure before any write is dispatched rejects that operation but does not permanently block subsequent retries. Errors retain their typed cause; pool metadata unavailability reaches S3 as `503 ServiceUnavailable`, without exposing internal error details. Cancellation during preflight or a rejected first conditional write is also retryable. After any identity, prepare, or commit write may have started, cancellation, an uncertain write result, or abandoned runtime publication blocks further metadata-dependent mutations.
|
||||
|
||||
An existing deployment with committed, initialized identity and unchanged topology can publish its startup metadata under a shared `pool.bin` namespace fence when no writer-side repair or completion is required. Pending bootstrap, identity completion, topology changes, and recovery retain the exclusive writer fence. Observing an initialized identity clears bootstrap provenance, so later initialization attempts use the same read-admission path. Startup releases the shared fence before acquiring the writer and reloads the identity and replicas under exclusive ownership; observations made by the reader never authorize later writes. This avoids requiring write-lock quorum merely to start a readable survivor.
|
||||
|
||||
During startup, an unreadable identity or `pool.bin` replica returns typed `ReadUnavailable` before a partial metadata view is installed or the object store is published. Initialization retries from the complete identity and metadata read at the existing five-second retry interval, within the existing startup attempt budget. A later readable result must still pass the original identity, generation, topology, conditional-write and readback checks. Read-only unavailability does not create a permanent startup block; an existing validation or uncertain-write block is never cleared by a clean retry.
|
||||
|
||||
The node checks for interrupted pool metadata transactions every five seconds. Failed recovery attempts back off to at most sixty seconds; each attempt has a thirty-second budget and stops on shutdown. Healthy nodes do not read metadata for this worker. Recovery:
|
||||
|
||||
@@ -124,7 +124,7 @@ A challenge is `HMAC-SHA256(root_secret, "rustfs-mfa-challenge:v1" ‖ access_ke
|
||||
| `DELETE /v3/user/mfa` | `admin:EnableUser` |
|
||||
| `PUT /v3/set-user-secret-key` | `admin:CreateUser` |
|
||||
|
||||
Eligible callers need no explicit Allow to rotate their password or manage their MFA. A matching Deny on `admin:ChangeOwnPassword` or `admin:CreateUser` blocks self-rotation (including via `add-user`); a Deny on `admin:ManageOwnMFA` blocks enrollment, activation, disabling and recovery-code replacement. The checks use the signing credential, including its applicable session policy. Administrative reset routes still require their ordinary admin permission and also honor the self-service deny when targeting the caller's own identity. Allowing a self-service action grants no authority over another identity.
|
||||
The self-service routes carry no admin action and are registered as `CredentialOnly` in the route-policy matrix. Giving them one would be wrong in both directions: it would stop an ordinary user from changing their own password, and it would let any holder of that action change somebody else's.
|
||||
|
||||
`POST /v3/account/password` and `POST /v3/account/mfa/disable` require a proof-of-knowledge step because a signature only proves a credential was used. The Console signs with a short-lived session, so without it a hijacked browser tab could rewrite the account's credentials or strip its second factor; requiring the password makes disabling the factor as hard as the thing the factor protects.
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user